謜idZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m ZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZmZdd lmZm Z m!Z!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0ddl1m2Z2m3Z3m4Z4e4jje6Z7dZ8dZ9dZ:dZ;dZe3e*Gdde-Z?e?Z@y)z Tokenization classes for fast tokenizers (provided by HuggingFace's tokenizers library). For slow (python) tokenizers see tokenization_utils.py N) defaultdict)Iterable)copyfile)Any)is_offline_mode) AddedToken processors)Encoding) Tokenizer)Decoder)BPEUnigram) BpeTrainerUnigramTrainerWordLevelTrainerWordPieceTrainer cached_file)convert_gguf_tokenizer)load_gguf_checkpoint)INIT_TOKENIZER_DOCSTRING BatchEncodingPreTokenizedInputPreTrainedTokenizerBase TextInputTruncationStrategygenerate_merges)PaddingStrategyadd_end_docstringsloggingztokenizer.jsonzspecial_tokens_map.jsonztokenizer_config.jsonztokenizer.modelzadded_tokens.jsonu tokenizer_object ([`tokenizers.Tokenizer`]): A [`tokenizers.Tokenizer`] object from 🤗 tokenizers to instantiate from. See [Using tokenizers from 🤗 tokenizers](../fast_tokenizers) for more information. tokenizer_file ([`str`]): A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from 🤗 tokenizers. )r r WordLevel WordPiece)tokenizer_file vocab_filec) eZdZdZeZdZdZedLdZ fdZ e de fdZ e de fdZdMd ed edzdeefd Zd Ze d Ze dZej*dZej*dZdZe defdZdeeeffdZe deeeffdZe deeeffdZe deeeffdZeZeZ deeeffdZ!de fdZ"defdZ#e de$fdZ%e de&fdZ' dNde(de dzde dzd e d!e d"e d#e d$e deeee)fe*e(ffd%Z+d&edefd'Z,d(ededzfd)Z-dLd*e*eezdefd+Z.dLd,e defd-Z/dLd.ee*ezd/e dee*ezfd0Z0dOd1ed,edzd2e de*efd3Z1d4e2d5e3d6ed7ed8edzd9edzf d:Z4dde2jje3jldd;ddddddddddddfd1e7e8ze*e7ze*e8zde dzde dzde dzd e d!e d"e d#e d$e d?e dzde9f(d@Z:dAe*edefdBZ; dPdCee*ezd/e dDe dzdefdEZ< dQd ee=j|zdFeedGfdHe dzd edzdeedGff dIZ? dRdJZ@e dSdKZAxZBS)TTokenizersBackendaQ Base class for all fast tokenizers (wrapping HuggingFace tokenizers library). Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`]. Handles all the shared methods for tokenization and special tokens, as well as methods for downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary. This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...). NFc 4 t|}|jdd}|Qtjj |r2|t usd|j vs|rtj||d<|S|Ltjj |r,tj|j}t|d5}tj|}dddjdijdd}|j+t|t rt!t#t$|}n|jj&d k(r4|rt|d t t$fr|D cgc] } t%| }} n|jj&d k(rt)|D cic]\} } | |  }} } nu|jj&d k(s|jj&d k(rCt|t r3t)|D cic]\} } t| t r| d n| | }} } ||d<t+|dd} d|jdivr`| r^| j&d k(rO|dd} | Dcgc]7}t|t,rt%|j/dn t%|9} }| |d<|||d<|S|jd}|jd}|jd}|jd} t|t,rY|j1drHtjj |r)ddlm}||j7|\|d<|d<|St|t,rtjj |r|j1dr ddlm}||j:|jfi|} ddlm}|j|j&}|t?|dr|j@d*i|}t?|dr|jHd*i|}|S|9t|t,r)tjj |r ||d<|d}| 9t|t,r)tjj |r ||d<|d} | |j|jj&d k(rwt|trgd%tLtNd&t t,ffd' gd(}tQ}|D]"}||vs|jS||g$tU||)} | |d<|S#1swY{xYwcc} wcc} } wcc} } wcc}w#tB$r1}tDjGd|j&d|dYd}~d}~wwxYw#tB$r^}tDjGd|d |d!dd"lm%}|||jd#$j7|\|d<|d<Yd}~|Sd}~wwxYw)+zs Build a `tokenizers.Tokenizer` backend from the available serialization files (tokenizer.json, sentencepiece models, tekken.json, vocab/merges). r$N__init__tokenizer_objectutf-8encodingmodelvocabrrr"r r#merges post_processorr% merges_filez tekken.jsonr)MistralConverter)r%.model)SentencePieceExtractor)SLOW_TO_FAST_CONVERTERSconvert_from_spmz,Could not reorder vocab using converter for z due to z/. Falling back to raw SentencePiece extraction.convert_from_spm_modelz+Could not extract SentencePiece model from z$ using sentencepiece library due to z%. Falling back to TikToken extractor.)TikTokenConverterextra_special_tokens)r%r;valuesreturncg}|D]M}|t|ttfr|j|4|j t |O|SN) isinstancelisttupleextendappendstr)r< collectedval_iter_special_tokenss f/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/tokenization_utils_tokenizers.pyrHzHTokenizersBackend.convert_to_native_format.._iter_special_tokenss[') !3C{ !#e}5!(()=c)BC!((S2 3! ) pad_token unk_token bos_token eos_token sep_token cls_token mask_tokenadditional_special_tokensr;) skip_tokens)+dictpopospathisfiler'__dict__ TokenizerFast from_filer2openjsonloadgetr.r@rAmaprB__name__ enumerategetattrrEsplitendswithconvert_slow_tokenizerr4extract_vocab_merges_from_modelr6extractr7hasattrr8 Exceptionloggerwarningr9r:rrsetupdater)clstrust_remote_codekwargs local_kwargsfast_tokenizer_file processortokenizer_handletokenizer_jsonr/itemitoken model_typer0merger%r3r4r6r7converter_classer:special_tokens_keysrSkeyrHs @rIconvert_to_native_formatz*TokenizersBackend.convert_to_native_formatdsF| *../?F +23))Zs||-KO`/--gr:: zObObfkOk'0:kqrbgZs5K% C 01QVW\Q]]rr)/ X&$1: -. !%%l3 "&&}5   )!!(+ j# &:+>+>}+MRTRYRYR`R`akRl @>*+E*J]J]^fJg >JI5jAII#))dWcd  O&=&A&A#,,&OO&2wPb7c'G'G'G'W,'W 3 89#=3#=#=#M #ML  =Z C8RWW^^J=W$.L ! )E >jc:rww~~k?Z%0L "!(+F >cii3 8J8Je8SXbchjnXo !Xc] !tCy ! # %(EK* R,&&&';\#=NA*Mqrsqtu::F@Q) @P@PQg@hA11*=> W%|H'=  >shU0U, U" "U(5tt+| |jdd} n|j t-d|2|0|j$|j/dd|j/dd| | |_|j t-d|jj0}|q|jj2d3i||j/d|d|j/d|d|j/d|d|j/d|d n|jj5|jj6}||jj8d3i||j/d!|d!|j/d"|d#|j/d$|d|j/d|d%|j/d&|d&d'|vrd(|d'<d)|vxsd*|v}|jd)d|_|jd*d|_|jd+dx}r||j_|xs|jj>du|_ tC$|d3i||||_#||_$|jJ|j_&|jNDchc]}tQtS|}}tU|jWd,-Dcgc]\}}tQtS||vr|}}}t'|jXj[|Dcgc] }t]|c}z}|j^jaD])}|t]||vs||vs|jc|+|jdD]&}t]||vs||vs|jc|(t|dkDrg} |j^jaD!cgc]}!|!st]|!}"}!|D]a}t|t\rtg|d.}n0t|tfr |jhst]||"vrd|_4| jc|c| r|jk|  |jjm}#|#d/kDrtq|jd0dk|jdd|jr|j|jtjd df|jt|jd1d2||_|j@xs|jj>du|_ |j@r|jwyycc}}}wcc}wcc}}wcc}wcc}!w#tn$rd}#YwxYw)4Nr* gguf_filer$added_tokens_decoderadd_prefix_spaceFr%r/r0 name_or_pathconfigr{ tokenizertokenizer_configrT)r/r0fuse_unkdropoutunk_id)r/ra9Couldn't instantiate the backend tokenizer from one of: (1) a `tokenizers` library serialization file, (2) a slow tokenizer instance to convert or (3) an equivalent slow tokenizer class to instantiate and convert. You need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one.rMzrNzz3The backend tokenizer is not correctly initialized. max_lengthtruncation_side directionstridetruncation_strategystrategyrKpad_token_type_id pad_type_id padding_sidelengthpad_to_multiple_ofbackend tokenizers add_bos_token add_eos_tokenr2c |dSNrrT)xs rIz,TokenizersBackend.__init__..Ns STUVSWrJr)speciali pre_tokenizerfix_mistral_regex) init_kwargsrrT)   3[5L M OO ) ) +??**   *DOO * * 6X 6   k8K+@ A   18M3J K   nh{.C D   lHX,> ?   2H=Q4R S F " ,F9 %4%>%[/U[B["$jj%@$jj%@#ZZ(8$? ?> ?-;DOO *-G-q4??KiKimqKq* "6"  !(DO 0040I0I-DHD]D]$^5T$u+%6$^!$^!'';'A'A'C X uDK (AA   t005578Ta;b5CJ;bb$(#;#;#B#B#D : "*&'w6;NVc;c$$%89  :// ,E5z(U--G$$U+ , }  !F040H0H0O0O0QW1UVAW W& %eS)&ud;Ez2 ==SZ;K-K(,  e$ %' 779J  74??OT#R#^ JJ{D )7d77  $$^T:!,,"(**-@"A   DO  . . X$//2P2PTX2X *  - -  & & ( .qDl@%_ . dhijdkrJrrrrcontentrvks rIrz&TokenizersBackend.added_tokens_encoders; *00I0I0O0O0QWk)lmA 1 mmmAc6|jjS)z Returns the added tokens in the vocabulary as a dictionary of index to AddedToken. Returns: `dict[str, int]`: The added tokens. )rget_added_tokens_decoderrs rIrz&TokenizersBackend.added_tokens_decoder s7799rJct|jjdDcic]\}}|j|c}}Scc}}w)z Returns the added tokens in the vocabulary as a dictionary of token to index. Returns: `dict[str, int]`: The added tokens. c |dSrrTrs rIrz3TokenizersBackend.get_added_vocab.. rrJrrrs rIget_added_vocabz!TokenizersBackend.get_added_vocabs;*00I0I0O0O0QWk)lmA 1 mmmrcy)zN Returns True, to avoid expensive `assert tokenizer` gotchas. TrTrs rI__bool__zTokenizersBackend.__bool__"srJc:|jjdS)zD Size of the full vocabulary with the added tokens. Trr rs rI__len__zTokenizersBackend.__len__(s---EErJc|jS)zc `tokenizers.implementations.BaseTokenizer`: The Rust tokenizer used as a backend. )rrs rIbackend_tokenizerz#TokenizersBackend.backend_tokenizer.s rJc.|jjS)zU `tokenizers.decoders.Decoder`: The Rust decoder for this tokenizer. )rdecoderrs rIr#zTokenizersBackend.decoder5s &&&rJTr-return_token_type_idsreturn_attention_maskreturn_overflowing_tokensreturn_special_tokens_maskreturn_offsets_mapping return_lengthverbosec J|d|jv}|d|jv}|r|j|g|jz} n|g} tt} | D]} | dj | j |r| dj | j |r| dj | j|r| dj | j|r| dj | j|s| dj t| j | | fS)a Convert the encoding representation (from low-level HuggingFace tokenizer output) to a python Dict and a list of encodings, take care of building a batch from overflowing tokens. Overflowing tokens are converted to additional examples (like batches) so the output values of the dict are lists (overflows) of lists (tokens). Output shape: (overflows, sequence length) token_type_idsattention_mask input_idsspecial_tokens_maskoffset_mappingr) model_input_names overflowingrrArDidstype_idsr-r/offsetsr) rr-r$r%r&r'r(r)r* encodings encoding_dictr~s rI_convert_encodingz#TokenizersBackend._convert_encoding<s$( ! ($48N8N$N ! ($48N8N$N ! $)=)=)I! X%9%99I! I#D)  ;A + & - -aee 4$./66qzzB$./66q7G7GH)34;;A This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put this inside your training loop. Args: pair (`bool`, *optional*, defaults to `False`): Whether the number of added tokens should be computed in the case of a sequence pair or a single sequence. Returns: `int`: Number of special tokens added to sequences. )rnum_special_tokens_to_add)rrs rIrFz+TokenizersBackend.num_special_tokens_to_addzs&88>>rJr3skip_special_tokensc$t|tr|jj|Sg}|rt |j n t }|D]<}t|}||vr|j |jj|>|S)a Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and added tokens. Args: ids (`int` or `list[int]`): The token id (or token ids) to convert to tokens. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. Returns: `str` or `list[str]`: The decoded token(s). )r@r?rr>rnall_special_idsrD)rr3rGr ids_to_skiprs rIconvert_ids_to_tokensz'TokenizersBackend.convert_ids_to_tokenss c3 ??..s3 33Fc$../CE  >EJE # MM$//55e< =  >  rJtextrCc J|jd|||d|jS)N)rL text_pairrCrT) _encode_plusr)rrLrrCrrs rItokenizezTokenizersBackend.tokenizes, t  lddOaleklssuurJpadding_strategyrrrrrc|jj}|jj}|tjk(r|||jj na|||j |jd} |d} n | D cic]} | |j| d} } | | k7r|jjdi| |tjk(r||jjyy|tjk(r|nd} | ||n |j|j|j |j"|d} || k7r|jj$di| yycc} w)a Define the truncation and the padding strategies for fast tokenizers (provided by HuggingFace tokenizers library) and restore the tokenizer settings afterwards. The provided tokenizer has no padding / truncation strategy before the managed section. If your tokenizer set a padding / truncation strategy before, then it will be reset to no padding / truncation when exiting the managed section. Args: padding_strategy ([`~utils.PaddingStrategy`]): The kind of padding that will be applied to the input truncation_strategy ([`~tokenization_utils_base.TruncationStrategy`]): The kind of truncation that will be applied to the input max_length (`int`): The maximum size of a sequence. stride (`int`): The stride to use when handling overflow. pad_to_multiple_of (`int`, *optional*): If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta). padding_side (`str`, *optional*): The side on which the model should have padding applied. Should be selected between ['right', 'left']. Default value is picked from the class attribute of the same name. N)rrrr)rrpad_idrKrrrT)rrrrDO_NOT_TRUNCATErrrr`rr DO_NOT_PAD no_padding MAX_LENGTHr pad_token_idrKrr) rrQrrrrrrrtargetcurrentrrs rIset_truncation_and_paddingz,TokenizersBackend.set_truncation_and_paddingsZBoo00 ??** "4"D"D D&--/) /55!11 F"@FG11kooa66GG& 111;F; 99 9#**,$$47Q7Q#QZW[F -9-E\4K\K\++!^^#55&8 F6!...88"%HsErrNis_split_into_wordsreturn_tensorsrc d}||s td|||s td|r6t|ttfxr|xrt|dttf}nt|ttf}|rrt|tr t d|;t |t |k7r$tdt |dt |d|tt||n|}n |r||fgn|g}t|ttfst dt|d |j||||| | | |j}|jj|k7r||j_ |jj||| }|Dcgc]}|j|| | ||||| }}i}|ddD]'}|Dcgc]\}}||D]}|} }}}| ||<)|Dcgc]\}}|D]}|}!}}}|r2g}"t|D]\}#\}$}|"|#gt |$d zz }"|"|d<|d D]}%|j!|%||t#||!| }&|sb| `|s^t#|&j%D'cic].\}}'|t |'dkDrt|'dtr|'dn|'0c}'}|&j&}&|&Scc}wcc}}}wcc}}}wcc}'}w)Nct|tryt|ttfrt |dk(ryt|dtryt|dttfrtt |ddk(st|ddtryt|ddttfr/t |dddk(xst|dddtSyyy)NTrF)r@rErArBr)rs rI_is_valid_text_inputz._is_valid_text_inputs!S!Ae}-q6Q;!c*!tUm41Q4yA~AaDGS)A##AaDGdE];"1Q47|q0OJqtAwqz34OO$ rJztext input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs).rzdwhen tokenizing batches of text, `text_pair` must be a list or tuple with the same length as `text`.zbatch length of `text`: z- does not match batch length of `text_pair`: .z:batch_text_or_text_pairs has to be a list or a tuple (got ))rQrrrrr)rCis_pretokenized)r-r$r%r&r'r(r)r*r.overflow_to_sample_mapping) tensor_type)rr@rArBrE TypeErrorrziptyper[rrr encode_batchr8rc&_eventual_warn_about_too_long_sequencerrr6)(rrLrNrCrQrrrr\rrr]r$r%r&r'r(r)r*rrrr` is_batchedbatch_text_or_text_pairsr6r-tokens_and_encodingssanitized_tokensrrxrr~stacksanitized_encodingsrdrytoksr.batched_outputrs( rIrOzTokenizersBackend._encode_pluss0 ($D)W   )=i)HW  #D4-8hThjQUVWQX[_afZgFhJ#D4-8J )S)$Tc)n)D .s4yk:I'q*FOEZtCi,@'A`d $?Hy(9':dV $2UDMBLTRjMkLllmn  ''- 3!1% (  '#'#<#< ?? 0 04H H4HDOO 1OO00 $1/1 $&   " "!&;&;*C+E'=+ #   '*1- *C&:NN74DINqQNQNEN$) S ! *1ESSWQdSqSqSS %)+ & )*> ? K 9D!*qcC[8I4J.JJ* K=W 9 :)+6 XI  7 7 :w W X''79LZhin4=V*'5&:&:&<"Uc%j1nE!Hd9S%(Y^^(( NW "OS"s K K 3K3K" rc|jj%|jjj|Sdj|S)Nr1)r!r#decoder)rrs rIconvert_tokens_to_stringz*TokenizersBackend.convert_tokens_to_stringsJ%%--9  " " * * 1 1& 9 &! rJ token_idsclean_up_tokenization_spacesc |jddt|tr|g}t|tr|d}|jj ||}||n |j }|rt|dr(t|jr|j|}|S|jddjddjd d jd d jd djddjddjddjddjdd}|S)Nuse_source_tokenizerr.)rGclean_up_tokenizationz .raz ??z !!z ,,z ' 'z n'tzn'tz 'mz'mz 'sz'sz 'vez'vez 'rez're) rVr@r?rUrrtrwrjcallablerzreplace)rrvrGrwrrrLs rI_decodezTokenizersBackend._decodes)  )40 i %" I i &!+.I%%iEX%Y,7 )22 % (t45(4C]C]:^11$7  LLs+WT3'WT3'WT3'WUC(WVU+WUD)WUD)WVU+WVU+ rJ file_names. legacy_formatct|}tjj||r|dzndtz}|j j |||fz}|S)Nrr)rErWrXrTOKENIZER_FILEr!save)rrrrrr$s rI_save_pretrainedz"TokenizersBackend._save_pretraineds]^, o_s22Q__  ##N3>"33 rJc ` tj|jj}|j d}|j d} d} |dddk(ri|dd<g|dd<np|ddd k(r=|dd ]|dd } |dd| d } | | |vr|| } d |dd <| d gg|dd<n(|ddd vr i|dd<nt d|ddd|"d|dvr|dd|vr||dd|dd<t jtj|} g} |D]b}|j dd}|j dd}|ddd k7r|s5||d|vr ||d|d<| jtd(i|d|| j||dddk(rd|vr|dd |dd|d<|dddk(rd|vr|dd |dd|d<|ddd k(r| | |d<|dV|dddk(s*|dddk(r@d|dvr9td|ddDr!tjj|d<t |dd}|d(|| d|}| j#|||| &tj| j}d| vr| dD]}| d|d}||Dcgc]}|j%||}}|| d|d<|D] }| j'|}|t d |Dcgc]}| j'|c}| d|d!<d"D]?}|| vs| |\}}| ||vr||}| j'|}| t d ||g| |<A| |d<t jtj|} |j(j+}t,j.D]}t1||t1||}| ||vr||}|j2j%|d}t5|tr=t||j6|j8|j:|j<d#$||<|||<|j>r|j>j+ng}||j|tA|d kDr||d%<| |d&< |jBd(i|Scc}wcc}w#tD$rE}d'tG|vr2|j d&d|jBd(i|}| |_|cYd}~Sd}~wwxYw))uf Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline) as the current one. Args: text_iterator (generator of `list[str]`): The training corpus. Should be a generator of batches of texts, for instance a list of lists of texts if you have everything in memory. vocab_size (`int`): The size of the vocabulary you want for your tokenizer. length (`int`, *optional*): The total number of sequences in the iterator. This is used to provide meaningful progress tracking new_special_tokens (list of `str` or `AddedToken`, *optional*): A list of new special tokens to add to the tokenizer you are training. special_tokens_map (`dict[str, str]`, *optional*): If you want to rename some of the special tokens this tokenizer uses, pass along a mapping old special token name to new special token name in this argument. kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the trainer from the 🤗 Tokenizers library. Returns: [`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one, trained on `text_iterator`. added_tokensr2Nr.rhr r/r0rrrg)r"r#z;This method does not support this type of tokenizer (found z-) only BPE, Unigram, WordLevel and WordPiece.rLridrcontinuing_subword_prefixend_of_word_suffixr ByteLevelSequence pretokenizersc3,K|] }|ddk(yw)rhrNrT).0 pretokenizers rI z.;s"$!(K7sinitial_alphabet)rr)rtrainerrrzQAttempted to set a token in the post processor that does not exist in the mappingr3)rpsepT) single_wordlstriprstriprrr;r*z7multiple values for keyword argument 'tokenizer_object'rT)$r^loadsrto_strrVrr[from_strdumpsrDrrCanypre_tokenizers_fastralphabetMODEL_TO_TRAINER_MAPPINGtrain_from_iteratorr`r:rrrSPECIAL_TOKENS_ATTRIBUTESrdrr@rrrrr;rrrfrE)r text_iteratorrrnew_special_tokensspecial_tokens_maprrrwrr2rLrrr added_tokenrr trainer_classrtrained_tokenizer_jsonrrrztoken_id special_tokenspecial_token_fullr;r~ new_tokenizers rItrain_new_from_iteratorz)TokenizersBackend.train_new_from_iteratorsDDOO$:$:$<=%)).9 '++,<= ' "6 *e 3/1N7 #G ,02N7 #H - G $V , 9g&x0<'0:*73G4D3Ew'0 G $V ,0J J/1N7 #G ,Mn]dNeflNmMno>>   *~g66w' 48JJ3EnU\F]^iFj3kN7 #K 0!**4::n+EF ' =K!ooi6Gd+Ag&v.);G!-+i2HL^2^);K 1)*:; vC+,<=cB8LF)5TZ![5"4"8"8"F![![FLN#34S9(C!'#,#8#8#?#+", s#ouCuejIDYDYZ_D`CuN#34S9%@ v"0 F  N2-m>+F+ +{"\Dvj HCPQFR -t4 . 8 8 +4 ($$ s0SSS T-(9T(!T-'T((T-c ddlddlmddlm} ddlm} dtdtffd } trd }|*|s |s%| |r| |d |||d d | }d }|t|d 5}tj|}dddjd}|jd}|r-| j|| jdkr |r/|-|dvr)|S|r%| j|| jdkDr|Sd }|s |s^| |rU|rd|vrt|d|d| 5t!|dd s(t|dd t"j%d|d|S| d us t!|dd rt|dd ddl}|j(j+|j-dd}|j.j0}t3||j(j4r||j.j0d<|St3||j(j6r|j(j9d d }|j(j5||g|j._|S#1swYxYw)af Patches mistral related tokenizers with incorrect regex if detected 1) Local file with an associated config saved next to it >> Model type one of the mistral models (on older versions) 2) Remote models on the hub from official mistral models >> Tags including `base_model:.*mistralai` rN) model_info)versionrmodel_idr=c|}|j,jddj|jryy)Nzbase_model:.*mistralairTF)tagssearchr)rr.rres rIis_base_mistralz?TokenizersBackend._patch_mistral_regex..is_base_mistrals9x(Ezz%995rwwuzz7JKrJTz config.jsonF) cache_dirrzlocal_files_only%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errors _commit_hashr+r,transformers_versionr{z4.57.2)mistralmistral3voxtral ministralpixtralz4.57.3rz$The tokenizer you are loading from 'a' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.z[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+isolated)patternbehavior)r use_regex)rhuggingface_hubr packagingrtransformers.utils.hubrrEboolrr]r^r_r`parsesetattrrdrlrmrpre_tokenizersSplitRegexr!rr@r Metaspacer)rprpretrained_model_name_or_pathrzrrris_localrrrrrrr _config_filemistral_config_detectedf_configrtransformers_model_typersplit_pretokenizercurrent_pretokenizerrrs @@rIrz&TokenizersBackend._patch_mistral_regexs* .%6 c d   H ( 4 X/:W*X&-#!16;8=) L', #',9+Q"iilG+'.{{3I'J$*1++l*C' (GMM:N,OSZS`S`aiSj,j 3?3  )()gmm?\>]^eeH?'$.')EXZ_2`I':DA%)3)B)B)H)H * 0 0s!", *I*& ,5+F+F+T+T(!"6 8Q8Q8Z8Z[GY 33AA!D"&&:JA tCH~  nd38nnn:d3 ?&;::10nc3hn$ FF = '''.2-1*/+0',#-(-( $d{-( $d{ -( $( -( %) -(!%-(-(-( tCH~tL11 2-(^ 7#7#*76d3+;&<6WZ6 ?d?s?*tCyt`cfjknfo`o4vSvd vtvjnorjsvI9)I90I9 I9  I9  $J I9DjI9\gk#',;,F,F2D2T2T!%$))-#'&*-1-1*/+0',#,0)X++d9o=EV@WWX004 ?BTJ[E\\_ccX! X * X 0 X$JXX"X $JXDjXt X $d{X $d{X$(X %)!X"!%#X$%X&'X(#Tk)X, -Xt tCy S %*48 (c?("('+Tk ( (\&*&* bkk)#s(Od{  t  sCx * CJ }}rJr')Arrr^rW collectionsrcollections.abcrshutilrtypingrtokenizers.pre_tokenizersrrrrrrr r rr r[tokenizers.decodersr rtokenizers.modelsr rtokenizers.trainersrrrrrrintegrations.ggmlrmodeling_gguf_pytorch_utilsrtokenization_utils_baserrrrrrrutilsrr r! get_loggerrbrlrSPECIAL_TOKENS_MAP_FILETOKENIZER_CONFIG_FILETIKTOKEN_VOCAB_FILEADDED_TOKENS_FILErrr'PreTrainedTokenizerFastrTrJrIrs  #$7+-/16*^^.5=@?   H %"3/'( !! (6EXY,-C/C.CN&,rJ