謜i, UddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddlm Z ddlmZmZddlmZddlmZmZddlmZddlmZmZdd lmZdd lmZdd lmZm Z m!Z!dd l"m#Z#ddl$Z$dd l%m&Z&m'Z'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z/ddl$m0Z0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z8ddl9m:Z:ddl;mZ>m?Z?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGmHZHddlImJZJmKZKmLZLmMZMddlNmOZOmPZPmQZQmRZRmSZSmTZTmUZUddlVmWZWddlXmYZYdd lZm[Z[dd!l\m]Z]dd"l^m_Z_dd#l`maZadd$lbmcZcdd%ldmeZedd&lfmgZgdd'lhmiZimjZjmkZkmlZlmmZmmnZnmoZodd(lpmqZqdd)lrmsZsmtZtdd*lumvZvdd+lwmxZxdd,lymzZzdd-l{m|Z|dd.l}m~Z~dd/lmZdd0lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd1lmZmZdd2lmZmZmZdd3lmZmZmZdd4lmZmZdd5lmZmZdd6lmZer dd7lmZdd8lmZe$jjeZer7ddlmcm$Zdd9lmZe*jtee*jtd:k\Znd;ZejxeZej~jdd=jZe d?d@AZd;ad;adBdCdDZedEFGdGdHZdIZedJZedKZeddLe$jdMedzfdNZdOZdPZe$je$je$je$je$je$je$je$je$je$je$je$je$je$je$jdQZ ddReejzdSee$jzdTedUeee$j`ffdVZdWe$j`dUefdXZdYe1jdUeefdZZd[eeed\eee$j`fdUeeeeeeffd]Zd[eeed\eee$j`fdUeeeeeeffd^Zd\eee$j`fd_d@dUeee$j`ffd`Zd_d@daedWe$j`fdbZddceddedzdUefdeZ ddfeejzdzddedzdgedzdhedzdiedzdjedkedzdledzdUeeedzedzffdmZ ddLee$jzezdzdneedzdoe:dpedzd\edzdTedqezdzdUee:e$jffdrZGdsdteZGdudvZGdwdxZGdyd@e1jeeeeJZeeje_ejj;ejjjd_dzd{|ej_dd_e1jd}edUe1jfd~Zdeeze$jzdUefdZ dd_ededqezdzfdZd_ededqezdzfdZGddeZeZeed<GddeZy)N)abstractmethod) defaultdict)CallableIterator)contextmanager) dataclassfield)Enum)partialwraps)cycle)Thread)OptionalTypeVarget_type_hints) is_zipfile) create_repois_offline_mode"split_torch_state_dict_into_shards)version) safe_open) save_file)Tensornn) constraints) checkpoint)initialization)PreTrainedConfig)get_model_conversion_mapping)WeightConverterWeightRenaming$convert_and_load_state_dict_in_modelrevert_weight_conversion)DistributedConfig)custom_object_save) CompileConfigGenerationConfig)PeftAdapterMixindeepspeed_configis_deepspeed_zero3_enabledis_fsdp_enabled)_get_device_mapaccelerate_disk_offloadaccelerate_dispatchcheck_and_set_device_mapexpand_device_map get_deviceload_offloaded_parameter)!_load_state_dict_into_zero3_model)eager_paged_attention_forward)flash_attention_forward)paged_attention_forward)flex_attention_forward) is_kernel)maybe_load_adapters)sdpa_attention_forward)sdpa_attention_paged_forward)ALL_PARALLEL_STYLES_get_parameter_tp_plandistribute_modelgather_state_dict_for_saveinitialize_tensor_parallelismshard_and_distribute_moduleverify_tp_plan) LOSS_MAPPING)lazy_import_flash_attention!lazy_import_paged_flash_attention)ROPE_INIT_FUNCTIONS)id_tensor_storage) HfQuantizer)get_hf_quantizer)get_module_from_name)auto_conversion)ADAPTER_SAFE_WEIGHTS_NAME DUMMY_INPUTSSAFE_WEIGHTS_INDEX_NAMESAFE_WEIGHTS_NAMEWEIGHTS_INDEX_NAME WEIGHTS_NAMEContextManagers KernelConfigPushToHubMixin cached_filecheck_torch_load_is_safe copy_funchas_fileis_accelerate_availableis_bitsandbytes_availableis_env_variable_trueis_flash_attn_2_availableis_flash_attn_3_availableis_grouped_mm_availableis_kernels_availableis_torch_flex_attn_availableis_torch_mlu_availableis_torch_npu_availableis_torch_xpu_availablelogging)GeneralInterfaceis_flash_attention_requested)DownloadKwargscreate_and_tag_model_cardget_checkpoint_shard_files)#is_huggingface_hub_greater_or_equalis_sagemaker_mp_enabled is_tracing)LoadStateDictInfolog_state_dict_report)_CAN_RECORD_REGISTRYOutputRecorder)QuantizationMethod)add_hook_to_module)extract_model_from_parallel) __version__z1.10F XLA_USE_BF160XLA_DOWNCAST_BF16SpecificPreTrainedModelTypePreTrainedModel)boundzkernels-community/flash-attn2z"kernels-community/vllm-flash-attn3)flash_attention_2flash_attention_3T)frozencdeZdZUdZdZedzed<eeZ edzed<dZ e dzed<dZ e ed<dZ edzed <dZedzed <dZedzed <dZe ed <dZej$dzed <eeZeed<dZedzed<dZeded<dZe ed<dZeeezdzed<ede fdZy)LoadStateDictConfigze Config for loading weights. This allows bundling arguments that are just passed around. Npretrained_model_name_or_path)default_factorydownload_kwargsuse_safetensorsFignore_mismatched_sizessharded_metadata device_mapdisk_offload_folderoffload_buffersdtype dtype_plan hf_quantizerz(torch.distributed.device_mesh.DeviceMesh device_meshT weights_onlyweight_mappingreturnc|jduSN)rselfs W/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/modeling_utils.py is_quantizedz LoadStateDictConfig.is_quantizeds  ,,) __name__ __module__ __qualname____doc__rstr__annotations__r rhrrboolrrdictrrrrtorchrrrIrrrrlistr!r"propertyrrrrrs 15!3:4-2>-RO^d*R#'OTD['$)T)$(dTk("Jt "&*t*!OT! $E5;; $T2J2'+L+$+HLKDELL$DHND>9:TAH -d--rrctjjxrLtjjxr,t t j jdddk(S)N LOCAL_RANKz-1r)r distributed is_availableis_initializedintosenvirongetrrris_local_dist_rank_0rsQ &&( 9    , , . 9  |T2 3q 8rc#,Kda dday#dawxYwwNTF) _is_quantizedrrrset_quantized_statersM    c#,Kda dday#dawxYwwr)_is_ds_init_calledrrrset_zero3_staters"# "Urrmodel_class_namec#*K|js&||d|d}t|d|d}t|tj} tj|dtj|y#tj|wxYww)z Locally change the torch default dtype to `dtype`, and restore the old one upon exiting the context. If `model_class_name` is provided, it's used to provide a more helpful error message if `dtype` is not valid. Nz% cannot be instantiated under `dtype=z$` as it's not a floating-point dtypez Cannot set `z7` as torch's default as it's not a floating-point dtype)is_floating_point ValueErrorrget_default_dtypeset_default_dtype)rr error_messageoriginal_dtypes rlocal_torch_dtypers  " "  '#$$I%Ptu  ''+5'1hiM'',,.N0 &  //sAB A9#B9BBctjgj}tj}||k(r|tjdk7r|Sy|S)z Test if a device context manager is currently in use, or if it is not the case, check if the default device is not "cpu". This is used to infer the correct device to load the model on, in case `device_map` is not provided. cpuN)rtensordeviceget_default_device)device_in_contextdefault_devices r*get_torch_context_manager_or_global_devicersO  R(//--/NN* U\\%0 0! ! rc|jD] }|js|jcSt|dk(rtj St t|jjS)zt Returns the first found floating dtype in `state_dict` if there is one, otherwise returns the first dtype. r)valuesrrlenrfloat32nextiter) state_dictts rget_state_dict_dtypersh      77N  :!}} Z&&() * 0 00r)BOOLU8I8I16U16F16BF16I32U32F32F64I64U64F8_E4M3F8_E5M2checkpoint_file map_locationrrcX|jdrt|d5}i}|jD]}|dk(rk|j|}|j }|t vr t |}nt d|tj|j|d||<s|j|j|||<|cdddS|r ti} t|tr|dk7rt|rdd i} tj |f||d | S#1swYTxYw) zg Reads a `safetensor` or a `.bin` checkpoint file. We load the checkpoint on "cpu" by default. .safetensorspt) frameworkmetaz)Cannot load safetensors of unknown dtype )sizerrNmmapTrr)endswithrkeys get_slice get_dtypestr_to_torch_dtyperrempty get_shape get_tensortorW isinstancerrload) rrrfrk_slicek_dtyper extra_argss rload_state_dictr&s./ $ 7 1JVVX E6)[[^F$..0G"44 27 ;(+TU\T])^__$)KKV5E5E5Gu]c$dJqM$%LLO$6$6|$DJqM E    "J/3'LF,BzRaGbd^ ::o jL| j_i jj/  s B,D  D)rc|jr5|jddj|jz}|S|j}|S)N)nelementviewdata_ptr element_size)rstops r_end_ptrrHsO {{2r"++-0C0C0EE K  Krmodulec g}|jD]L\}}t|dixsi}|j|jDcgc] }|r|d|n|c}N|Scc}w)N_tied_weights_keys.) named_modulesgetattrextendr)rtied_weight_keysname submoduletiedrs r_get_tied_weight_keysr Qsx"$!//1Uiy"6;Artyy{ S!$D61#A!= STU !TsA$ tensorsrc0g}|D]}t|dkr|j|#g}|D]2}||}|j|jt||f4|j |d\}}} |j| h|ddD]4\} } }| |k\r|j|hn|dj || }6g} g} |D]A}t|dk(r | j|j 1| j|C| | fS)Nrrr)rappendrrsortaddpop)rrfiltered_tensorssharedareasr r_ last_stop last_namestartrdisjoint_tensorsshared_tensorss r_find_disjointrYsA v;?  # #F +  FD%F LL&//+Xf-=tD E F  "'(9i ,!&qr  E4 ! ''/ $((.I  &N#+ w<1   # #GKKM 2  ! !' * + + ++rc^g}g}|D]}t|dkrtjt}|D]A}||}|j|j t |f}||j|Ct|dk(r|j||j|||fS)Nrr) r collectionsrsetrrrrr) rrr identicalrrr rareas r_find_identicalr$xsNI * v;? '', "D%FMM6??#4hv6FGD $KOOD ! " u:?   V $  ! !& ) * 9 $$rmodelctjt}|jD]\}t |t j s|t|j>|jjdk(r/|j}|t|j|t|j|jDcic]\}}t|dkDs||}}}tt|}g}t} |h|j!D]U}d} t#|D]Ct%fd|D} | s|vs| dz } | t|ks3| j'EWt)|j!|\} } | D]|j+|<t-| |\} }|D]N}|j/| }|D]|=|j1| }t|dkDs>|j|P| r|j3| t|dkDrt5d|d|d|Scc}}w)aY Remove all tied weights from the given `state_dict`, making sure to keep only the main weight that `model` will expect when reloading (even if we know tie weights symmetrically, it's better to keep the intended one). This is because `safetensors` does not allow tensor aliasing - so we're going to remove aliases before saving. rrrc3JK|]}tj|ywrresearch).0patr s r z6remove_tied_weights_from_state_dict..s%fsbiiT&:%fs #z8The weights trying to be saved contained shared tensors z\ which are not properly defined. We found all the potential target tied weights keys to be: zo. This can also just mean that the module's tied weight keys are wrong vs the actual tied weights in the model.)r rritemsrrridrrtype get_parameterrHrr!r rsortedanyrrcloner$ intersection differencer RuntimeError)rr%ptrsrptrnames shared_ptrsall_potential_tied_weights_keys error_namesto_delete_namesfoundmatches_pattern shared_namesdisjoint_namesidentical_namesinamesknownunknownr s @r#remove_tied_weights_from_state_dictrGs  " "4 (D"((*9 f&%,,/ F  # #D ) ]]  6 )((.F F  # #D ) "6* + 2 24 89"15 O*#uE Q3:OKO'**?*F&G#KeO'2 '') 2EEu  2"%%fFe%f"f"tz'9QJEs5z)'++D1  2 2$2+2D2D2F #S L.4%d+113 44%4L*$M!L/!(##O4 !D4  !##O4 w? -9!ww||,I9Vtu ;DDE^_  -"''.. :I|TegnGop3 "ww||19lK\^e>f  !-"''.. :I|TkmtGuv3 "ww||19lKbdk>l  " $ :I|T`biGjk* "ww||19la  % :I|TfhoGpq* "ww||19lK]_f>g  "  *<8I7+S*TU56a9 *<8I7+S*TTYZfgsu|Z}Y~++H*ILWW^^BGGLL4QR S8LH.99;DDE^_  -'(97C' g>%"&$4 O#1(&499> + """ $%%$,-MNN$'&$O Y )44QS[(r_q(r%)0XN_ahAi5i,75$% ,\7 CDE 'y(ce &<=> ,\7 CDDVgipIqHrrsu$  KK/ ~> ?$0 ! KK/zI^H_` a  77>>) $$- ! '"0"$4($&499> + " %00My$o\n$o !-G ) !)-!$ . ** - --7T6_12ei - --{  01N0OP99V8WX::F|U\:]9^^_a   sFZ[  $[[ rconfigrrc|du}|t|tr|dk(rt|dr2|j&|j}tj d|dnx|r d|vr|d}n*| t |}nt|dd|}t |}tj d n,tt|rtt|}n td t|trtt|n|}nBt|ttjfs"td |tj}||j|t|tr`|jd tj}t|trtt|n|}tjd |dn|}||_|j D]} t|| x} || _||fS)aFind the correct `dtype` to use based on provided arguments. Also update the `config` based on the inferred dtype. We do the following: 1. If dtype is "auto", we try to read the config, else auto-detect dtype from the loaded state_dict, by checking its first weights entry that is of a floating type - we assume all floating dtype weights are of the same dtype 2. Else, use the dtype provided as a dict or str NautorzWill use dtype=z$ as defined in model's config objectrrrz{Since the `dtype` attribute can't be found in model's config object, will use dtype={dtype} as derived from model's weightsze`dtype` provided as a `str` can only be `'auto'`, or a string representation of a valid `torch.dtype`z`dtype` can be one of: `torch.dtype`, `'auto'`, a string of a valid `torch.dtype` or a `dict` with valid `dtype` for each sub-config in composite configs, but received rdzUsing different dtypes per module is deprecated and will be removed in future versions Setting different dtypes per backbone model might cause device errors downstream, therefore setting the dtype=z for all modules.)rrhasattrrrsrtrrrrrrr update_dtyper warning_once sub_configs) rrrrrrrru main_dtypesub_config_key sub_configs r _get_dtypers"-J  eS !67+ 0H"LLEKK/%8\ ]^!g1A&A 0 9#/ 4Z @%4,Q/fS_& !5Z @KKQ&u- { .8s-CGE5)EED%++#67JJOR  '')!!%(%YYr5#:#:#<= 3=j#3NWUJ/T^  !!+ ,= ?  FL ,,*!&.9 9J F)J * : rceZdZdZdZy)PipelineParallelrrN)rrrinputsoutputsrrrrrFs FGrrc eZdZdZedej fdZedejfdZdedefdZ e dZ dd ed e e d fd ejdzdefd Zddedede fdZy)ModuleUtilsMixinzH A few utilities for `torch.nn.Modules`, to be used as a mixin. rcBtd|jDS)z `torch.device`: The device on which the module is (assuming that all the module parameters are on the same device). c34K|]}|jywrrr+params rr-z*ModuleUtilsMixin.device..Vs@UELL@sr parametersrs rrzModuleUtilsMixin.devicePs @doo.?@@@rcBtd|jDS)zw `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). c3VK|]!}|js|j#ywr)rrrs rr-z)ModuleUtilsMixin.dtype..]s\E%BYBYB[EKK\s))rrs rrzModuleUtilsMixin.dtypeXs \T__->\\\rencoder_attention_maskc |jdk(r|dddddddf}|jdk(r |ddddddf}j|j}d|z tj|jj z}|S)z Invert an attention mask (e.g., switches 0. and 1.). Args: encoder_attention_mask (`torch.Tensor`): An attention mask. Returns: `torch.Tensor`: The inverted attention mask. Nrr?)dimrrrfinfomin)rrencoder_extended_attention_masks rinvert_attention_maskz&ModuleUtilsMixin.invert_attention_mask_s " % % '1 ,.DQaQR].S + ! % % '1 ,.DQdTUEU.V ++J*L*LSWS]S]*L*^'+.1P+PTYT_T_`d`j`jTkToTo*o'..rc|j}|\}}tj||}|ddddfj||d|ddddfk}|j |j }|j d|j dkr[|j d|j dz }tjtj|||f||j |gd}|dddddddf|ddddddfz}|S)Nrrrrraxis) rrarangerepeatrrshapecatones) input_shapeattention_maskr batch_size seq_lengthseq_ids causal_maskprefix_seq_lenextended_attention_masks r*create_extended_attention_mask_for_decoderz;ModuleUtilsMixin.create_extended_attention_mask_for_decoderus&&!, J,,z&9dD!m,33J ANRYZ^`acgZgRhh !nn^%9%9:   Q ."6"6q"9 9+11!4{7H7H7KKN))JJ JGPV^i^o^op K#.aq!m"<~aQUW[]^N^?_"_&&rNrr.rc| |j}|jdk(r|dddddddf}nk|jdk(r|r@|jDcgc]%\}}t|tjs!|d'}}}t |dd}|rddl}d}|j D]\}} |r|vr | js|r|r|t| jjr\t| dr| j} n%t| dr| jj} nd} || jd z| zz }|| jz }|Scc}}w) a Get number of (optionally, trainable or non-embeddings) parameters in the module. Args: only_trainable (`bool`, *optional*, defaults to `False`): Whether or not to return only the number of trainable parameters exclude_embeddings (`bool`, *optional*, defaults to `False`): Whether or not to return only the number of non-embeddings parameters Returns: `int`: The number of parameters. z.weightis_loaded_in_4bitFrNr quant_storagerr)rrr Embeddingr bitsandbytesnamed_parametersrJ Params4bitrrritemsizenumel) rrrr  module_typeembedding_param_namesrbnb total_paramsr num_bytess rnum_parameterszModuleUtilsMixin.num_parameterss! :>:L:L:N%%6T;R\]hjljvjvRw4& % !%$D*=uE  & 002 2KD%!d.C&C"".%E366;L;L)Mun5$)$6$6$8  8$)$7$7$@$@ $%  EKKMA$5 $AAL EKKM1L 2"5%s "DDrFF)rrrrrrrrrr staticmethodrtuplerrrrrrrrrKsA AA]u{{]] /F/v/,''2%) /'/'38_/'{{T! /'  /'b*T*t*`c*rrc`eZdZdZdZdej fdZdej fdZdZ dZ y ) EmbeddingAccessMixinz Base utilities to regroup getters and setters for embeddings. Introduces the `input_layer_embed` attribute, which indicates where the input embeddings come from and where they should be set. embed_tokensrct|dd}t||dx}|St|dr,t|j|rt|j|St|dr,t|j|rt|j|St|dr"|j}|||ur|j St d|jjd) z Returns the model's input embeddings. Returns: `nn.Module`: A torch module mapping vocabulary to hidden states. _input_embed_layerrN embeddingsr% base_modelu.`get_input_embeddings` not auto‑handled for "; please override in the subclass.) rrrr%rget_input_embeddingsNotImplementedError __class__r)rr default_embeddingrs rrz)EmbeddingAccessMixin.get_input_embeddingsst1>B")tT!: :  G$ $ 4 &74??D+I4??D1 1 4 !gdjj$&?4::t, , 4 &J%*D*@!6688!` (direct attribute) 2. `self.embeddings.<_input_embed_layer>` (nested embeddings for vision/audio models) 3. `self.model.<_input_embed_layer>` (encoder/decoder models) 4. delegate to the *base model* if one exists 5. otherwise raise `NotImplementedError` so subclasses still can (and should) override for exotic layouts. rrrr%ru.`set_input_embeddings` not auto‑handled for rN) rrrMrr%rset_input_embeddingsrrr)rrr s rrz)EmbeddingAccessMixin.set_input_embeddingsst1>B 4  D$ & T< (WT__d-K DOOT5 1 T7 # D(A DJJe , T< (T__D-H OO 0 0 7%@AXAX@YY{| rctt|dsy |j|jS#t$rYywxYw)Nlm_head)rrrrrs rget_output_embeddingsz*EmbeddingAccessMixin.get_output_embeddings.sBtY'   % % '||#  s + 77c,t|dr||_yy)ze Sets the model's output embedding, defaulting to setting new_embeddings to lm_head. rN)rr)rnew_embeddingss rset_output_embeddingsz*EmbeddingAccessMixin.set_output_embeddings9s 4 #)DL $rN) rrrrrrModulerrrrrrrrrs9( bii :"))< *rrc eZdZUdZdZeedzed<dZdZ e ed<dZ e ed<dZ ee dzed<d Ze ed <d Ze ee zed <dZee ee zdzed <dZe ee zdzed<dZee ee zdzed<dZee ee zdzed<dZee e fed<iZee e fed<dZee dzed<dZee dzed<dZee dzed<dZe ed<dZe ed<dZe ed<dZee e fed<dZ dZ!ee e"fdzed<dZ#e ed<dZ$e ed<dZ%e ed<dZ&edzed<e'e(jRjTdee e+ffd Z,e'dee e(jZffd!Z.fd"Z/d#effd$ Z0d%Z1e'dee e ffd&Z2e'dee e3e e fffd'Z4e2jjd(ee e fdzfd)Z2e4jjd(ee e3e e fffd*Z4dd+Z6d,Z7d-ee e zddfd.Z8e9d/Z:e'de;jxfd0Z=e9de fd1Z>dd2e de fd3Z?dd2e de fd4Z@dd2e de fd5ZAde fd6ZBdd2e de fd7ZC dd8e dzd2e de fd9ZDd:e dzde fd;ZEdde dzde fd?ZGe9de fd@ZHe9de fdAZId8e ezfdBZJd:e ezfdCZKdDZLdEZMddFe dzfdGZNddFe dzfdHZOdIZPdJZQe(jdKZSdLZTe(jeUjdMZWddNe defdOZXddQee dzdRe fdSZYdTZZ ddUe[dzdVe[dzdWe de;jfdXZ]ddYZ^ ddZe;jdUe[dzdVe[dzdWe de;jf d[Z_ dd\e;jdUe[dzd]e dWe de;jf d^Zad_Zb dd]e fd`ZcdaZddbZedce[fddZfde;je3e;jzfdeZgdfZhddgZidPejfdhe diekfdjZldkZme'de fdlZn ddme eojzdne doedzdpe dqe[e zdre dzdse e zdzdte due fdvZqeresjfdwZtddxZuere(jvjxjfdyZvere(jvjxjfdzZwfd{Zxfd|Zye9d}e(jd~e de fdZ{d}e(jdefdZ|dde}dzfdZ~e9dddddddddPd deede eojzdzd#ee zeojzdzde eojzdzde de de dse e zdzde de dzde defdZedddoedzdee dzdede3eeff dZedededefdZddZe9ddZdZe'dZe'dZe'dZe'dZejjdZddZe'de fdZejjde ddfdZdedzdekfdZe9dZdQee ddfdZdQee dedzdddedzddf dZd~e ddfdZdeddfdZdZde fdZ dde de dee3e e(jZffdZdde ffd ZdZxZS)rza Base class for all models. [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading, downloading and saving models as well as a few methods common to all models to: - resize the input embeddings Class attributes (overridden by derived classes): - **config_class** ([`PreTrainedConfig`]) -- A subclass of [`PreTrainedConfig`] to use as configuration class for this model architecture. - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP models, `pixel_values` for vision models and `input_values` for speech models). - **can_record_outputs** (dict): N config_classrdbase_model_prefixF _is_stateful model_tags input_idsmain_input_nametextinput_modalities_no_split_modules_skip_keys_device_placement_keep_in_fp32_modules_keep_in_fp32_modules_strictr_checkpoint_conversion_mapping_keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpected_keys_to_ignore_on_save_supports_sdpa_supports_flash_attn_supports_flex_attn_tp_plan_pp_plansupports_gradient_checkpointing_can_compile_fullgraph_supports_attention_backend_can_record_outputsrc"|jxsiS)a Maps output names (e.g., "attentions", "hidden_states") to either: - A module class (e.g., `LlamaDecoderLayer`), using default index conventions: * index=0 for "hidden_states" * index=1 for "attentions" - Or an `OutputRecorder(...)` with `target_class`, optional `index`, and `layer_name`. Examples: These two are equivalent: ```python _can_record_outputs = { "attentions": LlamaAttention, "hidden_states": LlamaDecoderLayer } _can_record_outputs = { "attentions": OutputRecorder(LlamaAttention, index=1), "hidden_states": OutputRecorder(LlamaDecoderLayer, index=0) } ``` This means you can record outputs from the same class, by specifying a layer name. Before collecting outputs, we check that they come from this layer. If you have cross attention that come from `LlamaAttention` and self attention that also come from `LlamaAttention` but from `self_attn` you can do this: ```python class LlamaModel(PreTrainedModel): _can_record_outputs = { "attentions": OutputRecorder(LlamaAttention, index=1, layer-name="self_attn"), "cross_attentions": OutputRecorder(LlamaAttention, index=1, layer_name="cross_attn") } ``` )rrs rcan_record_outputsz"PreTrainedModel.can_record_outputssR''-2-rc8dtjtiS)z^ `dict[str, torch.Tensor]`: Dummy inputs to do a forward pass in the network. r)rrrNrs r dummy_inputszPreTrainedModel.dummy_inputss U\\,788rc Rt|di||jjdijdd}|jjdd}t |jdd}|j }|||_y|||_y|||_y|||_yy)Nrrrr)super__init_subclass____dict__rrr)clsrmchild_annotationchild_attributefull_annotationfull_attributers rrz!PreTrainedModel.__init_subclass__s !+F+ <<++,=rBFFxQUV,,**>4@)-11(DA))  &.C   )/C   '-C   (.C )rrcVt|t|ts:t d|j j d|j j d||_|j|_|j|jjd|j_ |j|jj|j_|jrt!j"||_|j j }|t&vrYddj)t&d}t+j,||j j }t/|d kDr|d }nd}||_|j2t4t7|j <y) NzParameter config in `zt(config)` should be an instance of class `PreTrainedConfig`. To create a model from a pretrained model use `model = z(.from_pretrained(PRETRAINED_MODEL_NAME)`T is_init_check(|rr)r__init__rr TypeErrorrrr name_or_path%_check_and_adjust_attn_implementation_attn_implementation_attn_implementation_internal(_check_and_adjust_experts_implementation_experts_implementation _experts_implementation_internal can_generater(from_model_configgeneration_configrDror)findallr loss_typerrpr)rrrrmr loss_groupsrs rrzPreTrainedModel.__init__si &"23'(?(?'@A NN3344\^   "//594^4^ KK , ,D5_5  1 8<7d7d KK / /8  4    %5%G%G%OD "NN++ L (chh|45Q7K ;0G0GHI9~!%aL  "484L4LS01rc Ziiic|_|_|_|j|ur|jj $|jj j ni|_|jj$|jjj ni|_|jj$|jjj ni|_|jd|_ t|jxsg|_ t|jxsg|_ t|jxsg|_|jD]\}}t!|ddx}rP|jj#|j j%Dcic] \}}|d||c}}t!|ddx}rP|jj#|j j%Dcic] \}}|d||c}}t!|ddx}rP|jj#|j j%Dcic] \}}|d||c}}t!|ddx}rU|jj#|j j%Dcic]\}}|d||d|c}}t!|d dx}r|jj#|t!|d dx}r|jj#|t!|d dx} s|jj#|  |j'|j)ycc}}wcc}}wcc}}wcc}}w) a A method executed at the end of each Transformer model initialization, to execute code that needs the model's modules properly initialized (such as weight initialization). It is also used to obtain all correct static properties (parallelism plans, tied_weights_keys, _keep_in_fp32_modules, etc) correctly in the case of composite models (that is, the top level model should know about those properties from its children). NF all_submodels_ep_planrrrall_tied_weights_keysrrr)rr#rrrbase_model_pp_plancopybase_model_tp_planbase_model_ep_planget_expanded_tied_weights_keysr$r!rrrnamed_childrenrupdater. init_weights._backward_compatibility_gradient_checkpointing) rr rplanrv tied_keys keep_fp32keep_fp32_strictno_splits r post_initzPreTrainedModel.post_inits279"b3 t}dm ??d "EI[[EcEcEoDKK::??AuwDMEI[[EcEcEoDKK::??AuwDMEI[[EcEcEoDKK::??AuwDM%)%H%HW\%H%]"%()C)C)Ir%J",/0Q0Q0WUW,X)!$T%;%;%Ar!B!//1 8LD&vz488t8 $$499;CTCTCV%W41aasmQ&6%WXvz488t8 $$499;CTCTCV%W41aasmQ&6%WXvz488t8 $$499;CTCTCV%W41aasmQ&6%WX#F,CTJJyJ**11\e\j\j\l\r\r\t2uTXTUWXdV1QC=TF!A3-3O2uv#F,CTJJyJ**11)<#*63QSW#XXX11889IJ"6+>EExE&&--h7% 8*  ;;='&X%W%W3vs5N N 3N! N' ct|jdr,|jjjr |jS|j S)z: The full tp plan for the model's modules distributed_config)rrr6enable_expert_parallelr#rrs rtp_planzPreTrainedModel.tp_plan3s9 4;; 4 5$++:X:X:o:o== }}rc|jSrrrs rpp_planzPreTrainedModel.pp_plan<s }}rr.c |i|_yt|ts td|j D]<\}}|t vstd|d|dt t j|jDcgc]\}}| }}}|jD]S}|jdd}d}|D]} tj|| sd}n|r;tjd |d U||_ycc}}w) Nz&Can only set a dictionary as `tp_plan`z#Unsupported tensor parallel style 'z ' for layer 'z'. Supported styles are *z\d+FTzLayer pattern 'z' does not match any parameters in the model. This rule may not be applied during tensor parallelization, or may lead to dimension mismatches)rrrrr.r=rrrreplacer)matchwarningswarn) rr. layer_patternparallel_styler rmodel_param_names regex_patternpattern_matchedrHs rr8zPreTrainedModel.tp_plan@s1 <DM $%EF F.2ZZ\  )M>%88 9.9IWdVef,,01D1I1I1K,L+MO 261F1F1HIgdATII!YY[ M)11#v>M#O/  88M:6&*O # %m_5dd  !Js Dc||_yrr:)rr.s rr;zPreTrainedModel.pp_plands  rc\t|dd}| td|j||S)z Potentially dequantize the model in case it has been quantized by a quantization method that support dequantization. rNz?You need to first quantize your model in order to dequantize itr)rr dequantize)rrrs rrIzPreTrainedModel.dequantizehs: t^T:  ^_ _&&t5&99rc|jr?t|jddr'|jt |jdyyy)Ngradient_checkpointingF)rrrgradient_checkpointing_enabledelattrrs rr-z>PreTrainedModel._backward_compatibility_gradient_checkpointingts@  / /GDKKIach4i  . . 0 DKK!9 :5j /rtagsct|tr|g}|jg|_|D],}||jvs|jj|.y)a\ Add custom tags into the model that gets pushed to the Hugging Face Hub. Will not overwrite existing tags in the model. Args: tags (`Union[list[str], str]`): The desired tags to inject in the model Examples: ```python from transformers import AutoModel model = AutoModel.from_pretrained("google-bert/bert-base-cased") model.add_model_tags(["custom", "custom-bert"]) # Push the model to your namespace with the name "my-custom-bert". model.push_to_hub("my-custom-bert") ``` N)rrrr)rrNtags radd_model_tagszPreTrainedModel.add_model_tagszsS, dC 6D ?? " DO ,C$//)&&s+ ,rc |jd|j}|jddx}(tjd||jk7r|n|}t |t rt t|}d|vr|jd|_d|vr|jd|_ g}|%|jt||jtrbts\tsVtj!dddl}|j%|j&j)t+ t-gt/|5||fi|}ddd|S#1swYSxYw) z All context managers that the model should be initialized under go here. Args: dtype (`torch.dtype`, *optional*): Override the default `dtype` and load the model under this dtype. r torch_dtypeNz1`torch_dtype` is deprecated! Use `dtype` instead!attn_implementationexperts_implementation@Detected DeepSpeed ZeRO-3: activating zero.init() for this modelrconfig_dict_or_path)rrrsrrrrrrrrrrr+rrrt deepspeedrzeroInitr*rrS)rrrmrrS init_contextsrYr%s r _from_configzPreTrainedModel._from_configsC 7FLL1!::mT: :K G    S T"fll2E E eS !E5)E !F **0**5J*KF ' $v --3ZZ8P-QF *    !25#,,!G H % ' FX KKZ [   ).."5"5JZJ\"5"]_n_p!q r] + *)&)E *  * s E!!E+c0t||j|S)z@ `torch.nn.Module`: The main body of the model. )rrrs rrzPreTrainedModel.base_models tT33T::rc dt|jvry|jD]/}t|dsdt|vs|js/yt|dr"tj |j dy)a Returns whether this model can generate sequences with `.generate()` from the `GenerationMixin`. Under the hood, on classes where this function returns True, some generation-specific changes are triggered: for instance, the model instance will have a populated `generation_config` attribute. Returns: `bool`: Whether this model can generate sequences with `.generate()`. GenerationMixinTrrzprepare_inputs_for_generationu6 has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions. - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception). - If you are not the owner of the model architecture class, please contact the model code owner to update it.F)r __bases__rrrswarningr)rbases rrzPreTrainedModel.can_generates CMM 2 2MM D40 D 1d6G6G6I   37 8 NN<<. !   rrc<|jj}|jsFt|dds9t |j j d|jjdtsd}d}trtjdytr!tjd td d ytjj!d t#|d |t%j&tj(j%d }t*j$j,rg|t%j&dkrt#|d|d|t*j,j/st |dt#|d|t*j$j0r;|t%j&dkrt#|d|d|t#|d||tj3dnT|R|t*j4t*j6fvr0tj3d|j j d|d|st9|j;Dchc]}|j<c}}t?|dk(rq|dj@dk(r_t*j,j/rtj3dytCrtj3dyt dycc}w)aN Check the availability of Flash Attention 2 for a given model. Args: is_init_check (`bool`, *optional*): Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are fully instantiated. This is needed as we also check the devices of the weights, which are only available later after __init__. This allows to raise proper exceptions early before instantiating the full models if we know that the model does not support the requested attention. _supports_flash_attn_2Fz does not support Flash Attention 2.0 yet. Please request to add support where the model is hosted, on its model hub page: https://huggingface.co/k/discussions/new or in the Transformers GitHub repo: https://github.com/huggingface/transformers/issues/newzVFlashAttention2 has been toggled on, but it cannot be used due to the following error:zPlease refer to the documentation of https://huggingface.co/docs/transformers/perf_infer_gpu_one#flashattention-2 to install Flash Attention 2.z+Detect using FlashAttention2 on Ascend NPU.Tz*Detect using FlashAttention2 (via kernel `r|z `) on XPU. flash_attnz3 the package flash_attn seems to be not installed. z2.1.0zY you need flash_attn package version to be greater or equal than 2.1.0. Detected version z. z\ Flash Attention 2 is not available on CPU. Please make sure torch can access a CUDA device.z% Flash Attention 2 is not available. z2.0.4zY you need flash_attn package version to be greater or equal than 2.0.4. Detected version zuYou are attempting to use Flash Attention 2 without specifying a torch dtype. This might lead to unexpected behaviourzaFlash Attention 2 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in  is a;. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", dtype=torch.float16)`rrrzYou are attempting to use Flash Attention 2 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.zYou are attempting to use Flash Attention 2 with a model not initialized on MLU. Make sure to move the model to MLU after initializing it on CPU with `model.to('mlu')`.a+You are attempting to use Flash Attention 2 with a model not initialized on GPU and with no GPU available. This is not supported yet. Please make sure to have access to a GPU and either initialise the model on a GPU by passing a device_map or initialising the model on CPU and then moving it to GPU.)"rrrrrrr _name_or_pathr]rcrsrtrdFLASH_ATTN_KERNEL_FALLBACK importlibutil find_spec ImportErrorrparsemetadatarcudarhiprfloat16bfloat16rrrrr0rb)rrrprefaceinstall_messageflash_attention_versionr param_devicess r_flash_attn_2_can_dispatchz*PreTrainedModel._flash_attn_2_can_dispatchsT !!))WT;SUZ-[>>**+,WW[WbWbWpWpVqrnn  )*nGpO&' IJ%' @A[\oAp@qq{|~~'' 5=!WI-`ap`q"rss+2-- 8J8J8R8RS_8`*a'==%%.w1GG)&i(ABYAZZ\]l\mn#ZZ446(&i(DE*WI5Z[jZk*lmm]]&&.w1GG)&i(ABYAZZ\]l\mn*WI5Z[jZk*lmm =   H  50O#O   ((,(?(?'@UGLBB  DOO= 9.0, but found z with compute capability z.0.z$ Flash Attention 3 is not available.z\ Flash Attention 3 is not available on CPU. Please make sure torch can access a CUDA device.zuYou are attempting to use Flash Attention 3 without specifying a torch dtype. This might lead to unexpected behaviourzaFlash Attention 3 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in ria?. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `dtype` argument. Example: `model = AutoModel.from_pretrained("meta-llama/Llama-3.2-1B", attn_implementation="flash_attention_3", dtype=torch.float16)`alibiF use_alibizNModel is configured to use ALiBi, which is not supported by Flash Attention 3.attention_dropoutrzModel has attention_dropout=z., which is not supported by Flash Attention 3.rrzYou are attempting to use Flash Attention 3 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.a+You are attempting to use Flash Attention 3 with a model not initialized on GPU and with no GPU available. This is not supported yet. Please make sure to have access to a GPU and either initialise the model on a GPU by passing a device_map or initialising the model on CPU and then moving it to GPU.T)rrrrrrrjr^rlrmrnrorrrrget_device_capabilityrsrrtrurrrrrrrr0)rrrrvmajorrrrys r_flash_attn_3_can_dispatchz*PreTrainedModel._flash_attn_3_can_dispatchRs !!((>>**+,WW[WbWbWpWpVqrnn  )*nG~~''7?!WI-a"bcczz&&( ::;;=q19$")#dejeoeofFfFfHeIIbchbiilm& 1U&VWW i{| =   H  50O#O   ((,(?(?'@UGLFF  4;; /74;; UZ3[mn n 4;; 3 49V9VYZ9Z.t{{/L/L.MM{|   DOOtjdtjjjdy)aA Check the availability of SDPA for a given model. Args: is_init_check (`bool`, *optional*): Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are fully instantiated. This is needed as we also check the devices of the weights, which are only available later after __init__. This allows to raise proper exceptions early before instantiating the full models if we know that the model does not support the requested attention. a does not support an attention implementation through torch.nn.functional.scaled_dot_product_attention yet. Please request the support for this architecture: https://github.com/huggingface/transformers/issues/28005. If you believe this error is a bug, please open an issue in Transformers GitHub repository and load your model with the argument `attn_implementation="eager"` meanwhile. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="eager")`rz2.4.1zUsing the `SDPA` attention implementation on multi-gpu setup with ROCM may lead to performance issues due to the FA backend. Disabling it to use alternative backends.FT)rrrrrrrsrr device_countrprursrbackendsenable_flash_sdprrs r_sdpa_can_dispatchz"PreTrainedModel._sdpa_can_dispatchs"">>**+,OO  MM   ) '')A- e//07==3II   y  NN   0 0 7rc|js"t|jjdt s t dy)zI Check the availability of Grouped MM for a given model. z1 does not support setting experts implementation.zYPyTorch Grouped MM requirements in Transformers are not met. Please install torch>=2.9.0.T)_can_set_experts_implementationrrrr_rors r_grouped_mm_can_dispatchz(PreTrainedModel._grouped_mm_can_dispatchsI 335 7 788ijk k&(k  rc|js"t|jjdt s t dy)aK Check the availability of Flex Attention for a given model. Args: is_init_check (`bool`, *optional*): Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are fully instantiated. This is needed as we also check the devices of the weights, which are only available later after __init__. This allows to raise proper exceptions early before instantiating the full models if we know that the model does not support the requested attention. a does not support an attention implementation through torch's flex_attention. Please request the support for this architecture: https://github.com/huggingface/transformers/issues/34809. If you believe this error is a bug, please open an issue in Transformers GitHub repository and load your model with the argument `attn_implementation="eager"` meanwhile. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="eager")`z]PyTorch Flex Attention requirements in Transformers are not met. Please install torch>=2.5.0.T)rrrrrarors r_flex_attn_can_dispatchz'PreTrainedModel._flex_attn_can_dispatchsS''>>**+,tt ,-o  rrTc|}|duxr|jd}|duxr*|jddk(xs|jddk(}|rs|jrgts]t sSt rIt s?t|jd}tr|jddk(rd}|rd|}t|r7 |r t|n t||rtjd|d|S|j%||}t'| r t||S#t$r<}|r3|jdr|j!||j#|d}~wwxYw) a Check that the `attn_implementation` exists and is supported by the models, and try to get the kernel from hub if it matches hf kernels pattern. Args: attn_implementation (`str` or `None`): The attention implementation to check for existence/validity. is_init_check (`bool`, *optional*): Whether this check is performed early, i.e. at __init__ time, or later when the model and its weights are fully instantiated. This is needed as we also check the devices of the weights, which are only available later after __init__. This allows to raise proper exceptions early before instantiating the full models if we know that the model does not support the requested attention. Returns: `str`: The final attention implementation to use, including potential fallbacks from sdpa to eager, or from None to sdpa (to potentially eager). Nzpaged|r|r}Fz/You do not have `flash_attn` installed, using `z%` from the `kernels` library instead!2)"requested_attention_implementation) startswith removeprefixrr]r^r`rcrkrdr9rFrErsrrrrrzrget_correct_attn_implementationrg)rrTrapplicable_attn_implementationis_pagedrequested_original_flash_attnr~s rrz5PreTrainedModel._check_and_adjust_attn_implementations(*=&&d2_7J7U7UV^7_)<4(G)  , ,X 6:M M Q"//9=PP & *)).04M4O$&*,-GH[HhHhiqHr-s *%',?,L,LX,VZm,m16-39:X9Y1Z. 3 4 56TU/0NO1''IJhIij>>..-.2-Q-Q. . * ,Omn+,JK--' 0*33C8779 779 s 4D,, E157E,,E1rUc(|j|}|S)a> Check that the `experts_implementation` exists and is supported by the models. Args: experts_implementation (`str` or `None`): The experts implementation to check for existence/validity. Returns: `str`: The final experts implementation to use. )"get_correct_experts_implementation)rrU!applicable_experts_implementations rrz8PreTrainedModel._check_and_adjust_experts_implementation9s-1,S,STj,k)00rrequested_attentionc|dn|}|dgtjzvrTd|d}|js t|ddr|dz }|jr|dz }|j r|d z }t |d zd |vr|j||Sd |vr|j||Sd |vr|j||Sd|vr |j||S|S#t tf$r}|d|vr|d}Yd}~|Sd}~wwxYw)Nsdpaeager Specified `attn_implementation="zc"` is not supported. The only possible arguments are `attn_implementation="eager"`, `"paged|eager"`rfFz, `"attn_implementation=flash_attention_3"`, `"attn_implementation=flash_attention_2"`, `"attn_implementation=paged|flash_attention_2"`zB, `"attn_implementation=sdpa"`, `"attn_implementation=paged|sdpa"`z(, `"attn_implementation=flex_attention"`rr|r}flex_attention) ALL_ATTENTION_FUNCTIONS valid_keysrrrrrrzrrrro)rrrapplicable_attentionmessager~s rrz/PreTrainedModel.get_correct_attn_implementationFsi)<)DvJ] y3J3U3U3W'W W23G2HIAA  ((GD:RTY,Zee""__''EEWs]+ + "6 6  + +M :$#!$8 8  + +M :$#!5 5  ( ( 7$#+ + /'' 6 $###  , /&2vAT7TG'.$##  /s?CC8# C33C8requested_expertsc|dn|}|dvrd|d}t||dk(r |j|S|S#ttf$r}|dk(r|d}Yd}~|Sd}~wwxYw)N grouped_mm)rr batched_mmz#Specified `experts_implementation="z"` is not supported. The only possible arguments are `experts_implementation="eager"`, `"experts_implementation=grouped_mm"` and `"experts_implementation=batched_mm"`.r)rrro)rrapplicable_expertsrr~s rrz2PreTrainedModel.get_correct_experts_implementationhs->-F\L] %J J56H5IJEE W% %  - ---/ "!!!  , -$ 4G%,"!!  -s6A AActj|j}t|dsy|j}t |dd5}|j }dddtjdr d|vxrd |vSy #1swY*xYw) zDetect whether the class supports setting its attention implementation dynamically. It is an ugly check based on opening the file, but avoids maintaining yet another property flag. __file__Frutf-8encodingNzclass \w+Attention\(nn.Module\)eager_attention_forwardz&ALL_ATTENTION_FUNCTIONS.get_interface(T) sysmodulesrrropenreadr)r*r class_module class_filercodes r_can_set_attn_implementationz,PreTrainedModel._can_set_attn_implementation|s {{3>>2 |Z0!** *cG 4 668D  997 >,4i9aei9i i  s A??Bctj|j}t|dsy|j}t |dd5}|j }dddd|vS#1swYdvSxYw)zDetect whether the class supports setting its experts implementation dynamically. It is an ugly check based on opening the file, but avoids maintaining yet another property flag. rFrrrNz@use_experts_implementation)rrrrrrrrs rrz/PreTrainedModel._can_set_experts_implementationsq {{3>>2 |Z0!** *cG 4 668D -44 -44s A""A.c  t|ts|n%|jd|jj}||jjk7ra|j s-t j|jjdn$|j|d}||j_ |jD]E}||us t|ts|jj|jjk7sHt|jdr_|j s-t j|jjdn|}t|tre|jjD]L}t!|j||jus&|j||jj}n|j#|}||j_ d|j_H|jjD]}t!|j|x}t|ts|n|j||j}t|ds|||jk7rm|dgt&j)zvr/t+d |d |d t-t&j)||_ t jd |d |dt|ds|`y)a Set the requested `attn_implementation` for this model. Args: attn_implementation (`str` or `dict`): The attention implementation to set for this model. It can be either a `str`, in which case it will be dispatched to all submodels if relevant, or a `dict` where keys are the sub_configs name, in which case each submodel will dispatch the corresponding value. rdz does not support setting its attention implementation dynamically, because it does not follow the functional approach based on AttentionInterface (see https://huggingface.co/docs/transformers/en/attention_interface)Fr _attn_was_changedTNrrz"` is not supported for zd. The only possible arguments are "eager" (manual attention implementation)or one of the following: z8We set the attention implementation for the sub-config `z` to `z` without finding the associated sub-model. For this reason we could not check if the model supports it. You may encounter undefined behavior.)rrrrrrrsrcrrrrrrzrrrrrrrrr)rrTrequested_implementationr sub_implementation subconfig_key subconfigs rset_attn_implementationz'PreTrainedModel.set_attn_implementations148 $((T[[-M-MN ! $t{{'G'G G446~~../0\\ ,0+U+U,E,V,(=U 9" :I%y/:$$..$++2G2GG 0 02EF!==?NN$..7789``*B&!"5t<-1[[-D-D&M&t{{MBiFVFVV5H5L5L$193C3C3X3X6" 2!& &*3)R)RSe)f&EWI$$B6:   2E" :J"[[44 8M$T[[-@@ M&&94@-,00 @^@^_# +>?*i.L.LL)'=T=_=_=a1aa(>?Q>RRjkxjyz88<=T=_=_=a8b7ce ?QI;NNRS`Raaghzg{|@@y*=>%79 8rct|ts|n%|jd|jj}||jjk7r"|j |}||j_|jD]}||ust|ts|jj|jjk7sG|}t|tre|jjD]L}t|j||jus&|j||jj}n|j|}||j_y)a Set the requested `experts_implementation` for this model. Args: experts_implementation (`str` or `dict`): The experts implementation to set for this model. It can be either a `str`, in which case it will be dispatched to all submodels if relevant, or a `dict` where keys are the sub_configs name, in which case each submodel will dispatch the corresponding value. rdN) rrrrrrrrrzrrrr)rrUrr rrs rset_experts_implementationz*PreTrainedModel.set_experts_implementations>4d; #'++B 0S0ST ! $t{{'J'J J'+'T'TUm'n $;SDKK 8 WI%y/:$$..$++2G2GG&>"4d;)-)@)@" "4;; >)BRBRR1G1K1K -y/?/?/W/W2." "&/%Q%QRd%e"DV   A) Wrcd}g}t}d}|jD]}t|tr t |ds |j }| t |ds@t|}||vrP|j||j|j|d}||_ |r |d|_ |s-tj|jj dyy#t $rYwxYw) z Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping the model weights fixed. c&|jdyNT)requires_grad_)rinputoutputs rmake_inputs_require_gradszMPreTrainedModel.enable_input_require_grads..make_inputs_require_grads3s  ! !$ 'rFrNregister_forward_hookTra does not expose input embeddings. Gradients cannot flow back to the token embeddings when using adapters or gradient checkpointing. Override `get_input_embeddings` to fully support those features, or set `_input_embed_layer` to the attribute name that holds the embeddings.)r!rrrzrrrr/rrr_require_grads_hooks_require_grads_hookrsrrr)rrhooks seen_modulesfound_embeddingsrinput_embeddings embedding_ids renable_input_require_gradsz*PreTrainedModel.enable_input_require_grads-s  (u  lln $Fv7GFLb#>#@  'w7GI`/a./L|+   \ * LL)??@YZ [# % $(%*! ',QxD $   >>**+,ww  %'  sC** C65C6c~t|dd}|sy|D]}|jg|_t|dr|`yy)z4 Removes the `_require_grads_hook`. rNr)rremoverrr)rrhooks rdisable_input_require_gradsz+PreTrainedModel.disable_input_require_gradsYsQ4d;  D KKM %'! 4. /( 0rmodalityc:|dvrgd}n|dk(rgd}n|ddg}ntd||D]}t||st||cS|j|urCt|jdr-|jj | }||jk7r|S|S) ai Best-effort lookup of the *encoder* module. If provided with `modality` argument, it looks for a modality-specific encoder in multimodal models (e.g. "image_encoder") By default the function returns model's text encoder if any, and otherwise returns `self`. Possible `modality` values are "image", "video" and "audio". imagevideo vision_towervisual vision_modelvision_encoder image_toweraudio) audio_tower audio_encoderspeech_encoder text_encoderencoderHUnnrecognized modality, has to be "image", "video" or "audio" but found get_encoderr)rrrrr)rrpossible_module_namesr  base_encoders rrzPreTrainedModel.get_encoderhs ) )$o !  $V !  %3Y$? !ghpgqrs s) +DtT"tT** + ??$ &74??M+R??666ILt.## rc*|dvrgd}|dk(rddg}n|ddg}ntd ||D]}t||st|||y|j|urtj<|j@tj&|jByyd|j8j:vrt|dr|jDdk7rtF|jDn |jH}||j\}}tjJ|jL|tjJ|jN|yyy)ad Initialize the weights. This is quite general on purpose, in the spirit of what we usually do. For more complex initialization scheme, it should be overridden by the derived `PreTrainedModel` class. In case a model adds an explicit `nn.Parameter`, this method should also be overridden in order to initialize it correctly. initializer_rangeg{Gz?init_stdinitializer_factorweightNgmeanstd_is_hf_initializedF LayerNormRMSNormbias running_meanRotaryEmbeddingoriginal_inv_freqdefault)(rrrrrrget_text_configrrLinearConv1dConv2dConv3dConvTranspose1dConvTranspose2dinitnormal_rrzeros_r padding_idxMultiheadAttention_reset_parameters GroupNorm BatchNorm1d BatchNorm2d BatchNorm3drrones_r running_varnum_batches_tracked rope_typerGcompute_default_rope_parameterscopy_inv_freqr)rrrrope_fn buffer_valuers r _init_weightszPreTrainedModel._init_weightss 4;; 3 4++//74C T[[* -++&&C T[["6 7++00C$++5579LdSC fryy"))RYY 2K]K]_a_q_qr svx.: V]]#>{{& FKK('  - LLSc :!!-gfmmMach6i FMM&*<*<=>7j-  5 5 6  $ $ & v bnnbnnbnn] ^f..777F,,555vx.: 6==)vvt,8 FKK(v~t4@ F//0 6--. F667A &"2"2";"; ;Pc@d##y0$F$4$45;;  &fmm4OL! JJv 5 JJv// >Ae ;rct|ddryt|ddx}.t|ddr!t|jsd|_y|j |d|_y)zM Initialize the weights if they are not already initialized. rFNrT)rr named_buffersrr")rrrs r_initialize_weightsz#PreTrainedModel._initialize_weights si 6/ 7 vx6 6V C 4e<--/0(,F %  6"$(!rcttjjds"d}|tjj_|j |j y)a This is equivalent to calling `self.apply(self._initialize_weights)`, but correctly handles composite models. This function dynamically dispatches the correct `init_weights` function to the modules as we advance in the module graph along the recursion. It can handle an arbitrary number of sub-models. Without it, every composite model would have to recurse a second time on all sub-models explicitly in the outer-most `_init_weights`, which is extremely error prone and inefficient. smart_applyc|jD]?}t|tr|j|j/|j|A|||Sr)childrenrrzr'r%)rfnrs rr'z7PreTrainedModel.initialize_weights..smart_apply# sQ"mmo/F!&/:**6+E+EF**2. / 4 rN)rrrrr'r%)rr's rinitialize_weightsz"PreTrainedModel.initialize_weights sCuxx 6 +6EHHOO ' 112rr"c |ri}|jdD]k\}}t|ts|jd}|dk7r-|j Dcic]\}}|d||d|}}}|j |m|S|j }t|jdd} | siS|iStjdtfd|j|jzDr|jSi}|jdD chc]\}} | c} }|j!dD chc]\}} | c} }z} |j D]\d zd zt#t%fd | } t#t%fd | } t'| d kDr(t'| d kDrt'| t'| zd k7rt)d dd| d| t+| t-| D]%\}}||jvr ||||<!|||<'|Scc}}wcc} }wcc} }w)a Return the expanded tied weight keys (in case they contain modules or regex patterns) for only the current model, or recursively for all submodels if `all_submodels=True` (i.e. it will re-check the config values for all submodels). For almost all models, we only require to tie the embeddings, so the model has an internal property `_tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}`. In this case, the mapping is already "expanded", i.e. it already contains full parameters, and this function will simply return a copy of the property. For more complex patterns, e.g. for `DFineForObjectDetection`, we have the following attribute ``` _tied_weights_keys = { r"bbox_embed.(?![0])\d+": "bbox_embed.0", r"class_embed.(?![0])\d+": "class_embed.0", "model.decoder.class_embed": "class_embed", "model.decoder.bbox_embed": "bbox_embed", } ``` In this case, the function looks up all the model's parameters and buffers, and matches all the params, returning the following: ``` { 'bbox_embed.1.layers.0.bias': 'bbox_embed.0.layers.0.bias', 'bbox_embed.1.layers.0.weight': 'bbox_embed.0.layers.0.weight', 'bbox_embed.1.layers.1.bias': 'bbox_embed.0.layers.1.bias', 'bbox_embed.1.layers.1.weight': 'bbox_embed.0.layers.1.weight', 'bbox_embed.1.layers.2.bias': 'bbox_embed.0.layers.2.bias', 'bbox_embed.1.layers.2.weight': 'bbox_embed.0.layers.2.weight', 'bbox_embed.2.layers.0.bias': 'bbox_embed.0.layers.0.bias', 'bbox_embed.2.layers.0.weight': 'bbox_embed.0.layers.0.weight', ... 'class_embed.1.bias': 'class_embed.0.bias', 'class_embed.1.weight': 'class_embed.0.weight', 'class_embed.2.bias': 'class_embed.0.bias', 'class_embed.2.weight': 'class_embed.0.weight', ... 'model.decoder.class_embed.0.bias': 'class_embed.0.bias', 'model.decoder.class_embed.0.weight': 'class_embed.0.weight', 'model.decoder.class_embed.1.bias': 'class_embed.0.bias', 'model.decoder.class_embed.1.weight': 'class_embed.0.weight', ... 'model.decoder.bbox_embed.0.layers.0.bias': 'bbox_embed.0.layers.0.bias', 'model.decoder.bbox_embed.0.layers.0.weight': 'bbox_embed.0.layers.0.weight', 'model.decoder.bbox_embed.0.layers.1.bias': 'bbox_embed.0.layers.1.bias', 'model.decoder.bbox_embed.0.layers.1.weight': 'bbox_embed.0.layers.1.weight', ... } ``` i.e. all the parameters matching the regex and modules patterns in `_tied_weights_keys` F)remove_duplicater!rdrtie_word_embeddingsz ^[A-Za-z0-9_\.]+(weight)|(bias)$c3@K|]}j|ywr)r?)r+rcommon_case_regexs rr-zAPreTrainedModel.get_expanded_tied_weights_keys.. s_a &&q)_s^c0tj|Srr()x source_names rz@PreTrainedModel.get_expanded_tied_weights_keys.. BIIk14Mrc0tj|Srr()r3 target_names rr5z@PreTrainedModel.get_expanded_tied_weights_keys.. r6rrzAThere is an issue with your definition of `tie_weights_keys` for :z . We found z to tie into )rrrzr)r.r+rrrr)compileallrrr&rr$r2filterrrzipr )rr"expanded_tied_weightsprefixr submodel_tied_weightsrr/ tied_mappingr.rall_param_names source_params target_paramstarget_nsource_nr0r4r8s @@@rr)z.PreTrainedModel.get_expanded_tied_weights_keys2 sd $& !%)%7%7%7%O H! i9,5,T,Tch,T,i)|I^IdIdIf1AEAvhasOxq_<1-1*001FG H) (.. &dkk3H%P"I  !IJJ'JK _<3D3D3FI\I\I^3^_ _$$& &!#)-)>)>PU)>)VWA1W,,e,D[ !QA[  )5(:(:(< ? $K +K +K"6*M#_`M"6*M#_`M &*=)A-}%M(::a? WXcWddefqers -mM?L '*-}9M&N ?"(499;;6KH6U)(37?)(3 ?! ?6%$o12X[ sH8$ H> IT missing_keysrecompute_mappingc|s |j}n|jd}t|j}t |D]\}\}}|d}||v}||v} |r:| r8t j d|d|d|jj|R|s| r||}}nB|s@| s>||dzdD]\} } | |k(s | |v} | s| }nd}t j d |d|d |j|} d |vr'|jd d\}}|j|}n|}|}t||| |j|| |s|j|y) aM Tie the model weights. If `recompute_mapping=False` (default when called internally), it will rely on the `model.all_tied_weights_keys` attribute, containing the `{target: source}` mapping for the tied params. If `recompute_mapping=True`, it will re-check all internal submodels and their config to determine the params that need to be tied. This is the default when `model.tie_weights()` is called on its own, outside of `__init__`, and `from_pretrained`, in case the config values were changed somewhere. Note that during `from_pretrained`, tying is *symmetric*: if the mapping says "tie target -> source" but `source` is missing in the checkpoint while `target` exists, we *swap* source and target so we can still tie everything to the parameter that actually exists. Tr!NzDThe tied weights mapping and config for this model specifies to tie z to z, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warningrFzYThis checkpoint seem corrupted. The tied weights mapping for this model specifies to tie zk, but both are absent from the checkpoint, and we could not find another related tied weight for those keysr)r$r)rr. enumeratersrcrget_parameter_or_bufferrT get_submodulerM _adjust_biasdiscard)rrGrHr0itarget_param_namesource_param_nameremove_from_missingsource_is_theretarget_is_there target_backup source_backuptarget_backup_is_there source_param parent_namer rNs r tie_weightszPreTrainedModel.tie_weights s!22I;;$;OI*+ 9B99M; 8 5A5!#4'&*#"3<"G"3<"G#NN^_p^qqu,-.pp ..223DE(_;LN_'8%(8A!a%'8J4 }),==5B,5V2 64A 1 % /4+w016G5HI__ 778IJL''$5$<$= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities won't be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model. Nrr modifier_rank) _resize_token_embeddingsrrr+rYrZGatheredParametersrrrr vocab_sizerZ)rrdrerf model_embedsrrYrls rresize_token_embeddingsz'PreTrainedModel.resize_token_embeddings sH44^EWYfg  !&8&@ t^4V9J9JRV9V % ' 22<3F3FVZ2[ :)0066q9  : :&,,2215J4> ##%0$  : :s -C!!C*c|j}|j||||}t|dr|j}t |||j j }|j||j|t|dxr|jdu}|strP|sNddl } | jj|j d5|j jd}dddn|j jd}|j|j} t!| t"j$j&r|j| ||} n|j)| ||} t| dr| j}t | || j j } | j| |j+| |jS#1swYxYw)N_hf_hookrrrh)rf)r_get_resized_embeddingsrrprsrrJrrrr+rYrZrkrrrrrr_get_resized_lm_headr) rrdrerfold_embeddingsrrold_embeddings_requires_gradrrY old_lm_head new_lm_headold_lm_head_requires_grads rrjz(PreTrainedModel._resize_token_embeddings; s22455 N,>   >: .!**D ~t 4'5'<'<'J'J$%%&BC !!.1t^4V9J9JRV9V   ))+L ^^66~7L7L\`6aD%3%:%:%@%@%CNDD"0!6!6!= 7.5` (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities will not be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Embedding`: Pointer to the resized Embedding Module or the old Embedding Module if `new_num_tokens` is `None` Nz5Asking to pad the embedding matrix to a multiple of `z@`, which is not and integer. Please make sure to pass an integerrrzYou are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding dimension will be a.. This might induce some performance reduction as *Tensor Cores* will not be available. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tcrrhzOld embeddings are of type , which is not an instance of zj. You should either use a different resize function or make sure that `old_embeddings` are an instance of rrzThe new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`)rrrrrrsrtrrr+rYrZrkrrrrr0rrr"r(_init_added_embeddings_weights_with_meanrrar^r) rrsrdrerfrrYold_num_tokensold_embedding_dimradded_num_tokensnparamss rrqz'PreTrainedModel._get_resized_embeddingsb sV  )0#6 KL^K_``a%!/!6!6!3H3HX\2] Q4B4I4I4N4N4P1 1 Q Q1?0E0E0J0J0L -N- ^ +4N4P! !.",,7-d>.B-CCabdbnbnaopLL>$   !((// ''--   N *=   ~ . n ,   =  .> )+L ^^668M8M7N^b6cAA&HX =="NNDT  / % ' $++^-B-BCF2262K V4B4I4I4N4NrPQrSTu4U%%**2A2q51 V V1?0E0E0J0J2A2q50QN ! ! & &rr1u - & ' $++^-B-BCF2262K 6(6(=(=%0>0E0E0J0J0P0PQR0S-"--9~PQ?QUcUoUo>o15N. 6 *8)>)>)C)CN ! ! &,:,A,A,F,F,L,LQ,ON )))5>A;MQ_QkQk:k-1*u Q Q\$ V V 6s1<P?Q =Q"AQ&?Q  QQ#&Q0ru transposedc ||St|dxr|jdu}tr~|s|ddl}|jj |j d5|s|j jn'|j jj\}}dddnG|s|j jn'|j jj\}}|k(r ts|St|tjs:tdt|dtjdtjd|s|fn|f} |jdu} tj| | |j j|j j d } ||kDr|s|j#| n||kDr|rt$j'd ||z } trw|suddl}|j g} | r| |jgz } |jj | d5|j)|| ||| || r|j+|| | dddn+|j)|| ||| || r|j+|| | t-||}trq|soddl}|j |j| j | jg} |jj | d5|j/| |||| ddd| S|j/| |||| | S#1swYJxYw#1swYxYw#1swY| SxYw) a Build a resized Linear Module from a provided old Linear Module. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end Args: old_lm_head (`torch.nn.Linear`): Old lm head liner layer to be resized. new_num_tokens (`int`, *optional*): New number of tokens in the linear matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just returns a pointer to the input tokens `torch.nn.Linear` module of the model without doing anything. transposed (`bool`, *optional*, defaults to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is `lm_head_dim, vocab_size` else `vocab_size, lm_head_dim`. mean_resizing (`bool`): Whether to initialize the added embeddings from a multivariate normal distribution that has old embeddings' mean and covariance or to initialize them with a normal distribution that has a mean of zero and std equals `config.initializer_range`. Setting `mean_resizing` to `True` is useful when increasing the size of the embeddings of causal language models, where the generated tokens' probabilities will not be affected by the added embeddings because initializing the new embeddings with the old embeddings' mean will reduce the kl-divergence between the next token probability before and after adding the new embeddings. Refer to this article for more information: https://nlp.stanford.edu/~johnhew/vocab-expansion.html Return: `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if `new_num_tokens` is `None` Nrrrhz#Old language model head is of type ryzg. You should either use a different resize function or make sure that `old_lm_head` are an instance of r)rrraThe new lm_head weights will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`)rrr+rYrZrkrrrrrr rr0rrrr"rsr%_init_added_lm_head_weights_with_mean"_init_added_lm_head_bias_with_meanr!_copy_lm_head_original_to_resized)rrurdrrfrrYr{old_lm_head_dimnew_lm_head_shapehas_new_lm_head_biasrvr}rnum_tokens_to_copys rrrz$PreTrainedModel._get_resized_lm_head srH  ! t^4V9J9JRV9V % ' 22;3E3EUY2Z 5?K&&++-[EWEWEYEYE[E`E`Eb0   2< ""'')ASASAUAUAWA\A\A^ ,NO ^ +4N4P +ryy15d;6G5HHfgigpgpfqrII;a! FP_n=VdfuUv*//t; ii %%%,,$$**   N *=   { + n ,   =  .> )+L %,,-'{//00F^^66vT6Rl>>#[/>Sceo,?? [Zjk ll::o~O_ak(;;KVfg @ % ' !((+*:*:K)covariance_matrix) sample_shaperr)rrarrrrTrpositive_definitecheckr; distributionsmultivariate_normalMultivariateNormalsamplerr) rrsrr{r}old_embeddings_weightmean_embeddingsold_centered_embeddings covarianceepsilonis_covariance_psd distributions rrzz8PreTrainedModel._init_added_embeddings_weights_with_mean sc!/ 5 5 : : = =emm L**%:C"7/"I,..1HH>Y '99??*@TUYY[  ..BBUU7Z3GVLFREXEX.0FYFb&&,,-  ! ! & &r,<'<'>'A B  a(//0@!DGGH]H]HcHcd  ! ! & &r,<'<'>'A Brc|r^|jjj|j_|jjj|j_|j|||||r_|jjj|j_|jjj|j_yyr)rrarrz)rrurvrr{r}rs rrz5PreTrainedModel._init_added_lm_head_weights_with_mean s &1&8&8&=&=&?&?K   #&1&8&8&=&=&?&?K   # 55k;P^`pq &1&8&8&=&=&?&?K   #&1&8&8&=&=&?&?K   # rchtj|jjdtj}tj |jjdj tj}|jjd|zdj|d|zy)Nr)rrrrrr)rrrrarrrr)rrurvr} bias_meanbias_stds rrz2PreTrainedModel._init_added_lm_head_bias_with_mean sJJ{//441EMMR 99[--22;>>u}}Mb#3356>>ISWZbSb>crc`|s=|jjd|ddf|jjd|ddf<n<|jjddd|f|jjddd|f<|r1|jjd||jjd|yyr)rrar)rrvrurrrs rrz1PreTrainedModel._copy_lm_head_original_to_resized s>I>P>P>U>UViWiViklVl>mK   # #$7%7$7$: ;>I>P>P>U>UVWYlZlYlVl>mK   # #A':(:':$: ; 9D9I9I9N9NObPb9cK   ! !"5#5 6 rnew_num_position_embeddingsc |td|jd|jd|jjd)Nz4`resize_position_embeddings` is not implemented for B`. To implement it, you should overwrite this method in the class  in `modeling_.py`rrr)rrs rresize_position_embeddingsz*PreTrainedModel.resize_position_embeddings sH!B4>>BRS226..1APTP^P^PiPiOjjn p  rc |td|jd|jd|jjd)Nz1`get_position_embeddings` is not implemented for rrrrrs rget_position_embeddingsz'PreTrainedModel.get_position_embeddings sH!??OP226..1APTP^P^PiPiOjjn p  rcttjdk7r|j|j dy)z Initialize and tie the weights if needed. If using a custom `PreTrainedModel`, you need to implement any initialization logic in `_init_weights`. rF)rHN)rrrr+rZrs rr,zPreTrainedModel.init_weights s5 6 75<<;O O  # # % 51rc|js"t|jjd|ddi}t j t fi|}dtj|jjv}|s|jd|n;|jt |jdtjd |jd k(}|xs t|d d}|r|j!yy) a Activates gradient checkpointing for the current model. We pass the `__call__` method of the modules instead of `forward` because `__call__` attaches all the hooks of the module. https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2 Args: gradient_checkpointing_kwargs (dict, *optional*): Additional keyword arguments passed along to the `torch.utils.checkpoint.checkpoint` function. z) does not support gradient checkpointing.N use_reentrantFrT)enablegradient_checkpointing_funcrVYou are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it).Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model.r_hf_peft_config_loaded)rrrr functoolsr rinspect signature_set_gradient_checkpointingrapplyrsrcrrr)rgradient_checkpointing_kwargsr_is_using_old_formatneeds_embedding_gradsenable_input_gradss rrLz-PreTrainedModel.gradient_checkpointing_enable s33 7 788abc c ( 0->**+,]] -rcN|jr{dtj|jjv}|s|jdn;t j d|jt|jdt|ddr|jyy)zK Deactivates gradient checkpointing for the current model. rF)rrrrN) rrrrrrsrcrr rr)rrs rgradient_checkpointing_disablez.PreTrainedModel.gradient_checkpointing_disable s  / /$+g.?.?@`@`.a.l.l#l '000>L 74#C#C5QR 415 9  , , . :rcBtd|jDS)zT Whether gradient checkpointing is activated for this model or not. c3PK|]}t|dxr |j yw)rKN)rrK)r+ms rr-z.3 s'mYZ7167TAtA|||j4|r| s|j4jC||jEr|jFjC|| rtjId|jK|}|r9tjIdi}|jMD] \}}||d|<|}|jO}tQ|dkDr t d|d}|jR|}|jC|||jU}d}tW|dr~tQtY|jZj]dkDrOd |jZj]vsd!|jZj]vrd }t_j`d"tbr4tdjfjhjjD] \}}||}|jl|jlD] }||vs||= |j,to||jp|jr|j}tu||}| r| s tw||}| stx}t{||}nt|}|jd#d$jd%d&}t|||'} d}!| jr+d(|ji| j| jd)}!tj|D]}"tjj||"}#|jd#d*jd%d*}$|"jd#d*jd%d*}%tjd+}&|"j|$stjj|#s|"| jvs|s|&j|%tj|#tj| jjMd,-D]\}'}(tjj||'}"i})|(D]M}*|j!|*}+|r%|+jjd.k(r t||*}+|+j|)|*<Ot|)|"|/~)|!9tjj||},tjId0|,nt}-tjj|t{|-|}-t|-d1d235}.tj|!d4d 5d6z}/|.j|/dddtjId7|d8tQ| jd9|-d|r_t|j|:}0|0jtjj|d;|j|||<yy#1swYxYw)=a Save a model and its configuration file to a directory, so that it can be re-loaded using the [`~PreTrainedModel.from_pretrained`] class method. Arguments: save_directory (`str` or `os.PathLike`): Directory to which to save. Will be created if it doesn't exist. is_main_process (`bool`, *optional*, defaults to `True`): Whether the process calling this is the main process or not. Useful when in distributed training like TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on the main process to avoid race conditions. state_dict (nested dictionary of `torch.Tensor`): The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to only save parts of the model or if special precautions need to be taken when recovering the state dictionary of a model (like when using model parallelism). push_to_hub (`bool`, *optional*, defaults to `False`): Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). max_shard_size (`int` or `str`, *optional*, defaults to `"50GB"`): The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`). If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard which will be bigger than `max_shard_size`. variant (`str`, *optional*): If specified, weights are saved in the format model..safetensors. token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `hf auth login` (stored in `~/.huggingface`). save_peft_format (`bool`, *optional*, defaults to `True`): For backward compatibility with PEFT library, in case adapter weights are attached to the model, all keys of the state dict of adapters needs to be prepended with `base_model.model`. Advanced users can disable this behaviours by setting `save_peft_format` to `False`. save_original_format (`bool`, *optional*, defaults to `True`): For backward compatibility with the previous versions of `transformers` you can save the checkpoint with its reverse mapping. The reverse mapping needs to exists even if the model was loaded from a None legacy checkpoint. kwargs (`dict[str, Any]`, *optional*): Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. Nr`rFrzThe model is quantized with z and is not serializable - check out the warnings from the logger on the traceback to understand the reason why the quantized model is not serializable.z0.31.4z[Saving a model with tensor parallelism requires `huggingface_hub` version 0.31.4 or higher.zProvided path (z#) should be a directory, not a fileT)exist_okcommit_messagerepo_idr create_prrrformatrrFSDP)rzhDetected adapters on the model, saving the model in the PEFT format, only adapter weights will be saved.)rzTo match the expected format of the PEFT library, all keys of the state dict of adapters will be prepended with `base_model.model`.zbase_model.model.zMultiple active adapters detected, saving multiple active adapters is not supported yet. You can save adapters separately one by one by iteratively calling `model.set_adapter(adapter_name)` then `model.save_pretrained(...)`r hf_device_maprdiskz}Attempting to save a model with offloaded modules. Ensure that unallocated cpu memory exceeds the `shard_size` (50GB default)z.binz {suffix}.binrz{suffix}.safetensors)filename_patternrtotal_parameters)rq weight_maprdz(.*?)-\d{5}-of-\d{5}zWriting model shards)descr)rqzModel weights saved in wrrr)indent sort_keys z:The model is bigger than the maximum size per checkpoint (z) and is going to be split in z^ checkpoint shards. You can find where each parameters has been saved in the index located at )r`z README.md)rr`r)]rrrIis_serializablerquantization_config quant_method_tp_sizerkrorrUrprserrormakedirsrsplitseprr_get_files_timestampsget_state_dict_and_metadata unwrap_modelrrrrrr architectures _auto_classr&save_pretrainedrrrtget_adapter_state_dictr.active_adaptersr peft_configrrr!rrr@rAIS_SAGEMAKER_MP_POST_1_10smpstatemodule_managertranslate_functionsrr@r _device_meshrGr$rPrVrMr>rrurrqtensor_to_filenamelistdirror)r:rfilename_to_tensors fullmatchrretqdmrr0r3 contiguoussafe_save_filerOrjsondumpswriterirsave_upload_modified_files)1rrrrrrrRr`rrrmrrquantization_serializablerrrfiles_timestampsrq model_to_saverpeft_state_dictkeyractive_adaptercurrent_peft_config is_offloaded smp_to_hfr ignore_keyrQrstate_dict_splitindexrx full_filenameweights_no_suffixfilename_no_suffixreg shard_file tensor_namesshard_state_dict tensor_namerpath_to_weightssave_index_filercontent model_cards1 rrzPreTrainedModel.save_pretrained5 sv  #F7O!(/G!Ot^T:  $ qL+)N qS_SoSoSq "  #,BKd.|/O/O/\/\.]^uu  == $-PQY-Zm  77>>. ) LL?>*::]^ _  NT2 #ZZ(8$?NjjN,@,@,Mb,QRG ;6I!'CDCFCKKG#99.I   ##/#K#KD#Q J!%T* ##%(Z%5%5c%:1%= "/<.E.E.N.N.[.[\b.c-d *    ' t^DKK H )$$44^D  "//??O% ~+AAZAX #KK^')O&0&6&6&8K UEJ*;C5(ABK!0J!%!5!5!7~&*$u"0!2&*&6&6~&F##33NC  &113J D/ *C**113459$,,33554CUCUC\C\C^9^L MM:  % # 8 8 L L 3 1&z2  3  ' ' 3":: / +":. / == $3J tO`O`bfbobopJ9]S  (>1-LJ&,L' g>L4L'//GOOP^`vw= )9.   & &/1D1D1FdJZJcJcd.AAE  >2 )HGGLLBM!- 4 4VR @ H HY[ \ "*!1!1&"!=!E!EnVX!Y **45C##$56GGNN=1$4$H$HH#MM"45A -(# )()0  0 0 6 6 8?U)  ! $J ww||NJ?H! + D # 4  FMM$6$6&$@5m[QF170A0A0C - D +X I / !2 = ggll><HO KK1/1BC D5O ggll><Y`;abOosW= !**U1EL  ! KKL^L\] 0 D DEFG$$3#4A7  27DOOSXYJ OOBGGLLE F  ' ' -# (   ! !s +-_44_=c|j |jng}|jdg}t|tr|g}|D]}||vs|j ||r||d<t ||i|S)NrN)rrrrrrr)rrlrmrN tags_kwargsrPrs rrzPreTrainedModel.push_to_hubR s}"&//"=t2jj, k3 '&-K !C$ C  ! !F6Nw"D3F33rctd|jD}|r%td|jD}||z}|S)a Get the memory footprint of a model. This will return the memory footprint of the current model in bytes. Useful to benchmark the memory footprint of the current model and design some tests. Solution inspired from the PyTorch discussions: https://discuss.pytorch.org/t/gpu-memory-that-model-uses/56822/2 Arguments: return_buffers (`bool`, *optional*, defaults to `True`): Whether to return the size of the buffer tensors in the computation of the memory footprint. Buffers are tensors that do not require gradients and not registered as parameters. E.g. mean and std in batch norm layers. Please see: https://discuss.pytorch.org/t/what-pytorch-means-by-buffers/120266/2 c3^K|]%}|j|jz'ywrrrrs rr-z7PreTrainedModel.get_memory_footprint..n s%Ye%.."U%7%7%99Y+-c3^K|]%}|j|jz'ywrr)r+bufs rr-z7PreTrainedModel.get_memory_footprint..p s$Y33<<>C,<,<,>>Yr)sumrbuffers)rreturn_buffersmemmem_bufss rget_memory_footprintz$PreTrainedModel.get_memory_footprintb sBYtGXYY Y$,,.YYH.C rct|ddtjk(rpddlm}t ||i||jD]F}t||st|dkDr|d}n|jdd}|j |H|St|ddtjk(rt|ddr tdt ||i|S) Nquantization_methodr HQQLinearrrris_loaded_in_8bitFzCalling `cuda()` is not supported for `8-bit` quantized models. Please use the model as it is, since the model has already been set to the correct devices.) rrrHQQhqq.core.quantizer&rrrrrrrBITS_AND_BYTESr)rrlrmr&rrrs rrrzPreTrainedModel.cudat s 4. 59K9O9O O 3 GL$ )& ),,. (fi04y1}!%a!'Hf!=KK'  (K 4. 59K9Z9Z Zt0%8 sw|T,V,,rcd|v}|s%|D] }t|tjsd}nt|ddtj k(rsddlm}t |$|i||jD]I}t||sd|vr|d}n|d}d|vr|d}n|r}nd}|||_ |j|K|S|r)t|ddtjk(r tdt|ddtjk(r0|r tdt|d d rAt!d s6td t|ddtj"k(r |r td t |$|i|S)NrTr$rr%rzBCasting a Quark quantized model to a new `dtype` is not supported.zYou cannot cast a bitsandbytes model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired `dtype` by passing the correct `dtype` argument.r'Fz0.48zsYou need to install `pip install bitsandbytes>=0.48.0` if you want to move a 8-bit model across devices using to().zYou cannot cast a GPTQ model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired `dtype` by passing the correct `dtype` argument.)rrrrrrr(r)r&rrr compute_dtyperrQUARKrr*r[GPTQ) rrlrmdtype_present_in_argsargr&rrrrs rrzPreTrainedModel.to s!(6 1$ c5;;/,0)  4. 59K9O9O O 3 GJ ' ',,. (fi06)!'!1!%a&( &w. # $(/4,KK'# ($K WT3H$%OSeSkSk%kab b 4. 59K9Z9Z Z$ P t0%8AZ[aAb JT0$ 7;M;R;R R$ Hwz4*6**rcLt|ddr tdt| |S)NrFz`.half()` is not supported for quantized model. Please use the model as it is, since the model has already been casted to the correct `dtype`.)rrrhalfrrlrs rr2zPreTrainedModel.half s3 4 /I  7<& &rcLt|ddr tdt| |S)NrFz`.float()` is not supported for quantized model. Please use the model as it is, since the model has already been casted to the correct `dtype`.)rrrfloatr3s rr5zPreTrainedModel.float s3 4 /I  7=$' 'rrrrct||jtjg}t rddl}|si|sgt jd|jtj|jjttg|S|r.|jtjdt!g|S|j#tjd|S)NrrVrWr)rrrno_tie_weightsr+rYrsrtrno_init_weightsrZr[r*rrrrr)rrrrr\rYs rget_init_contextz PreTrainedModel.get_init_context s+5#,,?ATATAVW % '  (: ^_$$,,.!++@P@R+S') $$ell6&:=0.9.0. Please install the latest version with `pip install -U kernels`r)use_kernel_mappingr)$register_kernel_mapping_transformersN)inherit_mappingTF) r`rkernelsr?integrations.hub_kernelsr@rrTsanitize_kernel_mappingcreate_compatible_mappinguse_local_kernelkernel_mapping use_kernels)rrHr=r?r@rAs rset_use_kernelszPreTrainedModel.set_use_kernelss ') B 3 V 0 2(Z |-T55d;77= '4&D&D"D' (D(DVef,'+D$,,$( $D  ,,s B!!B*rb) rr\rr]r_r`rarrrrr\rr]r_rarrc . | jdd} | jdd}| jdd}| jdd}| jdd}| jdd}| jd d}| jd d}| jd d}| jd d}| jd d}| jdd}| jdd}| jdd}| jdd}| jdixsij}| jdd}| jdd}| jdd}| jdd} | jdd}!| jdd}"| jdd}#| jdd}$| jdd}%| jdd}&| jdd}'|"| d } d!D]}(| j|(d})|||n|}|d }tr|sd"}|||||||d#}*i|*d$|i}+| || td%|d k(r>t t j jd&d'rtjd(| |!t| |!|#|)\}}#}!|ts td*|i}t||+fi|\},}}t|}d+d,|d-}-|||-d.<t|tsU||n|}.|j j"|.fd"|||d/|*| \}}/d|/vr|/jd|/jd|}n$tj$|}| }/t'|d|}||+d$<d0| vr| jd0|_d1| vr| jd1|_t-|||| |-\}0}}|r@|0 td2|1t|t.rd3|j1vsd3|vr t3d4|&|%stj5d5d"}%t7|||| |+|-|j8dut'|d6d7\}1}2|0du}3|r=d8d9lm}4t?j@d:5||}5ddd|4|1d;d"5<d=} tC||1||2| | |0\}}||_"|jG||3tH}6tj$|}tK|65||g| i|/}7|0|0jM|7|||1|%>ddd7jO|}8tQ|7|'|0}9tRr|#tU|7| |"|#|!}7|tW|7|||0}tY|||2|||||8|0|#| |9| |*?}:|j[|7| |1|:\};}<|j]|7|:|;};|7j_|7ja|%|&|7jcr*te|7d@r|s|7jf||||fi|*d|$i| |5titk|j1d8kDrtm|7|0|||<||0|0|7_7|0jq|7|,|||dA<|7js|,||:|B};|r|7|;jufS|7S#1swYxYw#1swYxYw)Ca: Instantiate a pretrained pytorch model from a pre-trained model configuration. The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train the model, you should first set it back in training mode with `model.train()`. The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning task. The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those weights are discarded. Parameters: pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): Can be either: - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co. - A path to a *directory* containing model weights saved using [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`. - `None` if you are both providing the configuration and state dictionary (resp. with keyword arguments `config` and `state_dict`). model_args (sequence of positional arguments, *optional*): All remaining positional arguments will be passed to the underlying model's `__init__` method. config (`Union[PreTrainedConfig, str, os.PathLike]`, *optional*): Can be either: - an instance of a class derived from [`PreTrainedConfig`], - a string or path valid as input to [`~PreTrainedConfig.from_pretrained`]. Configuration for the model to use instead of an automatically loaded configuration. Configuration can be automatically loaded when: - The model is a model provided by the library (loaded with the *model id* string of a pretrained model). - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the save directory. - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a configuration JSON file named *config.json* is found in the directory. state_dict (`dict[str, torch.Tensor]`, *optional*): A state dictionary to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and [`~PreTrainedModel.from_pretrained`] is not a simpler option. cache_dir (`Union[str, os.PathLike]`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. ignore_mismatched_sizes (`bool`, *optional*, defaults to `False`): Whether or not to raise an error if some of the weights from the checkpoint do not have the same size as the weights of the model (if for instance, you are instantiating a model with 10 labels from a checkpoint with 3 labels). force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`): Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. local_files_only(`bool`, *optional*, defaults to `False`): Whether or not to only look at local files (i.e., do not try to download the model). token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `hf auth login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`. attn_implementation (`str`, *optional*): The attention implementation to use in the model (if relevant). Can be any of `"eager"` (manual implementation of the attention), `"sdpa"` (using [`F.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html)), `"flash_attention_2"` (using [Dao-AILab/flash-attention](https://github.com/Dao-AILab/flash-attention)), or `"flash_attention_3"` (using [Dao-AILab/flash-attention/hopper](https://github.com/Dao-AILab/flash-attention/tree/main/hopper)). By default, if available, SDPA will be used for torch>=2.1.1. The default is otherwise the manual `"eager"` implementation. Accept HF kernel references in the form: /[@][:] - and are any non-"/" and non-":" sequences. - "@" is optional (branch, tag, or commit-ish), e.g. "@main", "@v1.2.0", "@abc123". - ":" is optional and selects a function inside the kernel repo. - Both options can appear together and in this order only: @revision first, then :kernel_name. - We intentionally allow a leading "|" prefix (e.g., "flash|...") because the code strips it before loading; '|' is not excluded in the character classes here. Examples that match: "org/model" "org/model@main" "org/model:custom_kernel" "org/model@v1.2.3:custom_kernel" experts_implementation (`str`, *optional*): The experts implementation to use in the model (if relevant). Can be any of: - `"eager"` (sequential implementation of the experts matrix multiplications). - `"batched_mm"` (using [`torch.bmm`](https://pytorch.org/docs/stable/generated/torch.bmm.html)). - `"grouped_mm"` (using [`torch.nn.functional.grouped_mm`](https://docs.pytorch.org/docs/main/generated/torch.nn.functional.grouped_mm.html)). By default, if available, `grouped_mm` will be used for torch>=2.9.0. The default is otherwise the sequential `"eager"` implementation. > Parameters for big model inference dtype (`str` or `torch.dtype`, *optional*, defaults to `"auto"`): Override the default `torch_dtype` and load the model under a specific `dtype`. The different options are: 1. `torch.float16` or `torch.bfloat16` or `torch.float`: load in a specified `dtype`, ignoring the model's `config.dtype` if one exists. If not specified - the model will get loaded in `torch.float` (fp32). 2. `"auto"` - A `dtype` or `torch_dtype` entry in the `config.json` file of the model will be attempted to be used. If this entry isn't found then next check the `dtype` of the first weight in the checkpoint that's of a floating point type and use that as `dtype`. This will load the model using the `dtype` it was saved in at the end of the training. It can't be used as an indicator of how the model was trained. Since it could be trained in one of half precision dtypes, but saved in fp32. 3. A string that is a valid `torch.dtype`. E.g. "float32" loads the model in `torch.float32`, "float16" loads in `torch.float16` etc. For some models the `dtype` they were trained in is unknown - you may try to check the model's paper or reach out to the authors and ask them to add this information to the model's card and to insert the `dtype` or `torch_dtype` entry in `config.json` on the hub. device_map (`str` or `dict[str, Union[int, str, torch.device]]` or `int` or `torch.device`, *optional*): A map that specifies where each submodule should go. It doesn't need to be refined to each parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the same device. If we only pass the device (*e.g.*, `"cpu"`, `"cuda:1"`, `"mps"`, or a GPU ordinal rank like `1`) on which the model will be allocated, the device map will map the entire model to this device. Passing `device_map = 0` means put the whole model on GPU 0. To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For more information about each option see [designing a device map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). max_memory (`Dict`, *optional*): A dictionary device identifier to maximum memory if using `device_map`. Will default to the maximum memory available for each GPU and the available CPU RAM if unset. tp_plan (`Optional[Union[dict, str]]`, *optional*): A torch tensor parallel plan, see [here](https://pytorch.org/tutorials/intermediate/TP_tutorial.html). Use `tp_plan="auto"` to use the predefined plan based on the model. If it's a dict, then it should match between module names and desired layout. Note that if you use it, you should launch your script accordingly with `torchrun [args] script.py`. This will be much faster than using a `device_map`, but has limitations. tp_size (`str`, *optional*): A torch tensor parallel degree. If not provided would default to world size. device_mesh (`torch.distributed.DeviceMesh`, *optional*): A torch device mesh. If not provided would default to world size. Used only for tensor parallel for now. If provided, it has to contain dimension named `"tp"` in case it's > 1 dimensional, this dimension will be used for tensor parallelism offload_folder (`str` or `os.PathLike`, *optional*): If the `device_map` contains any value `"disk"`, the folder where we will offload weights. offload_buffers (`bool`, *optional*): Whether or not to offload the buffers with the model parameters. quantization_config (`Union[QuantizationConfigMixin,Dict]`, *optional*): A dictionary of configuration parameters or a QuantizationConfigMixin object for quantization (e.g bitsandbytes, gptq). subfolder (`str`, *optional*, defaults to `""`): In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. variant (`str`, *optional*): If specified load weights from `variant` filename, *e.g.* pytorch_model..bin. use_safetensors (`bool`, *optional*, defaults to `None`): Whether or not to use `safetensors` checkpoints. Defaults to `None`. If not specified and `safetensors` is not installed, it will be set to `False`. weights_only (`bool`, *optional*, defaults to `True`): Indicates whether unpickler should be restricted to loading only tensors, primitive types, dictionaries and any types added via torch.serialization.add_safe_globals(). When set to False, we can load wrapper tensor subclass weights. key_mapping (`dict[str, str], *optional*): A potential mapping of the weight names if using a model on the Hub which is compatible to a Transformers architecture, but was not converted accordingly. kwargs (remaining dictionary of keyword arguments, *optional*): Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., `output_attentions=True`). Behaves differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with `config`, `**kwargs` will be directly passed to the underlying model's `__init__` method (we assume all relevant updates to the configuration have already been done) - If a configuration is not provided, `kwargs` will be first passed to the configuration class initialization function ([`~PreTrainedConfig.from_pretrained`]). Each key of `kwargs` that corresponds to a configuration attribute will be used to override said attribute with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's `__init__` function. Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to use this method in a firewalled environment. Examples: ```python >>> from transformers import BertConfig, BertModel >>> # Download model and configuration from huggingface.co and cache. >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased") >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable). >>> model = BertModel.from_pretrained("./test/saved_model/") >>> # Update configuration during loading. >>> model = BertModel.from_pretrained("google-bert/bert-base-uncased", output_attentions=True) >>> assert model.config.output_attentions == True ``` rNr^output_loading_infoF_from_pipeline _from_autorrSr max_memoryoffload_folderrrrcrdrhrRadapter_kwargs adapter_namerrrWr8tp_sizer6rtrust_remote_coderHr= key_mappingr)mirror _fast_initlow_cpu_mem_usagefrom_tf from_flaxoffload_state_dictT)r\r]r^r_r`rarcrezq`state_dict` cannot be passed together with a model name or a `gguf_file`. Use one of the two loading strategies. WORLD_SIZErwaYou've set device_map=`auto` while triggering a distributed run with torchrun. This might lead to unexpected behavior. If your plan is to load the model on each device, you should set device_map={: PartialState().process_index} where PartialState comes from accelerate library)rRrrzIaccelerate is required when loading a GGUF file `pip install accelerate`.r%pytorch) file_typerfrom_auto_classusing_pipeline)return_unused_kwargsrWrMrLrTrUzYou cannot combine Quantization and loading a model from a GGUF file, try again by making sure you did not passed a `quantization_config` or that you did not load a quantized model from the Hub.rzxOne or more modules is configured to be mapped to disk. Disk offload is not supported for models loaded from GGUF files.zA kernel_config was provided but use_kernels is False; setting use_kernels=True automatically. To suppress this warning, explicitly set use_kernels to True.transformers_weights)rrRrWrrrXrYrZr)load_gguf_checkpointrr)return_tensors model_to_loadr)r%rrrrH)rrrrrrrrrrrrrradjust_generation_fnr`)rQ load_configrP);rr&rrrrrrrsrtrArZr:r0rrrfrom_pretraineddeepcopyrrrrJrrr7rrrmodeling_gguf_pytorch_utilsrbrrrrr9rrSpreprocess_modelr<r _torch_distributed_availabler?r-r_load_pretrained_model_finalize_model_loadingevalrIrrrerr!r/rpostprocess_model load_adapterto_dict)=rrrr\rr]r_r`rarr model_argsrmrr^rK from_pipeliner^rrSrrNrOrrrcrerRrPrQrrWr8rRr6rrSrHr=rTr rrdownload_kwargs_with_commit_adapter_model_pathrX config_path model_kwargsrrrrrb dummy_modelmodel_init_contextr%rweight_conversionsrf loading_infodisk_offload_indexs= rrgzPreTrainedModel.from_pretrained.s@ZZ d3 **Y-$jj)>F #3T:  **\59 7D)jj5 ZZ d3 ZZ d3 $4d; **%6>$jj)>EJJ{B/ jj6 **Y- **%5r:@bFFHzz.)< "JJ':DAJJ{D1 **Y-**Y-06 ;OQU0Vjj5 "JJ':DAjj6  ?D9 jj5  )goGp 'D 4&A '  "".EKE =E  %5# #, 0 " 'V&U-&U#  !'D'PT]TiD   C |S(I$J KKc   '"5/Lkj0 ,J W  )@)Bhi i  !NM` ) 'N N J:N .j9 #*Wfg  $+8J' (&"23$*$6&;GK]]6*F!L!&.+FK5@#M2 !F **0**5J*KF ' #v --3ZZ8P-QF *+; '\:, ( fj ' Y%J-&Jq.ARVfqrJ # #V-=z?VZVlVl{"!((0 1! )G cs*D"D'D'(%D'ct|ts |j}ddlmcm}t ||st|d||_y)aU Register this class with a given auto class. This should only be used for custom models as the ones in the library are already mapped with an auto class. Args: auto_class (`str` or `type`, *optional*, defaults to `"AutoModel"`): The auto class to register this new model with. rNz is not a valid auto class.) rrrtransformers.models.automodelsrrrr)r auto_class auto_modules rregister_for_auto_classz'PreTrainedModel.register_for_auto_classsC*c*#,,J66{J/ |+FGH H$rc t|ry||jjy|jj|ddddgfvrd}t|jdd}|jj-|jj|jjk(s^|jj -|jj |jjk(s|g||jjk(rN|d|jjd|jjd|jj d |d z }t j|yy) zv Shows a one-time warning if the input_ids appear to contain padding and no attention mask was given. NrrzWe strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked. sep_token_idz5 You may ignore this warning if your `pad_token_id` (z&) is identical to the `bos_token_id` (z), `eos_token_id` (z), or the `sep_token_id` (z ), and your input is not padded.)rmr pad_token_idr bos_token_id eos_token_idrsr)rrr warn_stringrs r%warn_if_padding_and_no_attention_maskz5PreTrainedModel.warn_if_padding_and_no_attention_masksJ i   &DKK,D,D,L  ;; # #yRG'< <F #4;;EL))5$++:R:RVZVaVaVnVn:nKK,,8T[[=U=UY]YdYdYqYq=q ,AYAY1YLT[[MeMeLfg..2kk.F.F-GGZ[_[f[f[s[sZtu..:^;[]    ,- =rc||jyt|jddy|jjyy)zJ Returns whether the model has a tensor parallelism plan. NTrF)rrrrr'rs rsupports_tp_planz PreTrainedModel.supports_tp_plans= == $ 4??J 5 A ;; ) ) 5rc|jS)z@ Returns the model's tensor parallelism degree. )rrs rrRzPreTrainedModel.tp_sizes }}rcN|jyt|jddyy)NTrF)rrrrs rsupports_pp_planz PreTrainedModel.supports_pp_plans( == $ 4??J 5 Arct|dr |jSt|dd}||tvrtj d|dd}t|S)N_loss_functionrz `loss_type=zZ` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. ForCausalLM)rrrrDrsr)rrs r loss_functionzPreTrainedModel.loss_functionse 4) *&& &D+t4    =   i[)=> &II&&rc||_yr)rrrs rrzPreTrainedModel.loss_functions #rcts tdddlm}m}m}|j s |jn| |jn|}||||jj|d|_ y)Nz]Kernels are not available. To use kernels, please install kernels using `pip install kernels`r)DeviceMode kernelize)r0)rmodeT) r`rrBrrrtraining INFERENCETRAININGrr0 _use_kernels)rrrrrs rrzPreTrainedModel.kernelizes_#%o  43%)]]t~~ [_$v4;;+;+;<4H rct|ddS)NrF)rrs rrHzPreTrainedModel.use_kernels&st^U33rrct|rt|ddry|r|jyt|ddrtj dd|_y)NrFzmDisabling kernels at runtime is a no-op as there is no 'unkernelize' routine; keeping current kernels active.)rrrrsrrrs rrHzPreTrainedModel.use_kernels*sP ;74?   NN t^U3##D!&D rcompile_configcpd|jjvr |jS|xs t}t |j ddxs t}t |drt |d||k7r:||_tj|jfi|j|_ |jS)aReturn a `torch.compile`'d version of `self.__call__`. This is useful to dynamically choose between non-compiled/compiled `forward` during inference, especially to switch between prefill (where we don't want to use compiled version to avoid recomputing the graph with new shapes) and iterative decoding (where we want the speed-ups of compiled version with static shapes).llama4rN_compiled_call_last_compile_config) r model_type__call__r'rrrrrr:rqr)rrdefault_configs rget_compiled_callz!PreTrainedModel.get_compiled_call9s t{{-- -== ':=? !7!79I4PcTaTc./t3^DV(6D %"'-- "ZAWAWAY"ZD """rc|jSr)r)rs ris_backend_compatiblez%PreTrainedModel.is_backend_compatibleKs...rctt}jjD]'\}}||j j |)|j Dcgc]:}t|dkDr*tfd|Dstfd|Ds|<}}|Dcic]}|ddD]}||d }}}jj|ycc}wcc}}w)a Adds keys to `self.all_tied_weights_keys` by checking if any group of params share the same data ptr. It helps us support remote code where the weight tying is done in old-T5 style, by manually assigning the same module to different param names. If we don't add them back in `self.all_tied_weights_keys`, they will be re-initialized and all params in tied group get random weights. rc3VK|] }|jjv"ywr)r$r)r+r rs rr-zGPreTrainedModel._adjust_tied_keys_with_tied_pointers..as%Td : : ? ? AATs&)c3&K|]}|v ywrr)r+r rGs rr-zGPreTrainedModel._adjust_tied_keys_with_tied_pointers..bs? ,?sNr) rrrr.rrrrr3r;r$r+) rrGparam_pointersrH param_valuer:tied_param_namesgrouptied_weights_keys_by_pointerss `` rrz4PreTrainedModel._adjust_tied_keys_with_tied_pointersOs%T*'+'8'>'>'@ F #J ;//1 2 9 9* E F(..0 5zA~TeTT???   /?) %*QVWXWYQZ) CMJa ) ) %)  ""))*GH ) s ,?C(2C-rrz/torch.distributed.device_mesh.DeviceMesh | Nonerc |du}tr|sytrts{|sy|jD])\}}t j |d}t |||+|jD])\}} t j | d}t |||+y||jjz D]h}|j|}t||d} t j || }|!t||||dd|j|\t |||j|jD]7\}} t||d} t j | | }t |||9y)aMove the missing keys (keys that are part of the model parameters, but were NOT found in the loaded state dicts) back from meta device to their device according to the `device_map` if any, else cpu. Takes care of sharding those missing parameters if `device_mesh` is provided, i.e. we are using TP. All non-persistent buffers are also moved back to the correct device (they are not part of the state_dict, but are not missing either). NrrT)valid_torch_deviceF)r+r,rrr empty_likerPr$r$rrKr2rBget_local_ranknamed_non_persistent_buffers) rrGrrrrrrrbuffer param_device buffer_devices rrz6PreTrainedModel._move_missing_keys_from_meta_to_devicelsz$4/ % '   %9%;L"335 = U((u=*4e< = $113 = V((>*4e< =   $"<"<"A"A"CC =C005E%j#$OL$$U<@E&+%T5+:T:T:VXc +4e< = <<> 9KC&z34PM$$VMBE &tS% 8 9rc ^tr|sddl}t|jdj Dchc]}t |ddr|c}}|j j|d5|jdddy|jycc}w#1swYyxYw)a Initialize the missing keys (keys that are part of the model parameters, but were NOT found in the loaded state dicts), according to `_initialize_weights`. Indeed, since the corresponding weights are missing from the state dict, they will not be replaced and need to be initialized correctly (i.e. weight initialization distribution). Params that are not missing have the `is_hf_initialized` flag. rNT) keep_varsrFrh) r+rYrrrrrZrkr+)rrrYr/not_initialized_parameterss rrz(PreTrainedModel._initialize_missing_keyss & ' *. OOdO;BBDtqGTUWkmrLst* &223M]^2_ *'') * *  # # % u * *sB B3B##B,chtd|jD}|rdgng}|jxsg}|jxsg|z}d\}}t |dkDr+t j djd|D}t |dkDr+t j djd|D}|1|jDchc]}|j||c}|_|2|jDchc]}|j||c}|_ yycc}wcc}w) zAdjust the `missing_keys` and `unexpected_keys` based on current model's exception rules, to avoid raising unneeded warnings/errors. This is performed in-place. c3DK|]\}}|jdyw)zrotary_emb.inv_freqN)r)r+rrs rr-zFPreTrainedModel._adjust_missing_and_unexpected_keys..s"pifVW6??3H#I"ps zrotary_emb\.inv_freqNNrrc3(K|] }d|d ywrrNrr+patterns rr-zFPreTrainedModel._adjust_missing_and_unexpected_keys..s6g7!G9A6gc3(K|] }d|d ywrrrs rr-zFPreTrainedModel._adjust_missing_and_unexpected_keys..s9mgQwiq/9mrN) r3r$rrrr)r:rorGr*r) rr{has_inv_freq_buffersadditional_unexpected_patternsmissing_patternsunexpected_patternsignore_missing_regexignore_unexpected_regexrs rrz3PreTrainedModel._adjust_missing_and_unexpected_keyss; #"p[_[m[m[o"ppFZ*A)B`b&??E2#FFL"Pnn8B55  1 $#%::chh6gVf6g.g#h " #a '&(jj9mYl9m1m&n # ++88)tCy4!7> :>3s8d3i/$6=@D #c(T#Y"6"=D*.S#X-57"DcN78<#T#Y%5<;?&S D(8?04T#Y-4!ND !&$& %% $Hd38n#H48Hd3(()D07-2#T1#(D().-'++  ^^""'.Dn)<$='.#'.R9d3 #4599 /0#M/#MJ/>bc3hc5c?23 ^^!DcNT1!!F ^^DeCHo!56 :; ,49s?,t,@))V;BII;; #T##J]]]~KKKZ<$ Td8FKO.#&:O.>BO. O.b 1sUYz 1^a 1 $3: $^b $or $D"C$J"SV"(T$ 5 5 5a83:a8F*Wt*WX*X )C$J@%S4Z%4.%"U]]_7?7?r)$U]]_$T$$&3'34p%Dp%Tp%dN8C4N8SWN8` M&*)-" 9d 9 $J9 9  9v%+T&*)-" \ \d \ $J \  \  \B&* " YYd      B@!@@,d d c  bll8K)K 2(.T:>gq$\d,/(n4nn!%"&!$*"#'!%%)[bkk)[[4K [  [ c [t[TzD [[#[z > % %& 4' 4$ 588??   -!-2 588??  :+:+x'(U[[Z^.U[[T %%,:M%%N ?C.2(-$!&#''+!a - .a'*R[['84'?a!3&4t; a $t+ a "& aaaTzD aaaa %aaFX0 X04KX0s)d*X0) X0  $& ' X0X0t'/'?P' ''R!.%%*#-J   ' '$$ !4T44 & &$ & &# 0D##$//IcItI:/93i/94K/9G /9 "D( /9  /9b&T&d&*@QVZ<,]c]2>B # #6: # %U\\)* + #$ !rrz model file)object object_class object_files recursivectri}|r||d<t|fi|St|drt|jS|S)a Recursively unwraps a model from potential containers (as used in distributed training). Args: model (`torch.nn.Module`): The model to unwrap. recursive (`bool`, *optional*, defaults to `False`): Whether to recursively extract all cases of `module.module` from `model` as well as unwrap child sublayers recursively, not just the top-level distributed containers. rr)rZrtrrr)r%rrms rrrsL  "+F; *5;F;; 5( # - -LrrcP|dk(rytj|jdvS)zCheck if the device is an accelerator. We need to function, as device_map can be "disk" as well, which is not a proper `torch.device`. rF)rr)rrr0rs ris_accelerator_devicer(s)||F#((??raccelerator_device_mapcTtd}|jj}tjj r*tjj r |jng}|jD]\}}||vr |j|}||j|||} n|j} |j| z} t|dkDr5t||ddu} | | rtjjndz} ||xx| z cc<|S)z This utility function calculates the total bytes count needed to load the model on each device. This is useful for caching_allocator_warmup as we want to know how much cache we need to pre-allocate. cy)Nrrrrrr5z&get_total_byte_count..:rNrT) is_weightr)rr$rrrrrrr.rKparam_element_sizerrrr>get_world_size) r%rrtotal_byte_countrr8rHrr dtype_sizeparam_byte_countis_part_of_plans rget_total_byte_countr2s#9-22779 % 1 1 > > @UEVEVEeEeEgennmoG4::<5 F ) ) --j9  #%88 ERJ++-J ;;=:5 w 1) as cudaMalloc is not a bottleneck at all anymore - Loading speed bottleneck is now almost only tensor copy (i.e. changing the dtype) and moving the tensors to the devices. However, we cannot really improve on those aspects obviously, as the data needs to be moved/copied in the end. N)rrxpugArrg333333ArF)rrrJ)r.rrrrr0rrcurrent_device mem_get_infomemory_reservedmemory_allocatedrrrrt)r%rrrrrr byte_countaccelerator_modulerfree_device_memorytotal_device_memory unused_memoryrs rrrTs|*:M9R9R9T(5vXmntXuu||F## "+E3I<X/446#g ;;/ )!( !< $*LL$>uEHZHkHklqHrrMM)M9'-7 m+m;!1$'99!"J"/!2J Z)<})LMJ KKJ!O,EMM&`e fG#gs EEc FeZdZdZeeeeeeee dZ de de de ffd Z xZS)AttentionInterfacea_ Dict-like object keeping track of allowed attention functions. You can easily add a new attention function with a call to `register()`. If a model needs to locally overwrite an existing attention function, say `sdpa`, it needs to declare a new instance of this class inside the `modeling_.py`, and declare it on that instance. )r}r|rrzpaged|flash_attention_3zpaged|flash_attention_2z paged|sdpaz paged|eagerrTrrc|tjdn|dk7r||vrtd|dt|||S)zcReturn the requested `attn_implementation`. Also strictly check its validity, and raise if invalid.a You tried to access the `AttentionInterface` with a `config._attn_implementation` set to `None`. This is expected if you use an Attention Module as a standalone Module. If this is not the case, something went wrong with the dispatch of `config._attn_implementation`rrzP` is not a valid attention implementation registered in the `AttentionInterface`)rsrKeyErrorrr)rrTrrs r get_interfacez AttentionInterface.get_interfaces_  &   K  !G +0C40O'((xy w{.88r)rrrrr6r8r;r7r<r5_global_mappingrrr-r r s@rr*r*sH540&#:#:24 O 9 9x 9H 9 9rr*rcdeZdZdZedej fdZedej fdZy)PreTrainedAudioTokenizerBasea Class that additionally defines the behavior of any `audio_tokenizer` to be added. Characteristic for any of them: 1. Encode raw audio into discrete audio codebooks (with x channels) 2. Decode from discrete audio codebooks back to raw audio It is possible that they can decode in different ways given a different representation but they are forced to support 2. nonetheless, e.g. see `DAC`. input_valuescy)z Encode raw audio retrieved from a respective `FeatureExtractor` into discrete audio codebooks (with x channels) Nr)rr1rlrms rencodez#PreTrainedAudioTokenizerBase.encoderr audio_codescy)z6Decode from discrete audio codebooks back to raw audioNr)rr4rlrms rdecodez#PreTrainedAudioTokenizerBase.decoderrN) rrrrrrrr3r6rrrr0r0sH 5<<  E%,,EErr0r)rTrr(r r&rimportlib.metadatarlrrrr)rr@abcrrcollections.abcrr contextlibr dataclassesrr enumr r r itertoolsr threadingrtypingrrrzipfilerrhuggingface_hubrrr packagingr safetensorsrsafetensors.torchrrrrtorch.distributionsrtorch.utils.checkpointrrdrrconfiguration_utilsrconversion_mappingr core_model_loadingr!r"r#r$rr%dynamic_module_utilsr& generationr'r( integrationsr)r*r+r,integrations.accelerater-r.r/r0r1r2r3integrations.deepspeedr4integrations.eager_pagedr5integrations.flash_attentionr6integrations.flash_pagedr7integrations.flex_attentionr8rCr9integrations.peftr:integrations.sdpa_attentionr;integrations.sdpa_pagedr<integrations.tensor_parallelr=r>r?r@rArBrCloss.loss_utilsrDmodeling_flash_attention_utilsrErFmodeling_rope_utilsrG pytorch_utilsrH quantizersrIquantizers.autorJquantizers.quantizers_utilsrKsafetensors_conversionrLutilsrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdre utils.genericrfrg utils.hubrhrirjutils.import_utilsrkrlrmutils.loading_reportrnroutils.output_capturingrprqutils.quantization_configrraccelerate.hooksrsaccelerate.utilsrtrrk!smdistributed.modelparallel.torch modelparallelrsmdistributed.modelparallelru SMP_VERSIONrpr get_loggerrrsrrupperrvrxryrrrkrrrrrrrrrruint8int8int16uint16rtruint32uint32rfloat64int64uint64 float8_e4m3fn float8_e5m2rrrrrrrrrr r!rrr$rGrPrVrrrrrrzrrrrrrrr*rrr0rrrrys8  #.%($44 \\!9+-$1< +47iiFCA=?/2?A*j4,#-=36J\\ KH93< %00==?33F - k :mgmmF>S S %   H %zz~~nc288: JJNN#6<BBD%&CK\] 9=  $---4##0U[[0C$J00.  1 JJ ++ ** ;; << == NN ;; << == == ;; <<""  (hlk2;;&k69ELL6Hk`dk #u|| kDU\\c"))S ,DSN,S%,,=N8O,TYZ^_bcf_gZhjnorjsZsTt,>%T#c(^%c5<<>O9P%UZ[_`cdg`h[iknorks[sUt%&MS%,,&'M0AM #u|| M`(&7(S(RWR^R^(sS4Z326-1F.#&#4t#;F. 4ZF.TzF.D[ F. t F.  F.%($JF.$d*F. 49t TD[ ()F.`(,N  t #d *N3i$&N NTk N t N  N$N U[[ ()Nbt [[|U*U*pD7!bii!57GYiD7!Nn((C(CD&&2*9*E*E*M*M*T*T[|+U+O'  dryy2@#)ell":@t@^b 48HSVZHZD@gO@g$@g^ilp^p@gF 9) 9H/A.B+BE?Er