謜iudZddlZddlZddlZddlmZddlmZmZm Z m Z ddl Z ddl mZmZddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZdd l m!Z!erdd l"m#Z#ejHe%Z&e d Z'e d d Z(GddeZ)Gdd eZ*ee*jVe*_+e*jVj8e*jVjjYddde*jV_yy)zH Feature extraction saving/loading class for common feature extractors. N)UserDict) TYPE_CHECKINGAnyTypeVarUnion) create_repois_offline_mode)custom_object_save) FEATURE_EXTRACTOR_NAMEPROCESSOR_NAMEPushToHubMixin TensorType_is_tensor_or_array_like copy_funcis_numpy_arrayis_torch_availableis_torch_deviceis_torch_dtypeloggingrequires_backendssafe_load_json_file) cached_file)SequenceFeatureExtractorrSpecificFeatureExtractorTypeFeatureExtractionMixin)boundc eZdZdZ ddeeefdzddezezdeee ezdzffd Z dedefd Z defd Z d Z d Zddeezdzfd Z ddeezdzdeee ezdzfdZddZxZS) BatchFeaturea Holds the output of the [`~SequenceFeatureExtractor.pad`] and feature extractor specific `__call__` methods. This class is derived from a python dictionary and can be used as a dictionary. Args: data (`dict`, *optional*): Dictionary of lists/arrays/tensors returned by the __call__/pad methods ('input_values', 'attention_mask', etc.). tensor_type (`Union[None, str, TensorType]`, *optional*): You can give a tensor_type here to convert the lists of integers in PyTorch/Numpy Tensors at initialization. skip_tensor_conversion (`list[str]` or `set[str]`, *optional*): List or set of keys that should NOT be converted to tensors, even when `tensor_type` is specified. Ndata tensor_typeskip_tensor_conversioncVt||||_|j|y)N)r!)super__init__r"convert_to_tensors)selfr r!r" __class__s a/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/feature_extraction_utils.pyr%zBatchFeature.__init__Ks+ &<# K8itemreturncVt|tr|j|Std)z If the key is a string, returns the value of the dict associated to `key` ('input_values', 'attention_mask', etc.). zRIndexing with integers is not available when using Python based feature extractors) isinstancestrr KeyErrorr'r+s r) __getitem__zBatchFeature.__getitem__Us( dC 99T? "op pr*cH |j|S#t$rtwxYwN)r r0AttributeErrorr1s r) __getattr__zBatchFeature.__getattr___s* !99T? " !  !s!cd|jiSNr r r's r) __getstate__zBatchFeature.__getstate__es ""r*c"d|vr |d|_yyr8r9)r'states r) __setstate__zBatchFeature.__setstate__hs U?f DI r*c|yt|ts t|}|tjk(r.ts t dddlfdj }|fSdfd t}|fS)NNNzMUnable to convert output to PyTorch tensors format, PyTorch is not installed.rcj|r|St|ttfr3t |dkDr%j|drj |St|ttfrt |dkDrt|dt jrt j|}n_t|dttfrFt |ddkDr5t|ddt jrt j|}t|t jrj|Sj|S)Nr) is_tensorr.listtuplelenstacknpndarrayarray from_numpytensor)valuetorchs r) as_tensorz5BatchFeature._get_is_as_tensor_fns..as_tensorys??5) LedE]3E Q5??[`ab[cKd ;;u--edE]3E Q!%(BJJ7 ""58dE];aMA-&uQx{BJJ? "eRZZ0 ++E22 <<..r*ct|ttfrt|dtttjfr`|Dcgc] }t |}}t t |dkDr1|/|Dcgc]}tj|c}t}tj||Scc}wcc}w)Nrr )dtype) r.rCrDrGrHrEsetasarrayobject)rLrPval value_lensrNs r)rNz5BatchFeature._get_is_as_tensor_fns..as_tensorsedE]3 58dTY[][e[eMf8g6;!TTabefgbhaij_`   sA,, C 5ACC c t|dgddl jdjddetdkDrWd}t |rnFt |t st|st |tr|ntdt |d fd |jDcic]\}}||c}}|_ |Scc}}w) a Send all values to device by calling `v.to(*args, **kwargs)` (PyTorch only). This should support casting in different `dtypes` and sending the `BatchFeature` to a different `device`. Args: args (`Tuple`): Will be passed to the `to(...)` function of the tensors. kwargs (`Dict`, *optional*): Will be passed to the `to(...)` function of the tensors. To enable asynchronous data transfer, set the `non_blocking` flag in `kwargs` (defaults to `False`). Returns: [`BatchFeature`]: The same instance after modification. rMrNdevice non_blockingFz*Attempting to cast a BatchFeature to type z. This is not supported.c0t|jr#j|r|jiSt|jr|jSt|tt frt |fd|DS|S)N)rcrdc3.K|] }|ywr4).0r+maybe_tos r) z4BatchFeature.to..maybe_to..s<$x~.maybe_tos!U\\*u/F/Fq/IqttT,V,,Au||,1Ctt6 tEEAe}-tAwc$i!mq'Cc"C%)=CQTAU!#McRUhZWo!pqq  15 =1Q ^=  >s?C)NNNr4r@)r,r)r_ __module__ __qualname____doc__dictr/rrrCrQr%r2r6r;r>rXr&rm __classcell__)r(s@r)rr:s$'+/3>B 938nt#9CZ*,9!%S CH 4t ; 9qqq!! #&0$z1AD1H0$h04>B7:%,7!%S CH 4t ;7r0r*rceZdZdZdZdZe ddeede e jzde e jzdzde de d e e zdzd e d efd Z dd e e jzde fdZede e jzd eee efee efffdZedee efd ededee effffdZd ee effdZede e jzd dfdZd e fdZde e jzfdZdZeddZy)rz This is a feature extraction mixin used to provide saving/loading functionality for sequential and audio feature extractors. Nc |jdd|jD]\}} t|||y#t$r%}tj d|d|d||d}~wwxYw)z'Set elements of `kwargs` as attributes.processor_classNz Can't set z with value z for )popr[setattrr5loggererror)r'rpr`rLerrs r)r%zFeatureExtractionMixin.__init__ss  $d+ ,,. JC c5) "  z#l5'tfMN  s : A( A##A(clspretrained_model_name_or_path cache_dirforce_downloadlocal_files_onlytokenrevisionr,c ||d<||d<||d<||d<|||d<|j|fi|\}}|j|fi|S)a Instantiate a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a feature extractor, *e.g.* a derived class of [`SequenceFeatureExtractor`]. Args: pretrained_model_name_or_path (`str` or `os.PathLike`): This can be either: - a string, the *model id* of a pretrained feature_extractor hosted inside a model repo on huggingface.co. - a path to a *directory* containing a feature extractor file saved using the [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] method, e.g., `./my_model_directory/`. - a path or url to a saved feature extractor JSON *file*, e.g., `./my_model_directory/preprocessor_config.json`. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model feature extractor should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force to (re-)download the feature extractor files and override the cached versions if they exist. proxies (`dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. token (`str` or `bool`, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use the token generated when running `hf auth login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. To test a pull request you made on the Hub, you can pass `revision="refs/pr/"`. return_unused_kwargs (`bool`, *optional*, defaults to `False`): If `False`, then this function returns just the final feature extractor object. If `True`, then this functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored. kwargs (`dict[str, Any]`, *optional*): The values in kwargs of any keys which are feature extractor attributes will be used to override the loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is controlled by the `return_unused_kwargs` keyword parameter. Returns: A feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`]. Examples: ```python # We can't instantiate directly the base class *FeatureExtractionMixin* nor *SequenceFeatureExtractor* so let's show the examples on a # derived class: *Wav2Vec2FeatureExtractor* feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained( "facebook/wav2vec2-base-960h" ) # Download feature_extraction_config from huggingface.co and cache. feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained( "./test/saved_model/" ) # E.g. feature_extractor (or model) was saved using *save_pretrained('./test/saved_model/')* feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("./test/saved_model/preprocessor_config.json") feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained( "facebook/wav2vec2-base-960h", return_attention_mask=False, foo=False ) assert feature_extractor.return_attention_mask is False feature_extractor, unused_kwargs = Wav2Vec2FeatureExtractor.from_pretrained( "facebook/wav2vec2-base-960h", return_attention_mask=False, foo=False, return_unused_kwargs=True ) assert feature_extractor.return_attention_mask is False assert unused_kwargs == {"foo": False} ```rrrrr)get_feature_extractor_dict from_dict) rrrrrrrrpfeature_extractor_dicts r)from_pretrainedz&FeatureExtractionMixin.from_pretrainedstj({#1 %5!"%z  #F7O)G)G)GHe)pio)p&s}}3>v>>r*save_directory push_to_hubc tjj|rtd|dtj|d|rw|j dd}|j d|j tjjd}t|fd di|j}|j|}|jt||| tjj|t}|j|t j#d ||r%|j%||j'd  |gS)az Save a feature_extractor object to the directory `save_directory`, so that it can be re-loaded using the [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] class method. Args: save_directory (`str` or `os.PathLike`): Directory where the feature extractor JSON file will be saved (will be created if it does not exist). push_to_hub (`bool`, *optional*, defaults to `False`): Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the repository you want to push to with `repo_id` (will default to the name of `save_directory` in your namespace). kwargs (`dict[str, Any]`, *optional*): Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. zProvided path (z#) should be a directory, not a fileT)exist_okcommit_messageNrepo_idr)configzFeature extractor saved in r)rr)ospathisfileAssertionErrormakedirsr}splitseprr_get_files_timestamps _auto_classr joinr to_json_filerinfo_upload_modified_filesrq)r'rrrprrfiles_timestampsoutput_feature_extractor_files r)save_pretrainedz&FeatureExtractionMixin.save_pretraineds/ 77>>. ) ?>2BBe!fg g NT2 #ZZ(8$?NjjN,@,@,Mb,QRG!'CDCFCKKG#99.I     ' t^D A)+ ^E[(\% 78 12O1PQR   ' ' -jj) ( ...r*c |jdd}|jdd}|jdd}|jdd}|jdd}|jdd}|jd d} |jd d} |jd d} d | d } | | | d<tr|stjdd}t |}t j j|} t j j|r$t j j|t}t j j|r|}d}d} n7t} t|t|||||| | |d }t|||||||| | |d }d}|4t|}d|vsd|vr!|j!d|j!d}| | t|}|td|d|dd| rtjd|||fStjdd|||fS#t$rt$rtd|d|dtdwxYw)a6 From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used for instantiating a feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`] using `from_dict`. Parameters: pretrained_model_name_or_path (`str` or `os.PathLike`): The identifier of the pre-trained checkpoint from which we want the dictionary of parameters. Returns: `tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the feature extractor object. rNrFproxies subfolderrrr_from_pipeline _from_autofeature extractor) file_typefrom_auto_classusing_pipelinez+Offline mode: forcing local_files_only=TrueT) filenamerrrrr user_agentrr%_raise_exceptions_for_missing_entriesz"Can't load feature extractor for 'z'. If you were trying to load it from 'https://huggingface.co/models', make sure you don't have a local directory with the same name. Otherwise, make sure 'z2' is the correct path to a directory containing a z filefeature_extractoraudio_processorzloading configuration file z from cache at )r}r rrr/rrisdirrr rrr OSErrorr\rrq)rrrprrrrrrr from_pipelinerris_localfeature_extractor_fileresolved_feature_extractor_fileresolved_processor_filerprocessor_dicts r)rz1FeatureExtractionMixin.get_feature_extractor_dictsJJ{D1 $4e<**Y-JJ{D1  7D)!::&8%@::j$/ #3T:  **\59#6?[  $+8J' (  %5 KKE F# (+,I(J%77==!>? 77==6 7%'WW\\2OQg%h " 77>>7 8.K +&* #H%; "' *51+'#1#%5)%':? +'3>13'#1#%5)%':? 3/:"& " .01HIN"n48I^8[)7););>+B+B'( F "}% v x  r* json_filect|d5}|j}dddtj}|di|S#1swY&xYw)a Instantiates a feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`] from the path to a JSON file of parameters. Args: json_file (`str` or `os.PathLike`): Path to the JSON file containing the parameters. Returns: A feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`]: The feature_extractor object instantiated from that JSON file. utf-8encodingNrg)openreadjsonloads)rrreadertextrs r)from_json_filez%FeatureExtractionMixin.from_json_fileUsN)g . !&;;=D !!%D!1,+,, ! !s AA c|j}|jD]3\}}t|tjs!|j ||<5t j|dddzS)z Serializes this instance to a JSON string. Returns: `str`: String containing all the attributes that make up this feature_extractor instance in JSON format. T)indent sort_keys )rr[r.rGrHtolistrdumps)r' dictionaryr`rLs r)to_json_stringz%FeatureExtractionMixin.to_json_stringhsb\\^ $**, 1JC%,"',,. 3 1zz*Q$?$FFr*json_file_pathct|dd5}|j|jdddy#1swYyxYw)z Save this instance to a JSON file. Args: json_file_path (`str` or `os.PathLike`): Path to the JSON file in which this feature_extractor instance's parameters will be saved. wrrN)rwriter)r'rwriters r)rz#FeatureExtractionMixin.to_json_filews<.# 8 0F LL,,. / 0 0 0s 8AcT|jjd|jS)N )r(r_rr:s r)__repr__zFeatureExtractionMixin.__repr__s(..))*!D,?,?,A+BCCr*ct|ts |j}ddlmcm}t ||st|d||_y)a Register this class with a given auto class. This should only be used for custom feature extractors as the ones in the library are already mapped with `AutoFeatureExtractor`. Args: auto_class (`str` or `type`, *optional*, defaults to `"AutoFeatureExtractor"`): The auto class to register this new feature extractor with. rNz is not a valid auto class.) r.r/r_transformers.models.automodelsautohasattrr]r)r auto_class auto_modules r)register_for_auto_classz.FeatureExtractionMixin.register_for_auto_classsC*c*#,,J66{J/ |+FGH H$r*)NFFNmain)F)AutoFeatureExtractor)r_rurvrwrr% classmethodr^rr/rPathLikeboolrrrDrxrrrrrrrrrrrgr*r)rr s K /3$!&#'^? . /^?'*R[['8^?$t+^? ^?  ^? TzD ^?^? &^?^?@./cBKK.?./d./`o.,/"++,=o. tCH~tCH~- .o.o.b$%%)#s(^$% '/GcSVh/W)XX Y$%$%L c3h -sR[['8-=U--$ G G 03+< 0D%%r*rrzfeature extractor file)rS object_class object_files)-rwrrr collectionsrtypingrrrrnumpyrGhuggingface_hubrr dynamic_module_utilsr utilsr r rrrrrrrrrrr utils.hubr!feature_extraction_sequence_utilsr get_loggerr_rPreTrainedFeatureExtractorrrrrformatrgr*r)rs 5584#K   H %"#=> ''EMefM8M`O%^O%d &//E/Q/Q%R"%%--91G1S1S1[1[1b1b"1GVn2c2&&.:r*