
    謜i                     N   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZ d dlZddlmZ dd	lmZ  e       rd dlZd d
lmZ d dlmZ  G d de      Z ej0                         j2                  Z e
e      j8                  d   Zde dj=                         Z	 d7dee dz   ejB                  d      f   fdZ" G d d      Z#de ejH                  z  de de fdZ%de
de#de de de&e'e e(f      f
dZ)d e d!e&e    fd"Z*de
d#e de de fd$Z+d#e de de&e'e e(f      fd%Z,d&e d'e-fd(Z.d&e de#d)e d'e'e e e&f   fd*Z/de
de#de de&e'e e(f      d'e f
d+Z0de
de#de&e'e e(f      fd,Z1de
de#de de de&e'e e(f      d-e(fd.Z2	 	 	 d8d/e d0e dz  d1edz  d2e dz  d'ef
d3Z3d4e d'e(fd5Z4d6 Z5y)9    N)Callable)date)Path)	AnnotatedAny   )is_libcst_available   )add_fast_image_processor)
CSTVisitor)matchersc                   z    e Zd ZdZd Zdej                  ddfdZdej                  fdZdej                  fdZ
y)	ClassFinderzC
        A visitor to find all classes in a python module.
        c                 .    g | _         g | _        d| _        y NF)classespublic_classesis_in_class)selfs    _/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/cli/add_new_model_like.py__init__zClassFinder.__init__*   s    !#DL(*D$D    nodereturnNc                 p    | j                   j                  |j                  j                         d| _        y)zwRecord class names. We assume classes always only appear at top-level (i.e. no class definition in function or similar)TN)r   appendnamevaluer   r   r   s     r   visit_ClassDefzClassFinder.visit_ClassDef/   s$    LL		0#Dr   c                     d| _         y r   )r   r   s     r   leave_ClassDefzClassFinder.leave_ClassDef4   s
    $Dr   c                    t        j                  t        j                  t        j                  t        j                               g      g      }| j
                  st        j                  ||      r|j                  d   j                  d   j                  j                  }|dk(  rM|j                  d   j                  j                  }|D cg c]  }|j                  j                   c}| _        yyyyc c}w )z:Record all public classes inside the `__all__` assignment.)target)targets)bodyr   __all__N)mSimpleStatementLineAssignAssignTargetNamer   matchesr&   r%   r$   r   elementsr   )r   r   !simple_top_level_assign_structureassigned_variabler.   elements         r   visit_SimpleStatementLinez%ClassFinder.visit_SimpleStatementLine7   s    010E0Ehhaffh(G'HIJ1- ##		$8Y(Z$(IIaL$8$8$;$B$B$H$H!$	1#yy|11::HNV*W77==+>+>*WD' 2 )[# +Xs   C:)__name__
__module____qualname____doc__r   cstClassDefr    r"   r)   r2    r   r   r   r   %   sG    		%
	$s|| 	$ 	$
	%s|| 	%		X#2I2I 		Xr   r      z
# coding=utf-8
# Copyright aN   the HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
	repo_pathzFWhen not using an editable install, the path to the Transformers repo.)helpc                 h    t               \  }}}}}t        | t        |       nt        |||||       y)zC
    Add a new model to the library, based on an existing one.
    Nr;   old_model_infosnew_lowercase_namenew_model_paper_namefilenames_to_addcreate_fast_image_processor)get_user_input_add_new_model_like_internalr   	REPO_PATHr>   s         r   add_new_model_likerG   Z   sE     	# !%.%:$y/	'-1)$?r   c                       e Zd ZdZdefdZy)
ModelInfoszI
    Retrieve the basic information about an existing model classes.
    lowercase_namec                    ddl m}m} ddlm} ddlm} ddlm} ddl	m
} ddlm} |j                         j                  dd	      j                  d
d	      | _        | j                  |vr| j                  j                  d	d
       | j                  |vrt!        | d      || j                     | _        || j                     | _        | j$                  j                  dd      | _        | j                  |v r7|| j                     | _        | j(                  dk(  rd n| j(                  | _        nd\  | _        | _        |j-                  | j                  d      \  | _        | _        |j-                  | j                  d       | _        |j-                  | j                  d       | _        |j-                  | j                  d       | _        y )Nr   )CONFIG_MAPPING_NAMESMODEL_NAMES_MAPPING)FEATURE_EXTRACTOR_MAPPING_NAMES)IMAGE_PROCESSOR_MAPPING_NAMES)PROCESSOR_MAPPING_NAMES)TOKENIZER_MAPPING_NAMES)VIDEO_PROCESSOR_MAPPING_NAMES _-z is not a valid model nameConfig PreTrainedTokenizerFast)NN)models.auto.configuration_autorL   rM   #models.auto.feature_extraction_autorN   !models.auto.image_processing_autorO   models.auto.processing_autorP   models.auto.tokenization_autorQ   !models.auto.video_processing_autorR   lowerreplacerJ   
ValueError
paper_nameconfig_classcamelcase_namefast_tokenizer_classtokenizer_classgetimage_processor_classfast_image_processor_classvideo_processor_classfeature_extractor_classprocessor_class)	r   rJ   rL   rM   rN   rO   rP   rQ   rR   s	            r   r   zModelInfos.__init__|   s   ^YUIKU -224<<S#FNNsTWX&::''S1&:://IJKK-d.A.AB01D1DE"//77"E "99(?@S@S(TD%115NNTXTmTm % ?I;D $";FcFgFgG
C"D$C &C%F%FtGZGZ\`%a"'F'J'J4K^K^`d'e$6::4;N;NPTUr   N)r3   r4   r5   r6   strr   r9   r   r   rI   rI   w   s    !Vs !Vr   rI   	file_namenew_content	add_afterc                    t        | dd      5 }|j                         }ddd       j                  |d      \  }}||z   |z   |z   }t        | dd      5 }|j                  |       ddd       y# 1 sw Y   RxY w# 1 sw Y   yxY w)ac  
    A utility to add some content inside a given file.

    Args:
        file_name (`str` or `os.PathLike`):
            The name of the file in which we want to insert some content.
        new_content (`str`):
            The content to add.
       add_after (`str`):
           The new content is added just after the first instance matching it.
    rutf-8encodingNr
   w)openreadsplitwrite)rn   ro   rp   fold_contentbeforeafters          r   add_content_to_filer      s     
iw	/ 1ffh  %%i3MFE9${2U:K	iw	/ 1	   s   A1A=1A:=Br?   r@   rA   rB   c           	      x   dj                  d |j                  dd      j                  d      D              }|j                  }|j                  }|dd D 	cg c]  \  }}	|j                  |d      |	f }}}	g }
|D ]B  \  }}	t        j                  d|      r|
d	   \  }}||xs |	f|
d	<   0|
j                  ||	f       D t        | d
z  dz  dz  dz  dz  d| d| dd       t        | d
z  dz  dz  dz  dz  d| d| dd       |
D ]  \  }}	|	s	|j                  dd      }| d
z  dz  dz  dz  |z  j                         }t        j                  d| d|t
        j                        }|D ]?  }t        | d
z  dz  dz  dz  |z  |j                  ||      j                  ||      |       A  yc c}	}w )a  
    Add a model to all the relevant mappings in the auto module.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rW   c              3   <   K   | ]  }|j                           y wNtitle.0xs     r   	<genexpr>z-add_model_to_auto_mappings.<locals>.<genexpr>        `1QWWY`   rU   rT   r
   Nautoz2(?:tokenization)|(?:image_processing)_auto_fast.pysrctransformersmodelszconfiguration_auto.pyz
        ("z", "z
Config"),
zOCONFIG_MAPPING_NAMES = OrderedDict[str, str](
    [
        # Add configs here
ro   rp   z"),
zcMODEL_NAMES_MAPPING = OrderedDict[str, str](
    [
        # Add full (and cased) model names here
_fast.py.pyz( {8,12}\(\s*"z",.*?\),\n)(?: {4,12}\(|\]))joinr`   ry   rJ   rd   researchr   r   	read_textfindallDOTALL)r;   r?   r@   rA   rB   new_cased_nameold_lowercase_nameold_cased_namefilenameto_addcorrected_filenames_to_addfileprevious_fileprevious_to_addmatching_linesmatchs                   r   add_model_to_auto_mappingsr      s?   * WW`0B0J0J3PS0T0Z0Z[^0_``N(77$33NWghihjWkCS8V		,f	5v>  "$( >f99JDQ-G-K*M?.;_=VPV-W&r*&--tVn=> EN*X5>AXX !3 4D8HTf
 EN*X5>AXX !3 4D9M8NeTz 7 &''
E:H%6AFJXU``bDZZ#$6#77TUW[]_]f]fN ( #%6AFJXU %.@BT U ] ]&! $1s   F6new_paper_namer   c                 "   d}t        j                  ddt              j                  dd      |z   }t	        j
                  d|  d|  d      }g }|D ]$  }d	| d
| }d|v r|dz  }|j                  |       & dj                  |      }||z   |z   S )a4  
    Create a new doc file to fill for the new model.

    Args:
        new_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        public_classes (`list[str]`):
            A list of all the public classes that the model will have in the library.
    u   

⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be rendered properly in your Markdown viewer.

-->

z# ?rW   zcoding=utf-8
z<!--z
        # z#

        ## Overview

        The a<   model was proposed in [<INSERT PAPER NAME HERE>](<INSERT PAPER LINK HERE>) by <INSERT AUTHORS HERE>.
        <INSERT SHORT SUMMARY HERE>

        The abstract from the paper is the following:

        <INSERT PAPER ABSTRACT HERE>

        Tips:

        <INSERT TIPS ABOUT MODEL HERE>

        This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/<INSERT YOUR HF USERNAME HERE>).
        The original code can be found [here](<INSERT LINK TO GITHUB REPO HERE>).

        ## Usage examples

        <INSERT SOME NICE EXAMPLES HERE>

        z## z

[[autodoc]] Modelz
    - forward

)r   sub	COPYRIGHTr`   textwrapdedentr   r   )	r   r   
added_notecopyright_for_markdowndoc_templatedoc_for_classesclass_doc	class_docs	            r   create_doc_filer      s    	K   VVFB	:BBCSU[\_ii??
   		L8 O  $F8+F84f$$Cs#	$ O,I!L09<<r   r   c                    | dz  dz  dz  dz  }t        |d      5 }|j                         }ddd       t        j                  d| d      j	                  d	      }d
| d| d}t        | dz  dz  dz  dz  ||       y# 1 sw Y   VxY w)a  
    Insert the new model in the doc `_toctree.yaml`, in the same section as the old model.

    Args:
        old_lowercase_name (`str`):
            The old lowercase model name.
        new_lowercase_name (`str`):
            The old lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
    docssourceenz_toctree.ymlrr   Nz- local: model_doc/z\n {8}title: .*?\nr   z      - local: model_doc/z
        title: 
r   )rw   rx   r   r   groupr   )	r;   r   r@   rA   toc_filer{   contentold_model_tocnew_tocs	            r   insert_model_in_doc_tocr   1  s     6!H,t3nDH	h	 &&( II!45G4HH\]_fgmmnopM)*<)==NOcNddfgGFX%,~=7^k s   BB
c                     |D cg c])  \  }}|j                  | |      j                  dd      |f+ }}}dj                  d |D              }t        t        j                  d| d      z   }|S c c}}w )a/  
    Create the `__init__.py` file to add in the new model folder.

    Args:
        old_lowercase_name (`str`):
            The old lowercase model name.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    r   rW   
            c              3   4   K   | ]  \  }}|s	d | d  yw)zfrom .z	 import *Nr9   )r   r   r   s      r   r   z#create_init_file.<locals>.<genexpr>[  s      #lvekfTF)$<#ls   
z
        from typing import TYPE_CHECKING

        from ...utils import _LazyModule
        from ...utils.import_utils import define_import_structure


        if TYPE_CHECKING:
            z
        else:
            import sys

            _file = globals()["__file__"]
            sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)
        )r`   r   r   r   r   )r   r@   rB   r   r   imports	init_files          r   create_init_filer   J  s     !1Hf 
		,.@	A	I	I%QS	TV\]  ###lQa#llGHOO I 		 I" -s   .A1module_namer   c                     t        | dd      5 }|j                         }ddd       t        j                        }t	               }|j                  |       |j                  |j                  fS # 1 sw Y   QxY w)z
    Find the name of all classes defined in `module_name`, including public ones (defined in `__all__`).

    Args:
        module_name (`str`):
            The full path to the python module from which to extract classes.
    rr   rs   rt   N)rw   rx   r7   parse_moduler   visitr   r   )r   r   source_codemodulevisitors        r   find_all_classes_from_filer   p  sj     
k3	1 "Tiik"k*FmG
LL??G2222" "s   A00A9r   c                 Z  	 t        |       \  }}dj                  | j                  dd       j                  dd      }|j                  	d| ddj                  d	 |D               }d
j                  	fd|D              }|D cg c]  }|j                  	       }}|||fS c c}w )a  
    Extract the modular structure that will be needed to copy a file `module_name` using modular.

    Args:
        module_name (`str`):
            The full path to the python module to copy with modular.
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_cased_name (`str`):
            The new cased model name.
    .Nr   rW   zfrom ..z import z, c              3       K   | ]  }|  y wr   r9   )r   r   s     r   r   z)find_modular_structure.<locals>.<genexpr>  s     :\f6:\s   r   c              3   P   K   | ]  }d |j                         d| d  yw)zclass (z):
    passN)r`   )r   r   r   r   s     r   r   z)find_modular_structure.<locals>.<genexpr>  s1      "[a&?@&V"s   #&)r   r   partsr`   rd   )
r   r?   r   all_classesr   import_locationr   modular_classesr   r   s
     `      @r   find_modular_structurer     s     #=["IKhh{0056>>ubIO$33N(:\P[:\1\0]^Gkk "ep" O Tbbfnn^^DbNbO^33 cs   B(c                    dj                  d |j                  dd      j                  d      D              }|j                  }| dz  dz  dz  |z  }d}d}g }	|D ]=  \  }
}|s	t	        ||
z  ||      \  }}}|d| z  }|d	| z  }|	j                  |       ? d
j                  d |	D              }t        j                  d| d      }t        |z   |z   |z   }|	D cg c]  }|j                  dd       }	}||	fS c c}w )a  
    Create a new modular file which will copy the old model, based on the new name and the different filenames
    (modules) to add.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rW   c              3   <   K   | ]  }|j                           y wr   r   r   s     r   r   z&create_modular_file.<locals>.<genexpr>  r   r   rU   rT   r   r   r   r   r   r   c              3   &   K   | ]	  }| d   yw),Nr9   )r   public_classs     r   r   z&create_modular_file.<locals>.<genexpr>  s     4oL~Q5G4os   z"

        __all__ = [
            z
        ]
        ")	r   r`   ry   rJ   r   extendr   r   r   )r;   r?   r@   rB   r   r   old_folder_rootall_imports
all_bodiesall_public_classesr   r   r   r&   r   public_classes_formattedall_statementmodular_filer   s                      r   create_modular_filer     sW   & WW`0B0J0J3PS0T0Z0Z[^0_``N(77%'.88CFXXO KJ, 6&,B(*O^-)GT> Ry>)KD-'J%%n56  0444o\n4ooOO && '		M {*Z7-GLL^_L,..sB7__+++ `s   C=c           	      f   dj                  d |j                  dd      j                  d      D              }|j                  }|j                  }|dd D cg c]  \  }}d|j                  ||      z   |f }}}g }	|D ]F  \  }
}t        j                  d| d	|
      r|	d
   \  }}||xs |f|	d
<   4|	j                  |
|f       H i }|	D ]  \  }}|s	|j                  ||      }| dz  dz  |z  |z  }|j                         s:t        |d      5 }|j                         }ddd       j                  d      }d}||   j                  d      r|dz  }||   j                  d      rt        dj                  ||d       z   }|j                  ||      ||<    |S c c}}w # 1 sw Y   xY w)a  
    Create the test files for the new model. It basically copies over the old test files and adjust the class names.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rW   c              3   <   K   | ]  }|j                           y wr   r   r   s     r   r   z$create_test_files.<locals>.<genexpr>  r   r   rU   rT   r
   Ntest_z+test_(?:tokenization)|(?:image_processing)_r   r   testsr   rr   r   r   #)r   r`   ry   rJ   rd   r   r   r   is_filerw   rx   
startswithr   )r;   r?   r@   rB   r   r   r   r   r   r   r   r   r   
test_filesnew_fileoriginal_test_fileoriginal_test_pathr{   	test_code
test_linesidxs                        r   create_test_filesr     s    WW`0B0J0J3PS0T0Z0Z[^0_``N(77$33N !1 4Hf 
8##$68JK	KVT 
 "$( >f99CDVCWW_`bfg-G-K*M?.;_=VPV-W&r*&--tVn=> J6 U&!)!1!12DFX!Y!*W!4x!?BT!TWi!i%--/(#. %!FFH	% #.JCS/,,S1q S/,,S1!DIIj.>$??I#,#4#4^^#TJx U" ?*% %s   !F!F''F0	rC   c                    t               st        d      |j                  }| dz  dz  dz  |z  }t        j                  |d       t        | |||      \  }}	t        |d| dz  d	      5 }
|
j                  |       d
d
d
       t        |||      }t        |dz  d	      5 }
|
j                  |       d
d
d
       t        | dz  dz  dz  dz  d| dd       t        | ||||       | dz  dz  |z  }t        j                  |d       t        |dz  d	      5  	 d
d
d
       t        | |||      }|j                         D ].  \  }}t        ||z  d	      5 }
|
j                  |       d
d
d
       0 t        ||	      }t        | dz  dz  dz  dz  | dz  d	      5 }
|
j                  |       d
d
d
       t        | |||       |rt        |       | dz  dz  dz  dz  }t!        j"                  dd|||dg| t         j$                         t!        j"                  dd|||g| t         j$                         t!        j"                  g d| t         j$                         t!        j"                  ddg| t         j$                         t!        j"                  dd|g| t         j$                         y
# 1 sw Y   6xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   9xY w) a  
    Creates a new model module like a given model of the Transformers library.

    Args:
        repo_path (`Path`):
            The path to the root of the Transformers repository.
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
        create_fast_image_processor (`bool`):
            If it makes sense, whether to add a fast processor as well, even if the old model does not have one.
    zHYou need to install `libcst` to run this command -> `pip install libcst`r   r   r   T)exist_okmodular_r   rv   Nz__init__.pyz
    from .z
 import *
zif TYPE_CHECKING:
r   r   r   r   r   	model_docz.md)
model_nameruffcheckz--fix)cwdstdoutformat)pythonzutils/check_doc_toc.pyz--fix_and_overwriter  zutils/sort_auto_mappings.pyz utils/modular_model_converter.py)r	   ra   rJ   osmakedirsr   rw   rz   r   r   r   r   itemsr   r   r   
subprocessrunDEVNULL)r;   r?   r@   rA   rB   rC   r   new_module_folderr   r   r{   r   tests_folderr   r   r   doc_filemodel_init_files                     r   rE   rE     s8   6  cdd(77 "E)N:XEHZZKK!D1 $7?$68H$ L. 
H-?,@"DDc	J a	 !!35GIYZI	-/	5 		 EN*X5E !3 4K@' y/;MOceuv w&14FFLKKt,	l]*C	0 "9o?QScdJ'--/ ',)3/ 	1GGG	 	
 3^DH	i& 8+d2[@FXEYY\C]]_b	c gh	I'9;MOcd # ,>?  %'.88CmSONN	+\?GT!!
 NN	,lOL!!
 NNC[e[m[m NNH;<)T^TfTfg NN	57IJPYblbtbts 
 " 	 	
 s<   +J1!J>KKK%1J;>KKK"	%K/questiondefault_value
convert_tofallback_messagec                     | j                  d      s| dz   } ||  d| d} d}|s=t        |       }|t        |      dk(  r|}|	  ||      }d}nd}|st	        |       |s=S # t        $ r d}Y w xY w)a4  
    A utility function that asks a question to the user to get an answer, potentially looping until it gets a valid
    answer.

    Args:
        question (`str`):
            The question to ask the user.
        default_value (`str`, *optional*):
            A potential default value that will be used when the answer is empty.
        convert_to (`Callable`, *optional*):
            If set, the answer will be passed to this function. If this function raises an error on the provided
            answer, the question will be asked again.
        fallback_message (`str`, *optional*):
            A message that will be displayed each time the question is asked again to the user.

    Returns:
        `Any`: The answer provided by the user (or the default), passed through the potential conversion function.
    rS   z [z] Fr   T)endswithinputlen	Exceptionprint)r  r  r  r  valid_answeranswers         r   get_user_fieldr  h  s    0 S!c> Zr-3Lx$V)9"F!%#F+#  L"#   M  %$%s   
A$ $A21A2r   c                 j    | j                         dv ry| j                         dv ryt        |  d      )z&
    Converts a string to a bool.
    )1yyestrueT)0nnofalseFz0 is not a value that can be converted to a bool.)r_   ra   )r   s    r   convert_to_boolr"    s;     	wwy--wwy--
sJK
LLr   c                     ddl m}  t        | j                               }d}|sut	        d      }||v rd}n`t        | d       t        j                  ||      }t        |      dk\  r.t        |      dkD  rdj                  |      }t        d	| d
       |sut              }t        d      }t        ddj                  d |j                  d      D                    }d}d}	d}
d}d}d}d}|j                  t        d| dt        d      }|j                  t        d| dt        d      }	|j                   t        d| dt        d      }
|j"                  t        d| dt        d      }|j$                  t        d| dt        d      }|j&                  t        d| dt        d      }|j(                  t        d| dt        d      }|j*                  }d| ddfd| ddfd| d|fd| d|	fd | d|
fd | d|fd!| d|fd"| d|fd#| d|ff	}d}|
r|st        d$t        d%d&'      }|||||fS )(zE
    Ask the user for the necessary inputs to add the new model.
    r   )rM   FzWWhat model would you like to duplicate? Please provide it as lowercase, e.g. `llama`): Tz is not a valid model type.r
   z or zDid you mean ?zSWhat is the new model name? Please provide it as snake lowercase, e.g. `new_model`?zUWhat is the fully cased name you would like to appear in the doc (e.g. `NeW ModEl`)? rW   c              3   <   K   | ]  }|j                           y wr   r   r   s     r   r   z!get_user_input.<locals>.<genexpr>  s     OAaggiOr   rT   )r  zHDo you want to create a new tokenizer? If `no`, it will use the same as z (y/n)?z.Please answer yes/no, y/n, true/false or 1/0. )r  r  zMDo you want to create a new fast tokenizer? If `no`, it will use the same as zNDo you want to create a new image processor? If `no`, it will use the same as zSDo you want to create a new fast image processor? If `no`, it will use the same as zNDo you want to create a new video processor? If `no`, it will use the same as zPDo you want to create a new feature extractor? If `no`, it will use the same as zHDo you want to create a new processor? If `no`, it will use the same as configuration_r   	modeling_tokenization_r   image_processing_video_processing_feature_extraction_processing_zA fast image processor can be created from the slow one, but modifications might be needed. Should we add a fast image processor class for this model (recommended) (y/n)? r  z-Please answer yes/no, y/n, true/false or 1/0.)r  r  r  )+transformers.models.auto.configuration_autorM   listkeysr  r  difflibget_close_matchesr  r   rI   r  ry   rf   r"  re   rh   ri   rj   rk   rl   rJ   )rM   model_typesvalid_model_typeold_model_typenear_choicesr?   r@   rA   add_tokenizeradd_fast_tokenizeradd_image_processorr   add_video_processoradd_feature_extractoradd_processorr   rB   rC   s                     r   rD   rD     s^    P*//12K e
 [(#^$$?@A"44^[QL< A%|$q(#);;|#<Ll^156  !0O (] *_ggO1C1I1I#1NOO M$!M&&2&VWeVffmn&M

 ++7+[\j[kkrs&M

 ,,8,\]k\llst&M

 11=#1abpaqqxy&M$
 
 ,,8,\]k\llst&M

 ..: .^_m^nnuv&M!

 &&2&VWeVffmn&M
 )77 ,-S	148'(	,d3+,C	0-@+,H	57IJ/0	46IJ/0	9;ST/0	46IJ12#	68MN)*#	.>
 #(#;&4^&L'
# .0DFVXsssr   r   )NNN)6r0  r  r   r  r   collections.abcr   datetimer   pathlibr   typingr   r   typerutilsr	   r   libcstr7   r   r   r(   r   todayyearCURRENT_YEAR__file__parentsrF   lstripr   rm   ArgumentrG   rI   PathLiker   r.  tupleboolr   r   r   r   setr   r   r   r   rE   r  r"  rD   r9   r   r   <module>rN     s    	 	   $   !  ' >
 !$Xj X< tzz|  N""1%	N  FH 
. 	d
NENN(pqq:&V &VR3#4 3 SV ,@@@ @ 	@
 5d+,@F5=C 5=c 5=p),BE]`2# ## #Y]^cdgimdm^nYo #L3C 3C 3 44'14CF4
3T>422,2,2, 2, 5d+,	2,
 	2,j11&01X\]bcfhlcl]mXn1hccc c 	c
 5d+,c "&cP !%"&#'	..:. 4. Dj	.
 	.bMs Mt Mntr   