
    i/                     l    d Z ddlmZ ddlmZ ddlmZ  ej                  e      Z	 G d de      Z
dgZy)zPhi-3 model configuration   )PreTrainedConfig)RopeParameters)loggingc            ,           e Zd ZdZdZdgZdddddZdgdgfd	d
gd	gfd	gd	gfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d'dedz  dedz  dedz  dedz  dedz  dedz  de	dz  de	dz  de	dz  de
dz  dedz  dedz  de	dz  dedz  dedz  dedz  deee
ef   z  dz  dedz  dedz  d edz  d!edz  f* fd"Z	 d(d#ee	z  d$edz  fd%Zd)d$edz  f fd&Z xZS )*
Phi3Configa  
    This is the configuration class to store the configuration of a [`Phi3Model`]. It is used to instantiate a Phi-3
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the
    [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct).

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 32064):
            Vocabulary size of the Phi-3 model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`Phi3Model`].
        hidden_size (`int`, *optional*, defaults to 3072):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 8192):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 32):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details, check out [this
            paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
            `num_attention_heads`.
        resid_pdrop (`float`, *optional*, defaults to 0.0):
            Dropout probability for mlp outputs.
        embd_pdrop (`int`, *optional*, defaults to 0.0):
            The dropout ratio for the embeddings.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio after computing the attention scores.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 4096):
            The maximum sequence length that this model might ever be used with.
        original_max_position_embeddings (`int`, *optional*, defaults to 4096):
            The maximum sequence length that this model was trained with. This is used to determine the size of the
            original RoPE embeddings when using long scaling.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon value used for the RMSNorm.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
        rope_parameters (`RopeParameters`, *optional*):
            Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
            a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE
            with longer `max_position_embeddings`.
        bos_token_id (`int`, *optional*, defaults to 1):
            The id of the "beginning-of-sequence" token.
        eos_token_id (`int`, *optional*, defaults to 32000):
            The id of the "end-of-sequence" token.
        pad_token_id (`int`, *optional*, defaults to 32000):
            The id of the padding token.
        sliding_window (`int`, *optional*):
            Sliding window attention window size. If `None`, no sliding window is applied.

    Example:

    ```python
    >>> from transformers import Phi3Model, Phi3Config

    >>> # Initializing a Phi-3 style configuration
    >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-mini-4k-instruct")

    >>> # Initializing a model from the configuration
    >>> model = Phi3Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```phi3past_key_valuescolwise_gather_outputrowwise_split_input)zlayers.*.self_attn.qkv_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormN
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_headsresid_pdrop
embd_pdropattention_dropout
hidden_actmax_position_embeddings original_max_position_embeddingsinitializer_rangerms_norm_eps	use_cachetie_word_embeddingsrope_parametersbos_token_ideos_token_idpad_token_idsliding_windowc                 v   || _         || _        || _        || _        || _        ||}|| _        || _        || _        |	| _        |
| _	        || _
        || _        || _        || _        || _        || _        |j!                  dd       || _        || _        || _        || _        || _        t-        | \  di | y )Npartial_rotary_factorg      ? )r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r#   
setdefaultr'   r$   r%   r&   r"   super__init__)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   kwargs	__class__s                          g/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/models/phi3/configuration_phi3.pyr-   zPhi3Config.__init__w   s    2 %&!2!2#6 &"5#6 &$!2$'>$0P-!2(".137,(((#6 "6"    default_thetaignore_keysc                    |j                  dd       }|xs | j                  | _        | j                  | j                  ni | _        | j                  j                  d|j                  d|             | j                  j                  d|d          | j                          | j                  j	                  dd       }||dv rd| j                  d<   | j                  |       |S )Nrope_scaling
rope_thetar)   	rope_type)suyarnlongroper4   )popr#   r+   standardize_rope_paramsgetvalidate_rope)r.   r3   r4   r/   r6   rope_parameters_types         r1   convert_rope_params_to_dictz&Phi3Config.convert_rope_params_to_dict   s     zz.$7+Ct/C/C7;7K7K7Wt33]_ 	''fjj}6]^''(?H_A`a$$&  $3377TJ+0D0V0:D  -{3r2   c                 <   t         |   |       t        | j                  t              st        d| j                         | j                  j                  dd      }| j                  j                  dd      }| j                  j                  dd      }t        | j                  | j                  z  | j                  d   z        }|dvrt        d	|       |^t        |t              rt        d
 |D              st        d|       t        |      |dz  k(  st        d|dz   dt        |             |_t        |t              rt        d |D              st        d|       t        |      |dz  k(  st        d|dz   dt        |             yy)z?
        Validate the `rope_parameters` configuration.
        r<   z/`rope_parameters` must be a dictionary but got r8   Nshort_factorlong_factorr)   )defaultr;   z@`rope_parameters`'s type field must be one of ['longrope'], got c              3   H   K   | ]  }t        |t        t        f        y wN
isinstanceintfloat.0xs     r1   	<genexpr>z+Phi3Config.validate_rope.<locals>.<genexpr>   s     Z
1sEl3Z    "zF`rope_parameters`'s short_factor field must be a list of numbers, got    z8`rope_parameters`'s short_factor field must have length z, got c              3   H   K   | ]  }t        |t        t        f        y wrH   rI   rM   s     r1   rP   z+Phi3Config.validate_rope.<locals>.<genexpr>   s     Y
1sEl3YrQ   zE`rope_parameters`'s long_factor field must be a list of numbers, got z7`rope_parameters`'s long_factor field must have length )r,   r@   rJ   r#   dict
ValueErrorr?   rK   r   r   listalllen)r.   r4   rA   rope_parameters_short_factorrope_parameters_long_factorrotary_ndimsr0   s         r1   r@   zPhi3Config.validate_rope   s    	+6 $..5NtOcOcNdeff#3377TJ'+';';'?'?PT'U$&*&:&:&>&>}d&S# 8 884;O;OPg;hh
  '>>_`t_uvww'37>Z=YZZ \]y\z{  348II N|_`O`Naaghk  mI  iJ  hK  L  '26=Y=XYY [\w[xy  23|q7HH Ml^_N_M``fgj  lG  hH  gI  J  I 3r2   )i@}  i   i        r\   N        r]   r]   silu   r_   g{Gz?gh㈵>TFN    }  ra   N)g     @NrH   )__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planrK   rL   strboolr   rT   r-   setrB   r@   __classcell__)r0   s   @r1   r   r      s   M^ J#4"5'>%:%<"7	 &(9:#%568IJ!"_$56 "'"&(,(**,*.$'#&*-!'.27;*.#'!%+0MQ#$#(#(%)-4#$J4# 4Z4# :	4#
 :4# !4Z4# !4Z4# T\4# DL4# !4<4# $J4# "%t4# +.*4# !4<4# Dj4#  $;!4#" "D[#4#$ ($sN/B*CCdJ%4#& Dj'4#( Dj)4#* Dj+4#, d
-4#n PT 5[BE*&*t * *r2   r   N)re   configuration_utilsr   modeling_rope_utilsr   utilsr   
get_loggerrb   loggerr   __all__r*   r2   r1   <module>rt      s@      3 1  
		H	%Q! Qh .r2   