
    i                        d Z ddlZddlmZ ddlmZ ddlZddlmc m	Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3  e%jh                  e5      Z6 e$       rddl7m8Z8  G d de      Z9 G d de,      Z: G d de-      Z;	 	 d>dejx                  dejz                  dejz                  d ejz                  d!eejz                  d"f   d#e>dz  d$e>dz  d%e?ejz                  ejz                  f   fd&Z@ e       ZAe@eAd'<    G d( d)ejx                        ZB G d* d+e*      ZC G d, d-ejx                        ZD G d. d/e      ZE G d0 d1e+      ZF G d2 d3e3      ZG	 	 	 	 d?d4ejz                  e?ejz                     z  dz  d5eHdz  d6eHdz  d7eHd!ejz                  dz  d%ejz                  eHz  fd8ZI G d9 d:e2      ZJ G d; d<e)      ZKg d=ZLy)@zPyTorch Doge model.    N)Callable)Union)nn   )initialization)ACT2FN)Cache)PreTrainedConfig)compile_friendly_flex_attention)GradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)RopeParameters)AttentionInterfacePreTrainedModel)Unpack)TransformersKwargsis_torch_flex_attn_availablelogging)OutputRecorder   )LlamaForSequenceClassificationLlamaMLPLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forward	repeat_kv)MixtralForCausalLMMixtralModel)	BlockMaskc            :           e Zd ZdZdZdgZddddddddddddZd	gd
gfddgdgfdgdgfdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d,dedz  dedz  dedz  dedz  de	dz  de
dz  de	dz  dedz  dedz  dedz  dedz  deee
ef   z  dz  dedz  dedz  dedz  de	dz  dedz  d edz  d!edz  d"edz  d#edz  d$edz  d%edz  d&edz  d'e	dz  d(edz  d)edz  d*edz  f8 fd+Z xZS )-
DogeConfiga  
    This is the configuration class to store the configuration of a [`DogeModel`]. It is used to instantiate an Doge
    model according to the specified arguments, defining the model architecture like [SmallDoge/Doge-320M](https://huggingface.co/SmallDoge/Doge-320M).

    Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PreTrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 32768):
            Vocabulary size of the Doge2 model. Defines the number of different tokens that can be represented by the `inputs_ids` passed when calling [`DogeModel`]
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 2048):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        hidden_dropout (`float`, *optional*, defaults to 0.0):
            Dropout probability for each sequence transformation and state transformation module.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether the model's input and output word embeddings should be tied.
        max_position_embeddings (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model might ever be used with.
        rope_parameters (`RopeParameters`, *optional*):
            Dictionary containing the configuration parameters for the RoPE embeddings. The dictionary should contain
            a value for `rope_theta` and optionally parameters used for scaling in case you want to use RoPE
            with longer `max_position_embeddings`.
        num_attention_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention.
            If `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
            When converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed by meanpooling all the original heads within that group.
            For more details checkout [this paper](https://huggingface.co/papers/2305.13245).
            If it is not specified, will default to `num_attention_heads`.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        mlp_bias (`bool`, *optional*, defaults to `False`):
            Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.
        sliding_window (`int`, *optional*):
            Sliding window attention window size. If not specified, will default to `None`.
        keep_window_size (`int`, *optional*, defaults to 2048):
            The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
        is_moe (`bool`, *optional*, defaults to `False`):
            Whether to use the Cross Domain Mixture of Experts, if `True`, the MoE will inherit the MLP to initialize.
        num_experts (`int`, *optional*, defaults to 16384):
            Number of routed experts in the model. This is only used when `is_moe=True`.
        num_experts_per_tok (`int`, *optional*, defaults to 64):
            Number of selected experts to route per-token.
        norm_topk_prob (`bool`, *optional*, defaults to `False`):
            Whether to normalize the topk probabilities.
        output_router_logits (`bool`, *optional*, defaults to `False`):
            Whether or not the router logits should be returned by the model. Enabling this will also
            allow the model to output the auxiliary loss, including load balancing loss and router z-loss.
        router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
            The aux loss factor for the total loss.
        pad_token_id (`int`, *optional*):
            Padding token id.
        bos_token_id (`int`, *optional*):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*):
            End of stream token id.

    ```python
    >>> from transformers import DogeConfig, DogeModel

    >>> # Initializing a Doge-320M style configuration
    >>> configuration = DogeConfig()

    >>> # Initializing a model from the Doge-320M style configuration
    >>> model = DogeModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```dogepast_key_valuescolwiserowwisecolwise_gather_outputrowwise_split_input)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.dt_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projzlayers.*.mlp.router_gatezlayers.*.mlp.down_embedzlayers.*.mlp.up_embed	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormN
vocab_sizehidden_sizeintermediate_sizenum_hidden_layershidden_dropout
hidden_actinitializer_rangerms_norm_eps	use_cachetie_word_embeddingsmax_position_embeddingsrope_parametersnum_attention_headsnum_key_value_headsattention_biasattention_dropoutmlp_biassliding_windowkeep_window_sizeis_moenum_expertsnum_experts_per_toknorm_topk_proboutput_router_logitsrouter_aux_loss_coefpad_token_idbos_token_ideos_token_idc                    || _         || _        || _        || _        || _        || _        || _        || _        |	| _        || _	        || _
        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        |
| _        || _        || _        || _        || _        ||| _        t9        | t  di | y )N )r2   r3   r4   r5   r6   r7   r8   r9   r:   r<   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   r;   rK   rL   rM   r=   super__init__)selfr2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   kwargs	__class__s                                 a/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/models/doge/modular_doge.pyrQ   zDogeConfig.__init__   s   @ %&!2!2,$!2("'>$#6 #6 ,!2 , 0&#6 ,$8!$8!#6 (((. &':D$"6"    )i   i                  silug{Gz?gư>TFrW   N   NFrY   FNrW   Fi @  @   FFgMbP?NNN)__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planintfloatstrboolr   dictrQ   __classcell__rT   s   @rU   r$   r$   9   s   Un J#4"5 &/%.%.&/%."+ )"+$;#8!6 &(9:#%568IJ!"_$56 "'"&(,(*'*!'*.#(!%+0.2MQ*+*.&+*- %%)'+#"'*,&+,1-2#'#'#';C#$JC# 4ZC# :	C#
 :C# C# $JC# !4<C# DjC# $;C# "D[C# "%tC# ($sN/B*CCdJC# !4ZC# !4ZC#  t!C#" !4<#C#$ +%C#& d
'C#( *)C#* t+C#, 4Z-C#. !4Z/C#0 t1C#2 #Tk3C#4 $dl5C#6 Dj7C#8 Dj9C#: Dj;C# C#rV   r$   c                       e Zd Zy)DogeRMSNormNr]   r^   r_   rO   rV   rU   rm   rm          rV   rm   c                       e Zd Zy)DogeRotaryEmbeddingNrn   rO   rV   rU   rq   rq      ro   rV   rq   modulequerykeyvaluer.   r"   scalingsoftcapreturnc           
      .   d }d t        |t              r|}n|d d d d d d d |j                  d   f   fd}	t        ||||	|d|d      \  }
}|j	                  |j
                        }|
j                  dd      j                         }
|
|fS )Nc                 h    t        j                  | z        z  } | |   |   |   |   z   } | S N)torchtanh)score	batch_idxhead_idxq_idxkv_idxcausal_maskrw   s        rU   	score_modz)flex_attention_forward.<locals>.score_mod	  sI    ejj99E"K	28<UCFKKErV   T)r   
block_mask
enable_gqascale
return_lse   r   )
isinstancer"   shaper   todtype	transpose
contiguous)rr   rs   rt   ru   r.   rv   rw   rS   r   r   attn_outputattention_weightsr   s         `     @rU   flex_attention_forwardr      s     JK.),#
$!!Q?SYYr]?":; &E &"K" *,,U[[9''1-88:K)))rV   doge_flex_attentionc                       e Zd Zddededz  f fdZ	 	 	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	ej                  dz  d
eej                  ej                  dz  eej                     dz  f   fdZ	 	 ddej                  dej                  dedej                  dz  fdZ xZS )DogeAttentionNconfig	layer_idxc                    t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j&                  t)        j*                  |j                              | _        t        j                  |j                  | j                  z  |j                  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        t3        | j                  |j4                        | _        t3        | j                  |j4                        | _        y )Nhead_dimg      ࿩biaseps)rP   rQ   r   r   getattrr3   r>   r   r?   num_key_value_groupsrv   rA   rD   r   Linearr@   q_projk_projv_proj	Parameterr}   zerosAdt_projo_projrm   r9   q_normk_normrR   r   r   rT   s      rU   rQ   zDogeAttention.__init__(  s   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9 & 7 7ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ekk&*D*DEFyy&&68R8RY_YnYn
 ii&&68J8JQWQfQf
 "$--V5H5HI!$--V5H5HIrV   r-   position_embeddingsr.   r&   cache_positionrx   c                    |j                   d d }g |d| j                  }| j                  | j                  |      j	                  |            j                  dd      }	| j                  | j                  |      j	                  |            j                  dd      }
| j                  |      j	                  |      j                  dd      }|\  }}t        |	|
||      \  }	}
|'|||d}|j                  |
|| j                  |      \  }
}| j                  |j                  dd      j                  |j                   d   |j                   d   d            }t        j                  | j                   t#        j$                  |      z        j                  dd      }| j'                  ||| j(                  |      }t+        || j,                        }t.        j1                  | j2                  j4                  t6              } || |	|
|f|| j8                  sdn| j:                  | j<                  d	|\  }} |j                  g |d j?                         }| jA                  |      }||fS )
Nr   r   )sincosr   r   rz   )r-   	dt_statesrD   r.   rY   )r.   dropoutrv   )!r   r   r   r   viewr   r   r   r   r   updater   r   reshaper}   expr   Fsoftplusprepare_dynamic_maskrD   r   r   ALL_ATTENTION_FUNCTIONSget_interfacer   _attn_implementationr   trainingrA   rv   r   r   )rR   r-   r   r.   r&   r   rS   input_shapehidden_shapequery_states
key_statesvalue_statesr   r   cache_kwargsr   	attn_maskattention_interfacer   attn_weightss                       rU   forwardzDogeAttention.forwardF  se    $))#2.88b8$--8{{4;;}#=#B#B<#PQ[[\]_`a[[]!;!@!@!NOYYZ[]^_
{{=166|DNNqRST&S#7jRUWZ#[ j&#&snUL'6'='=j,X\XfXfht'u$J LL""1a(001C1CA1FHZHZ[]H^`bc
	 IIdffqzz)'<<=GGBO	--'!22)	 . 
	 i)B)BC	(?(M(MKK,,.E)
 %8		%

 %#}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((rV   r   rD   c           	         t        j                  |j                        j                  }|j                  }|dddddddf   j	                  dd|j
                  d   d      }|t        |t              s|j                  t         j                  k(  rC|j                  }t        j                  |t        j                  d|j                  |      |      }|j                  |ddddddd|j
                  d   f   dk7  |      }|j
                  d   |kD  rnt        j                  |||j                        }t        j                  ||ddd	
      j                  }	|j!                  d|	d      }|j                  |dk(  |      }|S )a8  
        The core idea of DMA is to calculate the dynamic attention mask to mask the tokens that should be masked, so as to form sparse attention.

        Combine `dt_states` with `attention_mask` to generate the final `attn_mask`.

        Args:
            hidden_states (`torch.Tensor`): The input hidden_states, used to determine the minimum value of the current input precision.
            dt_states (`torch.Tensor`): dt_states of shape `(batch_size, num_heads, key_sequence_length)`.
            keep_window_size (`int`): The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
            attention_mask (`torch.Tensor`, *optional*): attention mask of shape `(batch_size, 1, query_sequence_length, key_sequence_length)`.
        Nr   r   rY   )devicer   r   r   r   TF)dimlargestsortedg      ?)r}   finfor   minexpandr   r   r"   rh   wheretensorr   masked_fill
zeros_liketopkindicesscatter)
rR   r-   r   rD   r.   	min_dtyper   r   active_masktopk_indicess
             rU   r   z"DogeAttention.prepare_dynamic_mask~  sg   $ KK 3 3488	##aD!m,33M''*B
	 %j.S##uzz1%++!&"ELL^=R=RZ_$`bk" "--nQ1F[	XZH[F[=[.\`a.aclmI??2!11**9E)JZJZ[K ::i1ArSW`efnnL%--b,DK!--kS.@)LIrV   r|   )NNN)rW   N)r]   r^   r_   r$   re   rQ   r}   Tensortupler	   
LongTensorr   r   rj   rk   s   @rU   r   r   '  s   Jz JcDj JD /3(,266)||6) #5<<#=>6) t+	6)
 6) ((4/6) 
u||U\\D0%2E2LL	M6)x !%.2#||# <<# 	#
 t+#rV   r   c                       e Zd Zy)DogeMLPNrn   rO   rV   rU   r   r     ro   rV   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )	DogeCDMoEr   c                    t         |           |j                  | _        |j                  | _        t        |j
                     | _        |j                  | _        t        j                  t        j                  | j                              | _        |j                  | _        |j                  | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  |j"                        | _        t        j                   | j                  | j                  dz  d      | _        t        j,                  | j                  | j                        | _        t        j,                  | j                  | j                        | _        y )Nr   r   F)rP   rQ   r3   r4   r   r7   act_fnrF   mathfloorsqrtnum_keysrG   top_krH   r   r   rB   	gate_projup_proj	down_projrouter_gate	Embedding
down_embedup_embedrR   r   rT   s     rU   rQ   zDogeCDMoE.__init__  s_   !--!'!9!9V../!--

499T-=-=#>?//
$33 4#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRab 99T%5%5t}}q7HuU ,,t'7'79I9IJT%5%5t7G7GHrV   r-   rx   c                    |j                   \  }}}| j                  |      j                  d||z  d      }|j                  | j                  d      \  \  }}\  }	}
|j                  d      |j                  d      z   }|	j                  d      | j                  z  |
j                  d      z   } |j                  g |j                   d d d } |j                  g |j                   d d d }|j                  | j                  d      \  }}|j                  d|      }t        j                  |d      }| j                  r||j                  dd      z  }| j                  |      }| j                  |      }t        j                  ||j                  ||z  dd            j                  ||z  d      }| j!                  |      |z  }t        j                  |j                  ||z  dd      |      j                  ||d      }| j#                  | j!                  | j%                  |            | j'                  |      z        }||z   }||fS )Nr   r   r   rz   T)r   keepdimr   )r   r   r   r   r   	unsqueezer   gatherr   softmaxrH   sumr   r   r}   matmulr   r   r   r   )rR   r-   rS   bszseq_len_router_logitsscores_xscores_y	indices_x	indices_y
all_scoresall_indicesscoresposition_indicesr   routing_weightsr   r   experts_weightsexperts_statess                        rU   r   zDogeCDMoE.forward  sF   
 (--Wa ((7<<QgrR 8E7I7I$--]_7I7`484y)''+h.@.@.DD
))"-=	@S@STV@WW$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC#-??4::2?#F  $$R)9:))F322r42HHO __W-
==),,z=3E3EcGmUWYZ3[\aabehoboqst++o6Ho&:&:3=!R&PRZ[``adfmoqrt{{4>>-3P'QTXT`T`anTo'op%6m++rV   )	r]   r^   r_   r$   rQ   r}   r   r   rj   rk   s   @rU   r   r     s0    Iz I.,||, 
	,rV   r   c                   l    e Zd Zddededz  f fdZ	 	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dej                  dz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )DogeDecoderLayerNr   r   c                 *   t         |           |j                  | _        t        |j                  |j
                        | _        t        ||      | _        t        j                  t        j                  |j                              | _        t        |j                  |j
                        | _        |j                  st!        |      n
t#        |      | _        t        j                  t        j                  |j                              | _        y )Nr   )r   r   )rP   rQ   r6   rm   r3   r9   input_layernormr   	self_attnr   r   r}   onesinput_residualpost_attention_layernormrE   r   r   mlppost_attention_residualr   s      rU   rQ   zDogeDecoderLayer.__init__  s    $33*6+=+=6CVCVW&f	J ll5::f6H6H+IJ(3F4F4FFL_L_(`%*0--76?Yv=N')||EJJv?Q?Q4R'S$rV   r-   r   r.   position_idsr&   r:   r   rS   rx   c                    |}	| j                  |      } | j                  d|||||||d|\  }}
t        j                  || j                  | j
                        }| j                  |	z  |z   }|}	| j                  |      }| j                  |      }t        j                  || j                  | j
                        }| j                  |	z  |z   }|S )N)r-   r   r.   r  r&   r:   r   )pr   rO   )
r  r  r   r   r6   r   r  r  r  r  )rR   r-   r   r.   r  r&   r:   r   rS   residualself_attn_weightss              rU   r   zDogeDecoderLayer.forward  s     !,,];+94>> 	,
' 3)%+)	,
 	,
(( 		-43F3FQUQ^Q^_++h6F !55mD/		-43F3FQUQ^Q^_44x?-OrV   r|   )NNNNFN)r]   r^   r_   r$   re   rQ   r}   r   r   r   r	   rh   r   r   FloatTensorr   rj   rk   s   @rU   r  r    s   
Tz 
TcDj 
T IM.204(,!&26"||" #5<<#=>E" t+	"
 &&-" " $;" ((4/" +," 
u  %(9(95;L;L(L"MPT"TT	U"rV   r  c                   ^    e Zd ZdZdZ eed      eedZ	 e
j                         d        Zy)DogePreTrainedModelFr   )index)r   r-   
attentionsc                 z   t        j                  | |       t        |t              r-t	        |d      r t        j                  |j                         yyt        |t              rXt	        |d      rt        j                  |j                         t	        |d      r t        j                  |j                         yyy)zInitialize the weightsr   r  r  N)r   _init_weightsr   r   hasattrinitzeros_r   r  ones_r  r  )rR   rr   s     rU   r  z!DogePreTrainedModel._init_weights  s     	%%dF3fm,vs#FHH% $ 01v/0

6001v89

699: : 2rV   N)r]   r^   r_   _supports_flash_attn_can_compile_fullgraphr   r   r  r   _can_record_outputsr}   no_gradr  rO   rV   rU   r  r    sB     "'	;)# U]]_
; 
;rV   r  c                       e Zd Zy)	DogeModelNrn   rO   rV   rU   r)  r)  *  ro   rV   r)  gate_logitsrF   r   r   c                    | t        | t              sy| d   j                  }| d   j                  }g }g }| D ]  }	|	j	                  |      }	|	j                  |d      \  \  }
}\  }}|
j                  d      |j                  d      z   }|j                  d      |z  |j                  d      z   } |j                  g |j                  dd d } |j                  g |j                  dd d }|j                  |d      \  }}|j                  d|      }t        j                  |d      }|j                  |       |j                  |        t        j                  |d      }t        j                  |d      }|}|j                  d      }t        j                  |||      }t        j                   |||      }|j#                  d||      |j                  d   z  }t        j$                  |d      }nD|j                  \  }}t'        |       }|ddddddf   j)                  ||||f      j+                  d      j	                  |      }|j                  d      |j-                            }t        j                  |||      }t        j                   |||      }|j#                  d||      t        j.                  |      z  }|ddddddf   j)                  ||||f      j+                  d|      j	                  |      }t        j.                  ||z  d      t        j.                  |d      z  }t        j.                  ||z        }||z  S )a  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `router_gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [2, batch_size * sequence_length, num_keys].
        num_experts:
            Number of experts
        num_keys:
            Number of keys
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   r   r   rz   r   )r   r   r   r   r   r   r   r   r   r   r   r   appendr}   catr   	ones_likescatter_add_meanlenr   r   rh   r   )r*  rF   r   r   r.   compute_dtypecompute_deviceall_expert_indicesall_routing_weightslayer_gate_logitsr   r   r  r  r  r  r   r  expert_indicesr  tokens_per_expertpadrouter_prob_per_expert
batch_sizesequence_lengthr5   expert_attention_mask router_per_expert_attention_maskoverall_losss                                rU   load_balancing_loss_funcr@  .  s{   @ *[%"@N((M ^**N( 4-00@7H7M7Mh\^7M7_484y)''+h.@.@.DD
))"-89;N;Nr;RR$Z__@j&6&6s&;@R@
&k&&C(9(9#2(>CC(ooeo<$++B0@A))JB7!!.1""?3!4" #51=))$7Q?/44R8!KK=Q_`oo0n]-::1>PRUVYkYqYqrsYtt "',?Q!G&4&:&:#
O, 4At+,V&
OUKLWR[R	 	 044R89N9S9S9UV "KK=Q_`oo0n]-::1>PRUVY^YbYb!Z
 
 4At+,V&
O[QRWR%R	 	) "'+>Aa+agh!ilqlulu,!m
 "
 99.1GGHL+%%rV   c                   :    e Zd Z fdZ	 	 	 	 	 	 	 	 	 	 ddej
                  dz  dej                  dz  dej
                  dz  dedz  dej                  dz  dej
                  dz  d	e	dz  d
ej
                  dz  de
ej                  z  de	dz  dee   defdZ xZS )DogeForCausalLMc                 f    t         |   |       t        |      | _        |j                  | _        y r|   )rP   rQ   r)  modelrF   r   s     rU   rQ   zDogeForCausalLM.__init__  s*     v&
!--rV   Nr+   r.   r  r&   r,   labelsr:   r   logits_to_keeprI   rS   rx   c                    |
|
n| j                   j                  }
 | j                  d|||||||d|}|j                  }t	        |	t
              rt        |	 d      n|	}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |}d}|
rt        |j                  | j                  t        j                  t        j                  | j                              | j                   |      }|+|| j"                  |j%                  |j&                        z  z  }t)        ||||j*                  |j,                  |j.                  |j                        S )ah  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, DogeForCausalLM

        >>> model = DogeForCausalLM.from_pretrained("SmallDoge/Doge-320M")
        >>> tokenizer = AutoTokenizer.from_pretrained("SmallDoge/Doge-320M")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r+   r.   r  r&   r,   r:   r   )lossaux_losslogitsr&   r-   r  r   rO   )r   rI   rD  last_hidden_stater   re   slicelm_headloss_functionr2   r@  r   rF   r   r   r   rG   rJ   r   r   r   r&   r-   r  )rR   r+   r.   r  r&   r,   rE  r:   r   rF  rI   rS   outputsr-   slice_indicesrJ  rH  rI  s                     rU   r   zDogeForCausalLM.forward  sp   J %9$D $++JjJj 	
 +5$** 	+
)%+')	+
 	+
  118B>SV8W~ot4]kmA}a,?@A%4%%ffdooPPD/%%  

499T%5%567((H !11HKK4LLL(#33!//))!//
 	
rV   )
NNNNNNNNr   N)r]   r^   r_   rQ   r}   r   r   r	   r  rh   re   r   r   r   r   rj   rk   s   @rU   rB  rB    s   . .2.204(,26*.!%26-.,0Q
##d*Q
 t+Q
 &&-	Q

 Q
 ((4/Q
   4'Q
 $;Q
 ((4/Q
 ell*Q
 #TkQ
 +,Q
 
#Q
rV   rB  c                       e Zd Zy)DogeForSequenceClassificationNrn   rO   rV   rU   rR  rR    ro   rV   rR  )r$   rB  r)  r  rR  )NN)NNr   N)Mr`   r   collections.abcr   typingr   r}   torch.nn.functionalr   
functionalr    r   r!  activationsr   cache_utilsr	   configuration_utilsr
   integrations.flex_attentionr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.output_capturingr   llama.modeling_llamar   r   r   r   r   r   r   r   mixtral.modeling_mixtralr    r!   
get_loggerr]   logger!torch.nn.attention.flex_attentionr"   r$   rm   rq   Moduler   rf   r   r   r   r   r   r   r  r  r)  re   r@  rB  rR  __all__rO   rV   rU   <module>rj     sI      $      & !   3 J 9 Q 1 A & N N 4	 	 	 H 
		H	%!;q#! q#h	, 		. 	 ! +*II+*<<+* 
+* <<	+*
 %,,34+* T\+* T\+* 5<<%&+*\ -. 1G - .zBII zz	h 	6,		 6,r/1 /d;. ;.	 	 #*.g&ell 33d:g&tg& Djg& 	g&
 LL4'g& \\Cg&TW
( W
t	$B 	rV   