i_dZddlmZddlZddlmZddlmZddlm Z ddl m Z dd l mZmZdd lmZdd lmZdd lmZdd lmZmZddlmZmZddlmZddlmZm Z m!Z!m"Z"ddl#m$Z$ddl%m&Z&m'Z'ddl(m)Z)ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1ddl2m3Z3e"jhe5Z6Gdde-Z7Gdde.Z8GddejrZ:GddejrZ;Gd d!ejrZ<Gd"d#ejrZ=Gd$d%ejrZ>Gd&d'e)Z?e Gd(d)e,Z@e Gd*d+e+ZAGd,d-e@eZBGd.d/ee@ZCgd0ZDy)1zPyTorch JetMoe model.)CallableN)nn) functional)initialization)ACT2FN)Cache DynamicCache)GenerationMixin)create_causal_mask) GenericForSequenceClassification)MoeCausalLMOutputWithPastMoeModelOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)OutputRecordercapture_outputs)LlamaDecoderLayer) MixtralModelMixtralPreTrainedModelMixtralRMSNormMixtralRotaryEmbeddingapply_rotary_pos_embeager_attention_forwardload_balancing_loss_func) JetMoeConfigc eZdZy) JetMoeRMSNormN__name__ __module__ __qualname__e/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/models/jetmoe/modular_jetmoe.pyr&r&4r,r&c eZdZy)JetMoeRotaryEmbeddingNr'r+r,r-r0r08r.r,r0c6eZdZdedededdffd ZdZxZS)JetMoeParallelExperts num_experts input_size output_sizereturnNct|tjt j ||||_||_||_||_ y)a Initialize the JetMoeParallelExperts module. The experts weights are stored in [num_experts, output_size, input_size] format. Such that it's compatible with many MoE libraries, such as [Megablock](https://github.com/databricks/megablocks) and [ScatterMoE](https://github.com/shawntan/scattermoe), as well as the [MoE kernel](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/fused_moe/fused_moe.py) used in vllm. Args: num_experts (int): Number of experts. input_size (int): Size of the input. output_size (int): Size of the output. N) super__init__r Parametertorchemptyweightr3r4r5)selfr3r4r5 __class__s r-r9zJetMoeParallelExperts.__init__=sD" ll5;;{K#TU &$&r,c|j|d}g}t|jD]7}|jt j |||j |9tj|d}|S)a Forward pass of the JetMoeParallelExperts module. Args: inputs (Tensor): Input tensor. expert_size: Expert size information. Returns: Tensor: Output tensor. rdim) splitranger3appendFlinearr=r;cat)r>inputs expert_size input_list output_listiresultss r-forwardzJetMoeParallelExperts.forwardTsq\\+1\5  t''( HA   qxx 1 t{{1~F G H))KQ/r,r(r)r*intr9rO __classcell__r?s@r-r2r2<s)'C'S's't'.r,r2c2eZdZdededeffd ZdZxZS)JetMoeTopKGatingr4r3top_kct|||_||_||_t j ||d|_y)a Initialize the top-k gating mechanism. Args: input_size (`int`): Size of the input. num_experts (`int`): Number of experts. top_k (`int`): Number of top experts to select. FbiasN)r8r9r3r4rVrLinearlayer)r>r4r3rVr?s r-r9zJetMoeTopKGating.__init__js: &$ YYz;UC r,c|j|j}|j|jd\}}t j |dj |}t j|jd|jg|j|j}|jd|d}|jjd}|j}|j!} | j#d\} } | j%|jd} |j!}|| } | | | ||fS)Nr#rArdtypedevicetrunc) rounding_mode)r[floattopkrVr;softmaxtype_aszerossizer3r^r_scatterlongsumtolistflattensortdiv)r> hidden_stateslogits top_k_logits top_k_indices top_k_gatesrfgatesrJ top_k_experts_index_sorted_experts batch_index batch_gatess r-rOzJetMoeTopKGating.forward~s.M*002&,kk$**!k&D# mmmLa8@@O    a $"2"2 3;;L;LU`UgUg  a2jjl&&q) "((* &--/ "/"4"4Q"7 *..tzz.Q "))+ !"67 #[+{FRRr,rPrSs@r-rUrUis'D3DSDD(Sr,rUc.eZdZdZdeffd ZdZxZS) JetMoeMoEz A Sparsely gated mixture of experts layer with 1-layer Feed-Forward networks as experts. Args: config: Configuration object with model hyperparameters. configc@t||j|_|j|_t |j |_tjjtj|j|_ t|j|j|jdz|_t|j|j|j|_t#|j|j|j$|_y)Nrr4r3rV)r8r9 hidden_sizer4intermediate_sizeractivation_function activationr;rr:r<rYr2num_local_experts input_linear output_linearrUnum_experts_per_tokrouterr>r|r?s r-r9zJetMoeMoE.__init__s  ,,!33 !;!;<HH&&u{{4??'CD 1&2J2JDOO]a]m]mpq]qr263K3KTM]M]_c_n_no&00,,  r,c8|j\}}}|jd|}|j|\}}}}} ||} |j| |} | j dd} |j | d| dz} |j | |} | |dddfz} tj||z|jf| j| j}|jd|| }|j|||j}||jz}|S)a Forward pass of the mixture of experts layer. Args: layer_input (Tensor): Input tensor. Returns: Tensor: Output tensor. Tensor: Router logits. rrArr#Nr])rgreshaperrchunkrrr;rfr4r^r_ index_addviewrY)r> layer_inputbszlengthemb_sizervrxryrJ router_logits expert_inputsrochunked_hidden_statesexpert_outputsrf layer_outputs r-rOzJetMoeMoE.forwards%!, 0 0 2VX!))"h7 BF++kBZ?; [-#K0 ))-E - 3 3A2 3 >(=a(@ADYZ[D\\ ++M;G'+ag*>> S6\4??;>CWCW`n`u`uvq+~F #((fdooF #dii/ r,)r(r)r*__doc__r$r9rOrRrSs@r-r{r{s | r,r{c:eZdZdZdeffd ZdZdZdZxZ S) JetMoeMoAz A Sparsely gated mixture of attention layer with pairs of query- and output-projections as experts. Args: config: Configuration object with model hyperparameters. r|cht||j|_|j|_|j |jz|_|j|_ tjjtj|j |_t|j|j |j|_t|j|j|j |_t%|j |j|j|_y)Nr~)r8r9rr3rr4 kv_channelsnum_key_value_headsrrVr;rr:r<rYr2rrrUrrs r-r9zJetMoeMoA.__init__s !33 ,,!--0J0JJ// HH&&u{{4??'CD 1$2B2BDOOUYUeUef243C3CTEUEUW[WfWfg&((**  r,c|j\}}}|jd|}|j|\}}}}} ||||f} ||} |j| |} t j ||z|j z|jf| j| j} | jd|| }|j|||j d}|| | fS)z Map inputs to attention experts according to routing decision and compute query projection inside each experts. rr]r) rgrrrr;rfrVrr^r_rr)r>rrrrrwrxryrJr topo_inforrrfrs r-mapz JetMoeMoA.maps !, 0 0 2VX!))"h7 UYU`U`alUmRk; ]); [Q $K0 **=+F 6\DJJ &(8(8 9AUAU^l^s^s q*>O #((fdjj"E ]I55r,c|j\}}}}|jd|}|\}}} } ||} |j| | } | | dddfz} tj||z|j f| j | j} | jd|| }|j|||j }||jz}|S)zu Compute output projection inside each attention experts and merge the outputs of different experts. rNr]r) rgrrr;rfr4r^r_rrrY)r>rrrrkrrwrxryrJrrrfrs r-reducezJetMoeMoA.reduces'2&6&6&8#VQ !))"k: FOCk; $$89 ++M;G(+ag*>> S6\4??;>CWCW`n`u`uvq+~F #((fdooF #dii/ r,ctd)Nz-This module doesn't support call and forward.)NotImplementedError)r>rs r-rOzJetMoeMoA.forwards!"QRRr,) r(r)r*rr$r9rrrOrRrSs@r-rrs$ | $6.,Sr,rc eZdZdZd dededzffd Z ddejdejdzdejdzd e dzd ejdzd e ejejdze ejdzff d Z xZ S)JetMoeAttentionzH Multi-headed attention from 'Attention Is All You Need' paper. Nr| layer_idxcbt|||_||_d|_|-t j d|jjdd|_ |j|_ |j|_ |j|jz|_|j|_|j |_|j|_|j$dz|_t)||_t,j.j1|j2|jdzd |_y) z Initialize the JetMoeAttention module. Args: config: Configuration object with model hyperparameters. layer_idx: Index of the layer in the model. TNzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.r#grFrX)r8r9r|r is_causallogger warning_oncer?r(num_key_value_groupsrrVattention_dropoutrrkv_projection_sizenum_attention_heads num_headshead_dimscalingrexpertsr;rrZrkv_projr>r|rr?s r-r9zJetMoeAttention.__init__%s  "      !8!8 9:,,  %&!// !'!9!9"("4"4v7Q7Q"Q#)#=#= 33** }}d*  ( xxv'9'94;R;RUV;V]bc r,roattention_maskposition_embeddingspast_key_valuescache_positionr6c |jdd}g|d|j}|jj|\} } } |j |j dd\} } | j |jdd} | j |jdd} | j |jdd} |\}}t| | ||\} } |'|||d}|j| | |j|\} } tj|jjt}| j!d|j"dd} | j!d|j"dd} ||| | | |f|j$sdn |j&|j(d|\}}|j g||j"d}|jj+|| }|j g|d}||| fS)NrrrAr#)sincosr)dropoutr)shaperrrrrr transposer updaterr get_interfacer|_attn_implementationr!repeatrVtrainingrrr)r>rorrrrkwargs input_shape hidden_shape query_statesrr key_states value_statesrr cache_kwargsattention_interface attn_output attn_weightss r-rOzJetMoeAttention.forwardFs$))#2. 88b8$--8 151A1A-1P. mY#'<< #>#D#DQB#D#O L#((6@@AF __\2<rorrrrrrrresidualrvs r-rOzJetMoeDecoderLayer.forwards!,,]; 1d11  ')%+) 3     q!!=0 !55mD /  =0 r,r)NNNFNN)r(r)r*r$rQr9r;rrr boolrrrrOrRrSs@r-rr{s|d /304(,!&26HL|| t+&&-    $; ((4/#5<<#=>E+, r,rceZdZUeedeedgeeeddZee d<dZ dZ d gZ d gZ d Zd ZdZej$d Zy )JetMoePreTrainedModelr)indexr#)rro attentionsr|modelFrrTc,tj||t|tr7t j |j d|jjyt|ttzr t j|jyy)zInitialize the weights.r)meanstdN) r _init_weights isinstancer2initnormal_r=r|initializer_rangerr{zeros_rY)r>modules r-rz#JetMoePreTrainedModel._init_weightssa %%dF3 f3 4 LLSdkk6S6S T  I 5 6 KK $7r,N)r(r)r*rrrUr_can_record_outputsr$__annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraphr;no_gradrr+r,r-rrs)BNScklDmn+$_A> &+#-.#4"5N"U]]_%%r,rceZdZdeffd Zeee ddejdzdejdzdejdzde dzdejdzd e dzd ejdzd eed efd ZxZS) JetMoeModelr|c t|||j|_|j|_t j |j|j|j|_t jt|jDcgc]}t||c}|_ |j|_t|j|j |_ycc}w)N)eps)r8r9 pad_token_id padding_idx vocab_sizer Embeddingr embed_tokens ModuleListrDnum_hidden_layersrlayersrr& rms_norm_epsnormrs r-r9zJetMoeModel.__init__s  !.. ++LL):):Fr rrrr rrrpast_seen_tokens causal_maskror decoder_layers r-rOzJetMoeModel.forwards`  -t"; <YZ Z 0*$++>O   --i8M  !CRC^==?de "\\ "2]5H5H5K"KTaThThN  )33A6L(;;'))+%  & #oom\J![[)H4;;+H+HI M) $7* /#-)  M  -0 %++  r,)NNNNNNN)r(r)r*r$r9rrrr;rrr FloatTensorrrrrrOrRrSs@r-rrs O| O .2.204(,26!%26; ##d*;  t+; &&- ;  ; ((4/ ; $;; ((4/; +,;  ;  ; r,rcLeZdZddiZfdZee ddejdzdejdzdejdzde dzd ejdzd ejdzd e dzd ejdzd e ejzde dzdefdZxZS)JetMoeForCausalLMzlm_head.weightzmodel.embed_tokens.weightcpt||t||_|j|_|j |_t j|j|jd|_ |j|_ |j|_ |j|_ |jy)NFrX)r8r9rrr aux_loss_coefrrZrlm_headtie_word_embeddingsrr3r post_initrs r-r9zJetMoeForCausalLM.__init__s   (  ++#11yy!3!3V5F5FUS #)#=#= !33#)#=#=  r,Nr rrrr labelsrrlogits_to_keepoutput_router_logitsr6c N|jd|||||||| d| } | j} t| tr t | dn| }|j | dd|ddf}d}|*|j ||fd|jji| }d}| rYt| j|j|j|}|+||j|j|jzz }t!|||| j"| j$| j&| jS)N)r rrrr rrr r)lossaux_lossrprrorrr+)rr rrQslicer loss_functionr|rr"rr3rrtor_rrror)r>r rrrr rrrrr routputsro slice_indicesrpr"r#s r-rOzJetMoeForCausalLM.forwardsT +5$** + )%+')!5 +  +  11 8B>SV8W~ot4]k mA}a,?@A  %4%% ;;11 D /%%  (( H !**X[[-EEE(#33!//))!//  r,) NNNNNNNNrF)r(r)r*_tied_weights_keysr9rrr;rrr rrrQrrOrRrSs@r-rrs*,GH .2.204(,26*.!%26-.,1; ##d*;  t+; &&- ;  ; ((4/ ;   4'; $;; ((4/; ell*; #Tk;  #; ; r,rc eZdZy)JetMoeForSequenceClassificationNr'r+r,r-r+r+_sr,r+)rrrr+)Ercollections.abcrr;rtorch.nnrrFrr activationsr cache_utilsr r generationr masking_utilsr modeling_layersr modeling_outputsrrmodeling_utilsrrprocessing_utilsrutilsrrrr utils.genericrutils.output_capturingrrllama.modeling_llamarmixtral.modeling_mixtralrrrrr r!r"configuration_jetmoer$ get_loggerr(rr&r0Moduler2rUr{rrrrrrr+__all__r+r,r-r@sU$ $&!.)/RF&RR7E4/   H % N  2 *BII*Z.Sryy.Sb7 7tIS ISXX8biiX8v(*(V%2%%2K ,K K \M -M `d&FH]c kr,