imetddlmZddlmZddlZddlmZddlmZddlm Z m Z ddl m Z dd l mZmZmZdd lmZdd lmZdd lmZmZdd lmZmZddlmZmZddlmZddl m!Z!m"Z"m#Z#m$Z$ddl%m&Z&m'Z'ddl(m)Z)ddl*m+Z+e$jXe-Z.dZ/edd7dZ0dejbde2dejbfdZ3 d8dejhdejbdejbdejbd ejbdzd!e5d"e5d#ee!fd$Z6ee0Gd%d&ejhZ7ed'Gd(d)ejhZ8Gd*d+ejhZ9Gd,d-eZ:e"Gd.d/eZ;Gd0d1ejhZgd6Z?y)9)Callable)OptionalN)nn)ACT2FN)Cache DynamicCache)GenerationMixin)use_kernel_forward_from_hubuse_kernel_func_from_hubuse_kernelized_func)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)maybe_autocastmerge_with_config_defaults)capture_outputs) GraniteConfigc|dd|jddzf}|d|jddzdf}tj| |fdS)z*Rotates half the hidden dims of the input..Ndim)shapetorchcat)xx1x2s h/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/models/granite/modeling_granite.py rotate_halfr,/sZ 3"!''"+"" " #B 3 q " " #B 99rc2YB ''rotary_pos_embc|j|}|j|}||zt||zz}||zt||zz}||fS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. ) unsqueezer,)qkcossin unsqueeze_dimq_embedk_embeds r+apply_rotary_pos_embr86sY& -- &C -- &C3w;q>C/0G3w;q>C/0G G r- hidden_statesn_repreturnc|j\}}}}|dk(r|S|dddddddddfj|||||}|j|||z||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)r%expandreshape)r9r:batchnum_key_value_headsslenhead_dims r+ repeat_kvrCPso 2?1D1D.E h z!!Qa"23::5BUW\^bdlmM  (;e(CT8 TTr-modulequerykeyvalueattention_maskscalingdropoutkwargsc t||j}t||j} tj||j dd|z} || |z} t j j| dtjj|j} t j j| ||j} tj| | } | j ddj} | | fS)Nr"rr!)r$dtype)ptrainingr)rCnum_key_value_groupsr&matmul transposer functionalsoftmaxfloat32torMrJrO contiguous) rDrErFrGrHrIrJrK key_states value_states attn_weights attn_outputs r+eager_attention_forwardr\\s3 ; ;$>&B\B\$\!22 !'!9!9ii    : :T]] JQWQfQf  ii    : :T]] JQWQfQf  ii    : :T]] JQWQfQf  ii  & & 68J8JQWQfQf  r-r9position_embeddingsrHpast_key_valuescache_positionrKr;c |jdd}g|d|j}|j|j|j dd} |j |j|j dd} |j |j|j dd} |\} } t| | | | \} } |'| | |d}|j| | |j|\} } tj|jjt}||| | | |f|jsdn |j |j"d|\}}|j$g|dj'}|j)|}||fS)Nr!rr")r4r3rw)rJrI)r%rBrnviewrRrorpr8updater`r get_interfacer__attn_implementationr\rOrjrIr>rWrq)rsr9rurHrvrwrK input_shape hidden_shape query_statesrXrYr3r4 cache_kwargsattention_interfacer[rZs r+forwardzGraniteAttention.forwards$))#2. 88b8$--8 {{=166|DNNqRST [[/44\BLLQPQR {{=166|DNNqRST &S#7 jRUWZ#[ j  &#&snUL'6'='=j,X\XfXfht'u $J (?(M(M KK , ,.E) %8      % $}}C$2H2HLL %  % ! \*k));;;;FFH kk+. L((r-N)NNNN)__name__ __module__ __qualname____doc__rintrer&Tensortupler LongTensorrrr __classcell__rts@r+r^r^usG } t 4IM.2(,26 ))||))#5<<#=>E)) t+ ))  )) ((4/ ))+,)) u||U\\) *))r-r^RMSNormcheZdZddeddffd Zdej dej fdZdZxZ S) GraniteRMSNormepsr;Nct|tjt j ||_||_y)z= GraniteRMSNorm is equivalent to T5LayerNorm N)rdrer Parameterr&onesweightvariance_epsilon)rsrgrrts r+rezGraniteRMSNorm.__init__s1 ll5::k#:; #r-r9c"|j}|jtj}|j dj dd}|tj ||jzz}|j|j|zS)Nr"r!T)keepdim) rMrVr&rUpowmeanrsqrtrr)rsr9 input_dtypevariances r+rzGraniteRMSNorm.forwardsy#)) %((7  $$Q',,R,>% Ht?T?T4T(UU {{]--k:::r-c^t|jjd|jS)Nz, eps=)rrr%r)rss r+ extra_reprzGraniteRMSNorm.extra_reprs* ))*+6$2G2G1HIIr-)gư>) rrrfloatrer&rrrrrs@r+rrs7$$$$;U\\;ell;Jr-rc$eZdZfdZdZxZS) GraniteMLPct|||_|j|_|j|_t j |j|j|j|_t j |j|j|j|_ t j |j|j|j|_ t|j|_ y)Nrb)rdrer_rgintermediate_sizerrlmlp_bias gate_projup_proj down_projr hidden_actact_fnrsr_rts r+rezGraniteMLP.__init__s  !--!'!9!94#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_` 4#9#94;K;KRXRaRabV../ r-c|j|j|j||j|z}|Sr)rrrr)rsr(rs r+rzGraniteMLP.forwards6NN4;;t~~a/@#ADLLQRO#ST r-)rrrrerrrs@r+rrs 0r-rcfeZdZdedeffd Z ddejdejdzdejdzde dzd e dzd e dzd ejdzd e ejejfdzd e eje ejejfdzffdZ xZS)GraniteDecoderLayerr_r`cBt||j|_t|||_t ||_t|j|j|_ t|j|j|_ |j|_ y)N)r_r`r) rdrergr^ self_attnrmlpr rms_norm_epsinput_layernormpost_attention_layernormresidual_multiplierrrs r+rezGraniteDecoderLayer.__init__sz !--)9Mf%-f.@.@fFYFYZ(6v7I7IvObOb(c%#)#=#= r-Nr9rH position_idsrvoutput_attentions use_cacherwrur;c |} |j|}|jd||||||||d| \}} | ||jzz}|} |j|}|j |}| ||jzz}|f} |r| | fz } | S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, query_sequence_length, key_sequence_length)` if default attention is used. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_values (`Cache`, *optional*): cached past key and value projection states cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. kwargs (`dict`, *optional*): Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code into the model )r9rHrrvrrrwru)rrrrr) rsr9rHrrvrrrwrurKresidualself_attn_weightsoutputss r+rzGraniteDecoderLayer.forwardsD!,,]; ,:4>> , ')%+/) 3 ,  , ( (!=43K3K#KK !55mD /  =43K3K#KK  "  )+ +Gr-)NNNFFNN)rrrrrrer&rrrboolr FloatTensorrrrs@r+rrs>}>>/304(,).!&26HL?||? t+?&&- ?  ?  $; ?$;?((4/?#5<<#=>E? u  %(9(95;L;L(L"MPT"TT U?r-rcJeZdZUeed<dZdZdgZdgZdZ dZ dZ dZ dZ eedZy)GranitePreTrainedModelr_modelTrrv)r9 attentionsN)rrrr__annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendrr^_can_record_outputsrr-r+rr.sQ &*#./#4"5N!"&,&r-rc eZdZUejed<d deffd Ze d dedzde dde dzde d e ffd Z ejed ZxZS)GraniteRotaryEmbeddinginv_freqNr_ct||j|_|j|_||_|j j d|_|j}|jdk7rt|j}||j |\}|_ |jd|d|jd|jdy)N rope_typedefaultrF) persistentoriginal_inv_freq) rdremax_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr_rope_parametersrcompute_default_rope_parametersrattention_scalingregister_bufferclone)rsr_device rope_init_fnrrts r+rezGraniteRotaryEmbedding.__init__Ds "("@"@$*$B$B! 44[A!%!E!E >>Y &.t~~>L+7 V+L($( ZeD 0(..2BuUr-rz torch.deviceseq_lenr;z torch.Tensorc|jd}t|ddxs|j|jz}d}d|t j d|dtj j|tj|z zz }||fS) a Computes the inverse frequencies according to the original RoPE implementation Args: config ([`~transformers.PreTrainedConfig`]): The model configuration. device (`torch.device`): The device to use for initialization of the inverse frequencies. seq_len (`int`, *optional*): The current sequence length. Unused for this type of RoPE. Returns: Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). rope_thetarBNg?rr"rM)rrM) rrfrgrhr&arangeint64rVr)r_rrbaser$attention_factorrs r+rz6GraniteRotaryEmbedding.compute_default_rope_parametersTs&%%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjj k )))r-cN|jddddfjj|jdddj |j }|dddddfj}t |j jtr/|j jdk7r|j jnd}t|d5|j|jzjdd}tj||fd }|j|jz}|j|jz} dddj |j  j |j fS#1swYAxYw) Nrr!rmpscpuF) device_typeenabledr"r#r)rrr=r%rVr isinstancetypestrrrRr&r'r3rr4rM) rsr(rinv_freq_expandedposition_ids_expandedrfreqsembr3r4s r+rzGraniteRotaryEmbedding.forwardrsR!MM$4-8>>@GG HZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk  U C 5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C  5 vvAGGv$cff177f&;;;  5 5s BFF$r)NNN)rrrr&rrrre staticmethodrrrrrno_gradrrrrs@r+rrAsllV}V '++/"*$*(*t* ~u$ % **:U]]_ < O  !CRC^==?de "\\ "2]5H5H5K"KTaThThN  )33A6L(;;'))+%  & "oom,oW#7BD0d![[)H4;;+H+HI 6M#!m%55!) *) /"3#-$7  M*!,M =#3"55' 6* -0  -!1 1 &+/8Od+%   r-) NNNNNNNNN)rrrrrerrrr&rrrrrrrrrrrs@r+rrs}" .2.204(,26!%)-,026] ##d*]  t+] &&- ]  ] ((4/ ] $;]  $;] #Tk] ((4/] +,]  !]  ] r-rczeZdZddiZddiZddgdgfiZfdZee dd e jdzd e jdzd e jdzd e dzd e jdzde jdzdedzdedzdedzde jdzdee jzdeedefdZxZS)GraniteForCausalLMzlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputr9logitsct||t||_|j|_t j |j|jd|_|jy)NFrb) rdrerrrrrlrgrr rs r+rezGraniteForCausalLM.__init__sU  !&)  ++yy!3!3V5F5FUS  r-Nr rHrrvr labelsrrr rwlogits_to_keeprKr;c ||n|jj}| | n|jj} |jd|||||||| | d | } | j}t | t r t| dn| }|j|dd|ddf}||jjz }d}|*|jd|||jjd| }t||| j| j| jS)a Example: ```python >>> from transformers import AutoTokenizer, GraniteForCausalLM >>> model = GraniteForCausalLM.from_pretrained("meta-granite/Granite-2-7b-hf") >>> tokenizer = AutoTokenizer.from_pretrained("meta-granite/Granite-2-7b-hf") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) r rHrrvr rrr rw)rr r)lossrrvr9rr)r_rr rrrrslicerlogits_scaling loss_functionrrrvr9r)rsr rHrrvr r rrr rwr!rKrr9 slice_indicesrr#s r+rzGraniteForCausalLM.forwards/D2C1N-TXT_T_TqTq$8$D $++JjJj  ,64:: , )%+'/!5) ,  ,  11 8B>SV8W~ot4]k mA}a,?@A$++444  %4%%pVFt{{OeOepiopD%#33!//))   r-) NNNNNNNNNNr)rrr_tied_weights_keys_tp_plan_pp_planrerrr&rrrrrrrrrrrrs@r+rrs`*,GH23H_-z:;H.2.204(,26*.!%)-,026-.C ##d*C  t+C &&- C  C ((4/ C   4'C $;C  $;C #TkC ((4/C ell*C +,C  C C r-r)rrr)r)ry)@collections.abcrtypingrr&r activationsr cache_utilsrr generationr integrationsr r r masking_utilsrmodeling_layersrmodeling_outputsrrmodeling_rope_utilsrrmodeling_utilsrrprocessing_utilsrutilsrrrr utils.genericrrutils.output_capturingrconfiguration_graniter get_loggerrrr,r8rrrCModulerr\r^rrrrrrr__all__rr-r+r>s,% !.)ff/9OKF&RRG50   H %(*+,2 UU\\ U# U%,, U&% II% <<% % << % LL4' %  %%'(%2)*C)ryyC)+C)LY'JRYYJ(J(   J4JZ_$>