iW~dZddlZddlZddlmcmZddlmZddlmZddl m Z ddl m Z ddlmZmZdd lmZdd lmZmZdd lmZdd lmZmZd dlmZej:eZGddej@Z!Gddej@Z"Gddej@Z#Gddej@Z$Gddej@Z%Gddej@Z&Gddej@Z'Gddej@Z(Gdd ej@Z)Gd!d"ej@Z*Gd#d$ej@Z+eGd%d&eZ,eGd'd(e,Z-ed)*Gd+d,e,eZ.gd-Z/y).zPyTorch CPMAntN)nn)CrossEntropyLoss)initialization)ACT2FN)Cache DynamicCache)GenerationMixin)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)auto_docstringlogging) CpmAntConfigcHeZdZdZdeffd ZdejfdZxZ S)CpmAntLayerNormz~ We use Root Mean Square (RMS) Layer Normalization, please see https://huggingface.co/papers/1910.07467 for details." configct||j|_|j|_t j tj|j|_ yN) super__init__eps hidden_sizedim_normr Parametertorchemptyweightselfr __class__s f/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/models/cpmant/modeling_cpmant.pyrzCpmAntLayerNorm.__init__)sE ::** ll5;;v/A/A#BC hidden_statescp|jd|jk7r td|j}|j t j jdjdd}|t j||jzzj ||jz}|S)f Args: hidden_states (`torch.Tensor` of shape `(batch, seq_len, dim_in)`) z'hidden_states.size(-1) != self.dim_normT)dimkeepdim) sizerAssertionErrordtypetorfloat32powmeanrsqrtrr)r!r% old_dtypevariances r#forwardzCpmAntLayerNorm.forward0s   b !T]] 2 !JK K!''  ##EMM266q9>>2t>T&X5H)IIMMiX[_[f[ff r$) __name__ __module__ __qualname____doc__rrrTensorr6 __classcell__r"s@r#rr$s&D|D U\\ r$rceZdZd deffd Z ddej dej dejdej dedzd e dzd edzd ej dzfd Z xZ S)CpmAntAttentionNrcVt||j|_|j|_|j |_||_tj|j|j |j zd|_ tj|j|j |j zd|_ tj|j|j |j zd|_ tj|j |j z|jd|_ tjjd|_|j"0tjj%|j"|_yd|_y)NFbiasr(r*)p)rrr dim_modelnum_attention_heads num_headsdim_head layer_idxrLinear project_q project_k project_v attention_outrSoftmaxsoftmax dropout_pDropoutdropoutr!rrIr"s r#rzCpmAntAttention.__init__>s ++33 "4>>4>>DMM3QX]^4>>4>>DMM3QX]^4>>4>>DMM3QX]^YYt~~ 'Et~~\abxx''B'/    ' 88++f.>.>+?DLDLr$hidden_q hidden_kvattention_mask position_biasoutput_attentionspast_key_values use_cachecache_positionc |jd} |jd} |jd} |j|} |j|} |j|}| j | | |j |j jdddd} | j | | |j |j jdddd} |j | | |j |j jdddd}|4|j| ||jd|i\} }| jd} tj| | jddtj|j z }||z}tj||j | d| | tj d k(tj"t%d |j&|j( }|j+|}tj||j | d| | tj d k(tj"d|j&|j( }|r|}nd}|j,|j-|}tj||}|j | |j | |j jdddd}|j/j | | |j |j z}|j1|}||fS) ad Args: hidden_q (`torch.Tensor`): Input of transformer block(self-attention block). It can be the raw embedding of a batch of sequences. hidden_kv (`torch.Tensor` of shape `(batch, len_k, dim_model)`)): Tensor *key_value* and *query* of shape `(batch, len_k, dim_model)` attention_mask (`torch.Tensor` of shape `(batch, len_seq, len_seq)`): Avoid invalid areas to participate in the calculation of self-attention. position_bias (`torch.Tensor` of shape `(batch, len_seq, len_seq)`): Provide positional information to self-attention block. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. past_key_values (`Cache`, *optional*): Cached past key and value projection states. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). rrr)rNr\r(Fz-inf)devicer.)r,rKrLrMviewrGrHpermuteupdaterIrmatmul transposemathsqrt masked_filltensor scalar_tensorfloatr_r.rPrS contiguousrN)r!rUrVrWrXrYrZr[r\ batch_sizelen_qlen_kquerykeyvaluescore attn_weightss r#r6zCpmAntAttention.forwardRs:]]1%  a q!x(nnY'y) :udnndmmLTTUVXY[\^_`hhz5$..$--HPPQRTUWXZ[\ :udnndmmLTTUVXY[\^_`  &(//UDNNM]_mLnoJCHHRLE UCMM"b$9:TYYt}}=UU %!!     Aue < U@S S   f ell%++ V   U#!!     Aue < U@S S   %,,ekk J   LL << #LL'E UE* :t~~udmmLTTUVXY[\^_`  "'' E4>>DMM;YZ""5)l""r$r)FNNN) r7r8r9rrrr; BoolTensorboolrr6r<r=s@r#r?r?=s | 4*/(,!%.2M#,,M#<<M#(( M# || M#  $; M#M#$;M# t+M#r$r?ceZdZd deffd Z d dej dej dej dzdedzdedzd edzd ej dzfd Z xZ S)CpmAntSelfAttentionBlockNrct|t||_t |||_|j r/tjj|j |_ yd|_ yN)rI) rrrlayernorm_before_attentionr?self_attentionrQrrrRrSrTs r#rz!CpmAntSelfAttentionBlock.__init__sV *9&*A'-f J    88++F,<,<=DLDLr$r%rWrXrYrZr[r\c |j|}|j||||||||\}} |j|j|}||z}|| fS)a Args: hidden_states (`torch.Tensor` of shape `(batch, len_seq, dim_model)`): Input of transformer block(self-attention block). It can be the raw embedding of a batch of sequences. attention_mask (`torch.Tensor` of shape `(batch, len_seq, len_seq)`): Avoid invalid areas to participate in the calculation of self-attention. position_bias (`torch.Tensor` of shape `(batch, len_seq, len_seq)`): Provide positional information to self-attention block. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. past_key_values (`Cache`, *optional*): Cached past key and value projection states. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). )rzr{rS) r!r%rWrXrYrZr[r\outputsrss r#r6z CpmAntSelfAttentionBlock.forwardsp411-@ $ 3 3         !  << #ll7+G%/ l**r$rNFNNN r7r8r9rrrr;rurr6r<r=s@r#rwrws | .2).(,!%.2*+||*+ *+||d* *+  $; *+  *+$;*+ t+*+r$rwcDeZdZdeffd Zdej fdZxZS)CpmAntDenseGatedACTrc,t|tj|j|j d|_tj|j|j d|_tjj|_ yNFrA) rrrrJrdim_ffw_0w_1rGELUactr s r#rzCpmAntDenseGatedACT.__init__s[ 99V//UK99V//UK88==?r$r%cr|j|j|}|j|}||z}|S)zTransform an input tensor from one feature space to another via a nonlinear operation Args: hidden_states (`torch.Tensor` of shape `(batch, seq_len, dim_in)`) )rrr)r!r% gate_scores r#r6zCpmAntDenseGatedACT.forwards9 XXdhh}56 / "]2 r$ r7r8r9rrrr;r6r<r=s@r#rrs#|# U\\ r$rcDeZdZdeffd Zdej fdZxZS)CpmAntFeedForwardrc(t|t||_|j/t j j|j|_nd|_t j|j|jd|_ yr) rrrw_inrQrrrRrSrJrrw_outr s r#rzCpmAntFeedForward.__init__sg '/    ' 88++F,<,<=DLDLYYv}}f.@.@uM r$r%c|j|}|j|j|}|j|}|S)r')rrSrr!r%s r#r6zCpmAntFeedForward.forwards>  -0 << # LL7M =1 r$rr=s@r#rrs!N|N U\\ r$rcDeZdZdeffd Zdej fdZxZS)CpmAntFFNBlockrct|t||_t ||_|j r/tjj|j |_ yd|_ yr) rrrlayernorm_before_ffnrffnrQrrrRrSr s r#rzCpmAntFFNBlock.__init__sS $3F$;!$V,    88++F,<,<=DLDLr$r%c|j|}|j|}|j|j|}||z}|S)z Args: hidden_states (`torch.Tensor` of shape `(batch, len_seq, dim_model)`): Hidden states before feed forward layer. )rrrS)r!r% ln_outputsr}s r#r6zCpmAntFFNBlock.forwardsJ..}= ((:& << #ll7+G%/ r$rr=s@r#rrs  | ||r$rceZdZd deffd Z d dej dej dej dzdedzdedzd edzd ej dzfd Z xZ S)CpmAntTransformerBlockNrcft|t|||_t ||_yry)rrrwself_attrrrTs r#rzCpmAntTransformerBlock.__init__#s( 09M !&)r$r%rWrXrYrZr[r\c b|j|||||||\}}|j|}||fS)a Args: hidden_states (`torch.Tensor`): Input to the layer of shape `(batch, seq_len, dim_model)` attention_mask (`torch.Tensor`): Avoid invalid areas to participate in the calculation of shape `(batch, seq_len, seq_len)` position_bias (`torch.Tensor`): Provides position information to attention mechanism of shape `(num_heads, seq_len, seq_len)` output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. past_key_values (`Cache`, *optional*): Cached past key and value projection states use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). )rWrXrYrZr[r\)rr) r!r%rWrXrYrZr[r\rss r#r6zCpmAntTransformerBlock.forward(sL4'+mm )'/+)'4' # |/ l**r$rr~rr=s@r#rr"s*|*.2).(,!%.2%+||%+ %+||d* %+  $; %+  %+$;%+ t+%+r$rceZdZdeffd Z d dej dej dej dedzdedzd edzd edzd ej dzfd Z xZ S) CpmAntEncoderrc t||j|_t j t |jDcgc]}t||c}|_t||_ ycc}wry) rrnum_hidden_layers num_layersr ModuleListrangerlayersroutput_layernorm)r!rir"s r#rzCpmAntEncoder.__init__Qs_  22mmZ_`d`o`oZp$qUV%;Fa%P$qr / 7%rsA8Nr%rWrXrYoutput_hidden_statesrZr[r\c |rdnd} |rdnd} t|jD])\} } |r| |fz } | ||||||} | \}}|s$| |fz } +|j|}|r| |fz } || | fS)a Args: hidden_states (`torch.Tensor`): Input to the layer of shape `(batch, seq_len, dim_model)` attention_mask (`torch.Tensor`): Avoid invalid areas to participate in the calculation of shape `(batch, seq_len, seq_len)` position_bias (`torch.Tensor`): Provides position information to attention mechanism of shape `(num_heads, seq_len, seq_len)` output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. past_key_values (`Cache`, *optional*): Cached past key and value projection states use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). N)rYrZr[) enumeraterr)r!r%rWrXrYrrZr[r\all_hidden_statesall_self_attnsrlayer layer_outputsrss r#r6zCpmAntEncoder.forwardXs:#7BD0d!$++. 2HAu#!m%55!!"3 /# M+8 'M< ^>^^** ( $r$key_pos query_pos key_segment query_segmentc 0tj5|jd}|jd}|jd}|jd|jdk7r0td|jdd|jdd||jdk7s||jdk7r!td|d|jdd||jdk7r!td|d|jdd|j |d|}|j ||d}|j |d|}|j ||d}|j ||}||j z}|jtj|tj|j dddftj|tj|j dddfz |j |j } tj||k(| dddddf|}dddtj|j} | j!dd dd j#} | S#1swYMxYw) Nrrz>key_pos.size(0) should be equal to query_pos.size(0), but got z and !z7keylen should be equal to key_segment.size(1), but got z;querylen should be equal to query_segment.size(1), but got r(r.r_)rrrr))rno_gradr,r-r`!_segment_relative_position_bucketr_position_bucketarangeint32r_rwhereF embeddingrrark) r!rrrrbatchkeylenquerylenrelative_position_bucketabsolute_position_bucketembedss r#r6z&CpmAntSegmentPositionEmbedding.forwards]]_% LLOE\\!_F ~~a(H||A).."33$TU\UaUabcUdTeejktkykyz{k|j}}~))!,,M>r$c.d}|dz}|dkDjtj|z}tj|}|dz}||k}|tj|j |z t j||z z ||z zjtjz}tj|tj||dz }|tj||jtj|z }|S)Nrr)r) r/rrabslogrjremin full_liker)r!relative_positionrrrelative_buckets max_exactis_smallrelative_position_if_larges r#rz/CpmAntSegmentPositionEmbedding._position_buckets -155ekkB[P!II&781$ $y0%. II'--/); <hh|i/0 1Y& ( "U[[/ &" &+YY & OO6 a H& " EKK2C2F2Fu{{2SUoppr$) ) r7r8r9rrrr;r6rrr<r=s@r#rrsU  |  22<<2\\ 2 || 2h? r$rcneZdZfdZdej dej dej fdZxZS) CpmAntOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ y)N)r) rrrrJrrr LayerNormlayer_norm_epsrRhidden_dropout_probrSr s r#rzCpmAntOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= r$r% input_tensorrcr|j|}|j|}|j||z}|Sr)rrSr)r!r%rs r#r6zCpmAntOutput.forwards7 =1  ]3 }|'CD r$rr=s@r#rrs1> U\\RWR^R^r$rcVeZdZUeed<dZejfdZxZ S)CpmAntPreTrainedModelrcpmantct||t|tr t j |j yt|tr7t j|jd|jjyy)zInitialize the weightsg)r2stdN) r _init_weightsrrinitones_rrnormal_rrinit_std)r!moduler"s r#rz#CpmAntPreTrainedModel._init_weightss] f% fo . JJv}} %  > ? LL77ct{{G[G[ \@r$) r7r8r9r__annotations__base_model_prefixrrrr<r=s@r#rr s+  U]]_]]r$rceZdZdeffd ZdZdZdZe dde jdzde dzd e dzd e dzd e dzd e dzd e jdzde e jezfdZxZS) CpmAntModelrct||t||_t j |j |j|_t j |j|j|jzz|j|_ t||_|j|_ |j|_ |jyr)rrrencoderr Embeddingrrsegment_embedding vocab_size prompt_types prompt_lengthinput_embeddingrrX post_initr s r#rzCpmAntModel.__init__s  $V, !#f.B.BFDVDV!W!||    3 3f6J6J J JFL^L^ z&CpmAntModel.forward..sbcbos)last_hidden_staterZr% attentions)rrYruse_return_dictr[r.rrr/r_rsumrrrrrr,zerosfullr get_seq_lengthrkrrrrXrtupler )r!rrYrrZr[rr\rr.r_segmentrr seq_lengthrpositionr past_lengthr%segment_statesrWrXrall_attentionsnew_attentions attentionnew_hidden_states hidden_states r#r6zCpmAntModel.forward@s*2C1N-TXT_T_TqTq$8$D $++JjJj &1%O:I:Uo446[\ ((* ,,Y7 //8 ! +ArsAI6N%6 55iwPVW**8XwP ';<(:;%aKL!&;< %aq&89 ;?<<        < 8 (. ! )!T-?-?-A1*DEM)!#!/eI"yAt7I7I7KTM_M_Ma1a'b&ddNe!/ ,$&!$5UL%,q$:L:L:NPQ7Q*R)TT%U$5!)?>> from transformers import CPMAntTokenizer, CpmAntForCausalLM >>> texts = "今天天气不错，" >>> model = CpmAntForCausalLM.from_pretrained("openbmb/cpm-ant-10b") >>> tokenizer = CPMAntTokenizer.from_pretrained("openbmb/cpm-ant-10b") >>> input_ids = tokenizer(texts, return_tensors="pt") >>> outputs = model.generate(**input_ids) >>> output_texts = tokenizer.batch_decode(outputs) >>> print(output_texts) ['今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的'] ``` Nrr(r)losslogitsrZr%r&)rr'rr%rintslicer;rr`r,r rZr%r&)r!rrZr[rYrr<rrWr\r=r model_outputr% slice_indicesr@r? loss_funcoutputs r#r6zCpmAntForCausalLM.forwardsT&1%SV8W~ot4]k mA}a,?@A  (*IV[[V[[_=v{{2ODYab!11F)-)9TGf$ Ev E%(88&44#..   r$c.|jjSrrrr s r#r z&CpmAntForCausalLM.get_input_embeddings s{{***r$c&||j_yrrH)r!rs r#rz&CpmAntForCausalLM.set_input_embeddings s&0 #r$) NNNNNNNNNr)r7r8r9_tied_weights_keysrrrrr;rrurAr,r r6r rr<r=s@r#r9r9s +,KL|*.(,!%)-,0&*#'.2.2-.H <<$&H H $; H  $; H #Tk H  t#H D[H  t+H  t+H ell*H  ' 'H H T+1r$r9)r9rr)0r:rertorch.nn.functionalr functionalrtorch.nnrrr activationsr cache_utilsrr generationr modeling_outputsr r modeling_utilsr utilsrrconfiguration_cpmantr get_loggerr7loggerModulerr?rwrrrrrrrrrrr9__all__rr$r#rZsu %&!.)O-,.   H %bii2b#biib#J4+ryy4+n"))( 4RYY6++RYY++\<@BII<@@  Y RYYY z 299  ]O ] ]P 'P P f \1-\1  \1~ Hr$