謜i" UddlZddlmZddlZddlmcmZddlm Z ddl m Z ddl m Z mZddlmZddlmZmZdd lmZmZmZerdd lmZdd lmZmZn ej:Zed d Zedd Ze Z erddl!m"Z"ejFe$Z%dedefdZ&dedefdZ'de(de(de(de(de)f dZ*de(de(de(de(de)f dZ+de(defdZ,de(dej:defdZ-de(defd Z.de(defd!Z/de(defd"Z0de(dej:defd#Z1d$ej:defd%Z2d&ej:defd'Z3d(ed)e(d*e(defd+Z4d,ej:dzd-e(d*e(dej:dzfd.Z5d$ej:dzd/e(d-e(d0e(dzde)f d1Z6 dld$ej:dzd/e(d-e(d*e(d0e(dzde)f d2Z7d$ej:dzd-e(d0e(dzde)fd3Z8 dld$ej:dzd-e(d0e(dzde)fd4Z9d(edefd5Z:d6ej:d7ej:d8ej:d9ej:fd:Z;de*ddd d;d d;fde(dzd?e)d@e)dAe)dBe)dej:dzfdCZzand_masks..07x}7.All inputs should be callable mask_functions: c|jdtj}D])}||||||j|jz}+|SN)dtype)new_onestorchbooltodevice batch_idxhead_idxq_idxkv_idxresultmaskrs rand_maskzand_masks..and_mask3sS%**5" YDd9hvFII&--XXF Y all RuntimeError)rr3s` r and_masksr8.s1 77 7KNK[\]] Or4cRtdDstdfd}|S)zDReturns a mask function that is the union of provided mask functionsc32K|]}t|ywrrrs rrzor_masks..>r r!r"c|jdtj}D])}||||||j|jz}+|Sr$) new_zerosr(r)r*r+r,s ror_maskzor_masks..or_maskAsS5::6" YDd9hvFII&--XXF Y r4r5)rr=s` ror_masksr><s1 77 7KNK[\]] Nr4r-r.r/r0c ||kS)z: This creates a basic lower-diagonal causal mask. r%r-r.r/r0s rcausal_mask_functionrAJs U?r4c |dk\S)z This creates a full bidirectional mask. NOTE: It is important to keep an index-based version for non-vmap expansion. rr%r@s rbidirectional_mask_functionrCQs A:r4sliding_windowc Pdtdtdtdtdtf fd }|S)z This is an overlay depicting a sliding window pattern. Add it on top of a causal mask for a proper sliding window mask. r-r.r/r0rc||z kDSrr%r-r.r/r0rDs r inner_maskz*sliding_window_overlay..inner_mask`s...r4intr)rDrHs` rsliding_window_overlayrLZs3 /c/S//c/d/ r4 chunk_size left_paddingc Tdtdtdtdtdtf fd }|S)z This is an overlay depicting a chunked attention pattern. Add it on top of a causal mask for a proper chunked attention mask. r-r.r/r0rc2||z z||z zk(Srr%)r-r.r/r0rMrNs rrHz#chunked_overlay..inner_maskls.i00Z?ELYbLcDchrCrrrr4rI)rMrNrHs`` rchunked_overlayrQfs9 scsSsscsds r4c4tt|tS)zQ This return the mask_function function to create a sliding window mask. )r8rLrArDs r#sliding_window_causal_mask_functionrTrs +N;=Q RRr4c Pdtdtdtdtdtf fd }|S)zN This is an overlay depicting a bidirectional sliding window pattern. r-r.r/r0rc&t||z kS)zA token can attend to any other token if their absolute distance is within the (inclusive) sliding window size (distance <= sliding_window).)absrGs rrHz8sliding_window_bidirectional_overlay..inner_mask~s56>"n44r4rIrKs` r$sliding_window_bidirectional_overlayrXys3 5c5S55c5d5 r4c4tt|tS)z_ This return the mask_function function to create a bidirectional sliding window mask. )r8rXrCrSs r*sliding_window_bidirectional_mask_functionrZs 9.IKf ggr4c6tt||tS)zT This return the mask_function function to create a chunked attention mask. )r8rQrA)rMrNs rchunked_causal_mask_functionr\s _Z>@T UUr4 padding_maskc Pdtdtdtdtdtf fd }|S)zT This return the mask_function function corresponding to a 2D padding mask. r-r.r/r0rc||fSrr%)r-r.r/r0r]s rrHz)padding_mask_function..inner_masksIv-..r4rI)r]rHs` rpadding_mask_functionr`s3 /c/S//c/d/ r4packed_sequence_maskc Pdtdtdtdtdtf fd }|S)z\ This return the mask_function function corresponding to a 2D packed sequence mask. r-r.r/r0rc"||f||fk(Srr%)r-r.r/r0ras rrHz1packed_sequence_mask_function..inner_masks$#Iu$459MiY_N_9```r4rI)rarHs` rpacked_sequence_mask_functionrds9 acaSaacada r4 mask_functionq_offset kv_offsetc Xdtdtdtdtdtf fd }|S)z This function adds the correct offsets to the `q_idx` and `kv_idx` as the torch API can only accept lengths, not start and end indices. r-r.r/r0rc&|||z|zSrr%)r-r.r/r0rgrerfs rrHz0add_offsets_to_mask_function..inner_masksY%(2BFYDVWWr4rI)rerfrgrHs``` radd_offsets_to_mask_functionrjs9 XcXSXXcXdX r4attention_mask kv_lengthc|}|F||z|jdz x}dkDr,tjjj |d|f}|S)zh From the 2D attention mask, prepare the correct padding mask to use by potentially padding it. r)shaper(nn functionalpad)rkrlrglocal_padding_maskpadding_lengths rprepare_padding_maskrus[(!')3n6J6J26NN NNRS S!&!4!4!8!8!^I\!]  r4 query_lengthlocal_attention_sizect|ry|||k\ry| |dk(xs||k(S|dk(r|jS|ddd|fjxr|dd|dfj S)ar XPU-specific logic for determining if we can skip causal mask creation. For XPU devices, we have special handling: - Single query tokens (query_length == 1) use the same logic as CUDA - Multi-query tokens can skip if padding_mask is provided and correctly structured The mask must have all True values in the query window and all False after FNr)rr6any)r]rvrlrws r_can_skip_causal_mask_xpurzs,'I9M,Mq =I$==q!! =L=( ) - - / \ Q EU8V8Z8Z8\4\\r4c|A|jd|kDr/tj||j}||z }|dd|f}trt ||||St |s$|dk(s||k(r|||kr||jryy)a Detects whether the causal mask can be ignored in case PyTorch's SDPA is used, rather relying on SDPA's `is_causal` argument. In case no token is masked in the 2D `padding_mask` argument, if `query_length == 1` or `key_value_length == query_length`, we rather rely on SDPA `is_causal` argument to use causal/non-causal masks, allowing to dispatch to the flash attention kernel (that can otherwise not be used if a custom `attn_mask` is passed). Nrnr+rTF)ror(aranger+_is_torch_xpu_availablerzrr6)r]rvrlrgrw mask_indicess r_ignore_causal_mask_sdparsL$6$6r$:Y$F||Il6I6IJ  ! #A|O4  )|YPdee | $ Q )|"; ! )Y9M-M  !\%5%5%7 r4cPt|ry|||k\ry|y|jS)z XPU-specific logic for determining if we can skip bidirectional mask creation. For XPU devices, we have special handling: - Skip if no padding and no local attention constraint FT)rr6r]rlrws r _can_skip_bidirectional_mask_xpurs<,'I9M,M    r4cttr t|||St|s||jr|||kryy)a Detects whether the bidirectional mask can be ignored in case PyTorch's SDPA is used. In case no token is masked in the 2D `padding_mask` argument and no local attention constraint applies (i.e. `local_attention_size` is None or `kv_length < local_attention_size`), we skip mask creation, allowing to dispatch to the flash attention kernel (that can otherwise not be used if a custom `attn_mask` is passed). TF)r~rrr6rs r_ignore_bidirectional_mask_sdpar0sH0 iI]^^ | $  !\%5%5%7 ! )Y9M-M r4cLgd}|D]}tj||d}|S)aJ Used to vmap our mask_functions over the all 4 dimensions (b_idx, h_idx, q_idx, kv_idx) of the inputs. Using vmap here allows us to keep the performance of vectorized ops, while having a single set of primitive functions between attention interfaces (i.e. between flex and sdpa/eager, FA2 being a bit different). ))NNNr)NNrN)NrNN)rNNNr)in_dimsout_dims)r(vmap)re dimensionsdimss r_vmap_expansion_sdparOs3nJL =$K L r4 batch_indices head_indices q_indices kv_indicescf|dddddf}|dddddf}|dddddf}|dddddf}||||fS)a Used to broadcast our mask_functions over the all 4 dimensions (b_idx, h_idx, q_idx, kv_idx) of the inputs. Allows the usage of any index-based mask function without relying on vmap. NOTE: This is limited to index based functions only and is not guaranteed to work otherwise. Reference: - https://github.com/huggingface/optimum-onnx/blob/c123e8f4fab61b54a8e0e31ce74462bcacca576e/optimum/exporters/onnx/model_patcher.py#L362-L365 Nr%)rrrrs r_non_vmap_expansion_sdpar\s_"!T4"56Mat 34L$a-.ID$a/0J , : ==r4F batch_sizecache_position local_sizeallow_is_causal_skipallow_is_bidirectional_skipallow_torch_fixuse_vmapc  |jd} t|||} |rt| | |||ry|rt| ||ry| t |t | }t j||j}t jd|j}t j||j|z}| s&|t||||}|j|d| |}n9tr(t5t|||||}dddn tdts| r|t j |ddz}|S#1swY/xYw) u Create a 4D boolean mask of shape `(batch_size, 1, query_length, kv_length)` where a value of True indicates that the element should take part in the attention computation, and False that it should not. This function can only be used with torch>=2.5, as the context manager is otherwise not available. Args: batch_size (`int`): The batch size of the input sequence. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. kv_length (`int`): The size that the key and value states will have during the attention computation. kv_offset (`int`, optional): An optional offset to indicate at which first position the key and values states will refer to. mask_function (`Callable`): The mask factory function describing the mask pattern. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length) local_size (`int`, optional): The size of the local attention, if we do not use full attention. This is used only if `allow_is_causal_skip=True` to try to skip mask creation if possible. allow_is_causal_skip (`bool`, optional): Whether to allow to return `None` for the mask under conditions where we can use the `is_causal` argument in `torch.sdpa` instead. Default to `True`. allow_is_bidirectional_skip (`bool`, optional): Whether to allow to return `None` for the mask under conditions where we do not have to add any bias, i.e. full attention without any padding. Default to `False`. allow_torch_fix (`bool`, optional): Whether to update the mask in case a query is not attending to any tokens, to solve a bug in torch's older versions. We need an arg to skip it when using eager. By default `True`. use_vmap (`bool`, optional): Whether to use `vmap` during the mask construction or not. Allows powerful custom patterns that may not be index-based (for the cost of speed performance). By default `False`. ## Creating a simple causal mask: To create the following causal mask: 0 ■ ⬚ ⬚ ⬚ ⬚ 1 ■ ■ ⬚ ⬚ ⬚ 2 ■ ■ ■ ⬚ ⬚ 3 ■ ■ ■ ■ ⬚ 4 ■ ■ ■ ■ ■ You can do ```python >>> sdpa_mask(batch_size=1, cache_position=torch.arange(5), kv_length=5) >>> tensor([[[[ True, False, False, False, False], [ True, True, False, False, False], [ True, True, True, False, False], [ True, True, True, True, False], [ True, True, True, True, True]]]]) ``` ## Creating a sliding window mask: To create the following sliding window mask (`sliding_window=3`): 0 ■ ⬚ ⬚ ⬚ ⬚ 1 ■ ■ ⬚ ⬚ ⬚ 2 ■ ■ ■ ⬚ ⬚ 3 ⬚ ■ ■ ■ ⬚ 4 ⬚ ⬚ ■ ■ ■ You can do ```python >>> sdpa_mask(batch_size=1, cache_position=torch.arange(5), kv_length=5, mask_function=sliding_window_causal_mask_function(3)) >>> tensor([[[[ True, False, False, False, False], [ True, True, False, False, False], [ True, True, True, False, False], [False, True, True, True, False], [False, False, True, True, True]]]]) ``` ## Creating a chunked attention mask To create the following chunked attention mask (`chunk_size=3`): 0 ■ ⬚ ⬚ ⬚ ⬚ 1 ■ ■ ⬚ ⬚ ⬚ 2 ■ ■ ■ ⬚ ⬚ 3 ⬚ ⬚ ⬚ ■ ⬚ 4 ⬚ ⬚ ⬚ ■ ■ You can do ```python >>> sdpa_mask(batch_size=1, cache_position=torch.arange(5), kv_length=5, mask_function=chunked_causal_mask_function(3, torch.zeros(1, dtype=int))) >>> tensor([[[[ True, False, False, False, False], [ True, True, False, False, False], [ True, True, True, False, False], [False, False, False, True, False], [False, False, False, True, True]]]]) ``` rNr|rrnzThe vmap functionality for mask creation is only supported from torch>=2.6. Please update your torch version or use `use_vmap=False` with index-based masks.T)dimkeepdim)rorurrr8r`r(r}r+rexpand#_is_torch_greater_or_equal_than_2_6rr ValueError#_is_torch_greater_or_equal_than_2_5r6)rrrlrgrerkrrrrrkwargsq_lengthr] batch_arange head_arange kv_aranges r sdpa_maskroslb##A&H( 9ML  8xQZ\egq r"'F|U^`j'k!-1F|1TU << >3H3HIL,,q)>)>?K Y~/D/DE QI &(@{\jlu(vw'..z2xS -% & w@1-@{\jluvN w w  _   /?'%))^OUY*ZZ  w ws 4EE r&c  *| jdd} | jdd} td||||||d|d|d | } | Vtj|j} tj | tj d| j|| } | S) aX Create a 4D float mask of shape `(batch_size, 1, query_length, kv_length)` where a value of 0 indicates that the element should take part in the attention computation, and -inf (minimum value for the given `dtype`) that it should not. Args: batch_size (`int`): The batch size of the input sequence. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. kv_length (`int`): The size that the key and value states will have during the attention computation. kv_offset (`int`, optional): An optional offset to indicate at which first position the key and values states will refer to. mask_function (`Callable`): The mask factory function describing the mask pattern. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length) dtype (`torch.dtype`, optional): The dtype to use for the mask. By default, `torch.float32`. allow_is_bidirectional_skip (`bool`, optional): Whether to allow to return `None` for the mask under conditions where we do not have to add any bias, i.e. full attention without any padding. Default to `False`. use_vmap (`bool`, optional): Whether to use `vmap` during the mask construction or not. Allows powerful custom patterns that may not be index-based (for the cost of speed performance). By default `False`. rNrF) rrrlrgrerkrrrrgr+r&r%)poprr(finfominwheretensorr+) rrrlrgrerkr&rrr_r2 min_dtypes r eager_maskrsP  )40A $d+A  %#%"$?   D KK&** {{4c$++U!SU^_ Kr4c F||dd| df}|jrd}|S)a" Create the attention mask necessary to use FA2. Since FA2 is un-padded by definition, here we simply return `None` if the mask is fully causal, or we return the 2D mask which will then be used to extract the seq_lens. We just slice it in case of sliding window. Args: batch_size (`int`): The batch size of the input sequence. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. kv_length (`int`): The size that the key and value states will have during the attention computation. kv_offset (`int`, optional): An optional offset to indicate at which first position the key and values states will refer to. mask_function (`Callable`): The mask factory function describing the mask pattern. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length) N)r6)rrrlrgrerkrs rflash_attention_maskrUs58!'I:;7    !N r4c |jd|d}}||jdtzdztz} | |jdz } ts3| dkDr.tjj j |dd| f}t|||} t|t| }t|||}t||d|||jt} | S)a Create a 4D block mask which is a compressed representation of the full 4D block causal mask. BlockMask is essential for performant computation of flex attention. See: https://pytorch.org/blog/flexattention/ Args: batch_size (`int`): The batch size of the input sequence. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. kv_length (`int`): The size that the key and value states will have during the attention computation. kv_offset (`int`, optional): An optional offset to indicate at which first position the key and values states will refer to. mask_function (`Callable`): The mask factory function describing the mask pattern. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length) rNr)valuerr)mask_modBHQ_LENKV_LENr+_compile) roflex_default_block_sizerr(rprqrrrur8r`rjrr+) rrrlrgrerkrrrfpad_lenr] block_masks rflex_attention_maskr|s6(--a0.2ChH!#((+/FF!KOffN00332w{"XX0044^1STV]R^4_N+NIyQ !-1F|1TU 1)TM#  $$4J r4ceZdZeeeeedZy)AttentionMaskInterface)sdpaeagerflash_attention_2flash_attention_3flex_attentionN)__name__ __module__ __qualname__rrrr_global_mappingr%r4rrrs11- Or4rALL_MASK_ATTENTION_FUNCTIONS position_idsc|ddddfdz }tj||d}|dk7jd}t|s|dddfdk(j ry|S)a> Find the indices of the sequence to which each new query token in the sequence belongs when using packed tensor format (i.e. several sequences packed in the same batch dimension). Args: position_ids (`torch.Tensor`) A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences. Returns: A 2D tensor where each similar integer indicates that the tokens belong to the same sequence. For example, if we pack 3 sequences of 2, 3 and 1 tokens respectively along a single batch dim, this will return [[0, 0, 1, 1, 1, 2]]. If the there is only one sequence in each batch item (and we don't compile), then we return `None` indicating no packed sequences. This is the same as [[0, 0, 0, 0, 0, 0]] for the example above. Nrrn)prependrr)r(diffcumsumrr6)rfirst_dummy_value position_diffras rfind_packed_sequence_indicesrst*%QU+a/JJ|5FBOM)Q.66r: * +1Eae1LPQ1Q0V0V0X r4 input_embedsz5.6.0 inputs_embeds)versionnew_nameconfigpast_key_values layer_idxcFt|tjtfrt |j dk(rd|dddfS|j tjvry|:|jdk(r+|j|jtj}||j||\}}n%||j dd}}n|j d d}}d} |B|@|>|j d} | |j dk7r|j| d }t|} d || ||fS) as Perform some common pre-processing of the mask arguments we get from the modeling code. Mostly determine the key-value length and offsets, and if we should early exit or not. Args: config (`PreTrainedConfig`): The model config. inputs_embeds (`torch.Tensor`): The input embeddings of shape (batch_size, query_length, hidden_dim). This is used only to infer the batch size, query length and dtype. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences. layer_idx (`int`, optional): If `past_key_values` is not None, this is the layer index of the cache from which to get the key-value length and offset. Indeed, for hybrid caches, different layers may return different lengths. Returns: early_exit (`bool`): Whether we should early exit mask creation, and return the mask as-is. attention_mask (`torch.Tensor` or `BlockMask` or `None`): The attention mask to either return immediately, or to use in downstream mask creation. packed_sequence_mask (`torch.Tensor`, optional): In case we detected packed sequence format, this is a tensor where each similar integer indicates that the tokens belong to the same sequence. kv_length (`int`): The size that the key and value states will have during the attention computation. kv_offset (`int`): An offset to indicate at which first position the key and values states will refer to. TN)TNNNNrrrrnF) isinstancer(Tensorrlenro_attn_implementationrrndimr*r+r)get_mask_sizesrr) rrrkrrrrrlrgrars r_preprocess_mask_argumentsrsE^.5<<";<^EYEYAZ^_A_^T455""*F*V*VV+!n&9&9Q&>'**.2G2Guzz*Z".==niX 9  !#0#6#6q#91yI$2#7#7#;QyI N$:?V"((+ ++A. .'..z2>L;LI ."6 9 LLr4or_mask_functionand_mask_functionc t|ddst|||||St|dr*d|jvr|jj d}nd}t |||||||\} }} } } | r|S|j d|j}} t}t|j}d}tr#t|ddxr|j ddk( }nt|dd }|!ts td t||}d}d}|!ts td t||}d}d}| t|t!| }d}|| || | |||||| }|S) a. Create a standard causal mask based on the attention implementation used (stored in the config). If `past_key_values` has an hybrid cache structure, this function will return the mask corresponding to one of the "full_attention" layers (to align to what is needed in the `modeling_xxx.py` files). Args: config (`PreTrainedConfig`): The model config. inputs_embeds (`torch.Tensor`): The input embeddings of shape (batch_size, query_length, hidden_dim). This is used only to infer the batch size, query length and dtype. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences. or_mask_function (`Callable`, optional): An optional mask function to combine with the causal mask function (by doing the union of both). This is useful to easily overlay another mask on top of the causal one, for example for image tokens handling. and_mask_function (`Callable`, optional): An optional mask function to combine with the causal mask function (by doing the intersection of both). This is useful to easily overlay another mask on top of the causal one, for example for image tokens handling. is_causalTrr is_slidingFris_compileablerLUsing `or_mask_function` or `and_mask_function` arguments require torch>=2.6) rrrlrgrerkrr&rr)getattrcreate_bidirectional_maskhasattrrindexrror&rArrr~rrr>r8rd)rrrkrrrrrr early_exitrarlrgrr&mask_factory_functionmask_interfacerr causal_masks rcreate_causal_maskrAsP 6; -(   -/    -%?;U;U2U#..44U;  Mg ~~P\^gNJJ 4i%++A. 0C0CJ01&2M2MNN H$+O=Mu$U$vZhZnZnopZquvZvw#*?@P Q$$2kl l )*?AR S$' )*?A^_sAt u$!%+%1 K r4encoder_hidden_statesc tj|jd|jtj}||n|}t ||||ddd\}}} } } |r|S|jd|j } } t}t|j}d}d}|!ts tdt||}d}d}|!ts tdt||}d}d}|| || | ||d|| || }|S) a Create a standard bidirectional mask based on the attention implementation used (stored in the config). Args: config (`PreTrainedConfig`): The model config. inputs_embeds (`torch.Tensor`): The input embeddings of shape (batch_size, query_length, hidden_dim). This is only used to infer metadata such as the batch size, query length, dtype, and device. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, kv_length). It can also be an already prepared 4D mask of shape (batch_size, 1, query_length, kv_length), in which case it is returned as-is. encoder_hidden_states (`torch.Tensor`, optional): The input embeddings of shape (batch_size, kv_length, hidden_dim). If provided, it is used instead of `inputs_embeds` to infer the batch size, kv length and dtype. or_mask_function (`Callable`, optional): An optional mask function to combine with the base mask function (by doing the union of both). This is useful to easily overlay another mask on top, for example for image tokens handling. and_mask_function (`Callable`, optional): An optional mask function to combine with the base mask function (by doing the intersection of both). This is useful to easily overlay another mask on top, for example for image tokens handling. rrNrTFr) rrrlrgrerkrrr&rr)r(r}ror+longrr&rCrrrrr>r8)rrrkrrrrembedsrrrlrgrr&rrrrs rrrs:D\\-"5"5a"8AUAU]b]g]ghN&;&G "]F:TdA;7J9i QJ71&2M2MNN#'H #2kl l ()>@P Q&+#$2kl l )*?AR S&+#$%+%"$? N r4c t|ddst|||||St|dr*d|jvr|jj d}nd}t |||||||\} }} } } | r|St|dd} | t d|jd|j}}t| }t|j}d }t|d d  }|!ts t d t||}d }d}|!ts t d t||}d }d}| t|t| }d }|||| | |||| ||| }|S) a Create a sliding window causal mask based on the attention implementation used (stored in the config). This type of attention pattern was mostly democratized by Mistral. If `past_key_values` has an hybrid cache structure, this function will return the mask corresponding to one of the "sliding_attention" layers (to align to what is needed in the `modeling_xxx.py` files). Args: config (`PreTrainedConfig`): The model config. inputs_embeds (`torch.Tensor`): The input embeddings of shape (batch_size, query_length, hidden_dim). This is used only to infer the batch size, query length and dtype. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences. or_mask_function (`Callable`, optional): An optional mask function to combine with the sliding causal mask function (by doing the union of both). This is useful to easily overlay another mask on top of the sliding causal one, for example for image tokens handling. and_mask_function (`Callable`, optional): An optional mask function to combine with the sliding causal mask function (by doing the intersection of both). This is useful to easily overlay another mask on top of the sliding causal one, for example for image tokens handling. rTrrrrDNJCould not find a `sliding_window` argument in the config, or it is not setFrr rrrlrgrerkrrr&rr)r(create_bidirectional_sliding_window_maskrrrrrror&rTrrrr>r8rd)rrrkrrrrrrrrarlrgrDrr&rrrrrs r!create_sliding_window_causal_maskr sR 6; -7   -/    -$/:T:T2T#..44T:  Mg ~~P\^gNJJ 4iV%5t@P Q$$2kl l )*?AR S$' )*?A^_sAt u$!%+%1! K r4ctj|jd|jtj}t ||||ddd\}}}}} |r|St |dd} | td|jd|j} } t| } t|j}d}d}|!ts td t| |} d}d}|!ts td t| |} d}d}|| ||| | |d|| | || }|S) a Create a standard bidirectional sliding window mask based on the attention implementation used (stored in the config). Args: config (`PreTrainedConfig`): The model config. inputs_embeds (`torch.Tensor`): The input embeddings of shape (batch_size, query_length, hidden_dim). This is only used to infer metadata such as the batch size, query length, dtype, and device. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, kv_length). It can also be an already prepared 4D mask of shape (batch_size, 1, query_length, kv_length), in which case it is returned as-is. or_mask_function (`Callable`, optional): An optional mask function to combine with the base mask function (by doing the union of both). This is useful to easily overlay another mask on top, for example for image tokens handling. and_mask_function (`Callable`, optional): An optional mask function to combine with the base mask function (by doing the intersection of both). This is useful to easily overlay another mask on top, for example for image tokens handling. rrNrrDrFTr) rrrlrgrerkrrrr&rr)r(r}ror+rrrrr&rZrrrr>r8)rrrkrrrrrrlrgrDrr&rrrrs rrrsU<\\-"5"5a"8AUAU]b]g]ghN;U ~~tT1;7J9iV%5t@P Q&+#$2kl l )*?AR S&+##%+%"$?! N r4c Vt|dr*d|jvr|jjd}nd}t|||||||\} }} } } | r|St |dd} | t dt |r| | z| kDr t d|jd|j}}|9|jd tj|k(jd }n&tj||jt }t!| |}t"|j$}d }t |d d  }|!t&s t d t)||}d }d}|!t&s t d t+||}d }d}| t+|t-| }d }|||| | |||| ||| }|S)a Create a chunked attention causal mask based on the attention implementation used (stored in the config). This type of attention pattern was mostly democratized by Llama4. If `past_key_values` has an hybrid cache structure, this function will return the mask corresponding to one of the "chunked_attention" layers (to align to what is needed in the `modeling_xxx.py` files). Args: config (`PreTrainedConfig`): The model config. inputs_embeds (`torch.Tensor`): The input embeddings of shape (batch_size, query_length, hidden_dim). This is used only to infer the batch size, query length and dtype. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences. or_mask_function (`Callable`, optional): An optional mask function to combine with the chunked causal mask function (by doing the union of both). This is useful to easily overlay another mask on top of the chunked causal one, for example for image tokens handling. and_mask_function (`Callable`, optional): An optional mask function to combine with the chunked causal mask function (by doing the intersection of both). This is useful to easily overlay another mask on top of the chunked causal one, for example for image tokens handling. rTrattention_chunk_sizeNzQCould not find an `attention_chunk_size` argument in the config, or it is not setzFlash attention cannot handle chunked attention, and the key-value length is larger than the chunk size so the chunked pattern cannot be respected. You should use another `attn_implementation` when instantiating the modelrn)rrFrrr)rrrrrrr ror&rr( zeros_likesumzerosr+rJr\rrrr>r8rd)rrrkrrrrrrrrarlrgrMrr&left_padding_tokensrrrrrs rcreate_chunked_causal_maskrsP -$/:T:T2T#..44T:  Mg ~~P\^gNJJ 4i!7>Jlmm$F+ I0E 0R }  &++A. 0C0CJ!-444<@P@PQ_@``eejlem#kk*^=R=RZ]^8EXY1&2M2MNN H '8H%PP #2kl l ()>@P Q$$2kl l )*?AR S$' )*?A^_sAt u$!%+%1 K r4)full_attentionsliding_attentionchunked_attentionc $|j} | |||||||d} t| dr/i} t| jD]} t | di| | | <| St | dd t di| St | dd tdi| Stdi| S)a This function mimics how we create the masks in the `modeling_xxx.py` files, and is used in places like `generate` in order to easily create the masks in advance, when we compile the forwards with Static caches. Args: config (`PreTrainedConfig`): The model config. inputs_embeds (`torch.Tensor`): The input embeddings of shape (batch_size, query_length, hidden_dim). This is used only to infer the batch size, query length and dtype. attention_mask (`torch.Tensor`, optional): The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length). It can also be an already prepared 4D mask, in which case it is returned as-is. cache_position (`torch.Tensor`): A tensor of shape (query_length,) indicating the current indices of the input sequence elements. past_key_values (`Cache`, optional): The past key values, if we use a cache. position_ids (`torch.Tensor`, optional) A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences. or_mask_function (`Callable`, optional): An optional mask function to combine with the other mask function (by doing the union of both). This is useful to easily overlay another mask on top of the causal one, for example for image tokens handling. and_mask_function (`Callable`, optional): An optional mask function to combine with the other mask function (by doing the intersection of both). This is useful to easily overlay another mask on top of the causal one, for example for image tokens handling. )rrrkrrrrr layer_typesrDNrr%) get_text_configrsetr &LAYER_PATTERN_TO_MASK_FUNCTION_MAPPINGrrrr) rrrkrrrrrreffective_config mask_kwargs causal_masks layer_patterns rcreate_masks_for_generaterOsN--/#&((*$,. K/  !1!=!=> oM*PQ^*_*nbm*nL ' o !#3T : F0?;?? !#94 @ L)8K88  , ,,r4zzzu■u⬚u∙u⬕u⬔c>|dk(r d}d}d}d}d}nd}d}d}d }||||fS) Nmajongu🀞u🀙u🀆u🀛u█u░u▙u▜r%)style BLACK_SQUARE WHITE_SQUARE LOW_TRIANGLEUPPER_TRIANGLEs r get_stylersF         |^ CCr4original_tensorc t|\}}}}|j\}}|\} } || kr|| ksd|z|z } | dkDr%| }t| tdt | | z }n| }tdt | | z}|j dj d} t j| ||fd} n|} g} t|D]}d}t|D]}| ||fdk(r||z }| ||fdk(r||z }#|dkDr8| ||dz fdk(r||z };| ||dz fdk(r||z }N|| ||fdk(r|n|z }`|| ||fdk(r|n| ||fdk(r|n| ||dzfdk(r|n|z }| j|dj| S)Nrrr) output_size)rr ) rrormaxround unsqueezeFadaptive_avg_pool2drangeappendjoin)r grid_sizerrrrrhwmax_hmax_w aspect_ratiorr1irowjs rtensor_to_mask_visualr1s?H?O?AAAuU\123A!**1-77:&&vAq6B4H F 1Xq Aad|q |#1"|#q5aQh'1,|+1q5)Q.~-vad|q/@|lR!!Q$<1,% &ad|q0)4:1a!e84D4I.| C .  c36 99V r4cheZdZd dZdZd dZdZdZed de jde dzd dfd Z y) AttentionMaskNcT||_tjj||dS)NF) require_grad)rr(r_make_subclass)clsdatars r__new__zAttentionMask.__new__s% ||**35*IIr4cyrr%)selfr8s r__init__zAttentionMask.__init__s r4c |}|j^}}}g}ttj|Dcgc] }t |c}D]k\} } | |k(r5|j d|j d|j dn.t || ||j} |j | m|j dt|jd|jddj|Scc}w) z2Returns a string representation of the block mask.z...z7To print out more, set AttentionMask.to_string(limit=N)zRYou can also index (AttentionMask[batch, head]) to choose a specific batch or head)r(rztorch.Tensor(shape=z, dtype=)r) ro enumerate itertoolsproductr%r&r1rtupler&r') r;r(limit dense_mask batch_dimsnum_rowsnum_cols total_visr.idxr- block_viss r to_stringzAttentionMask.to_strings *4*:*:'Xx ' (9(9j;YE!H;Y(Z[ (NCe|  '  !Z[  !uv-j.Cy`d`j`jkI   Y ' ( .uTZZ/@.A$**UVWXyy##rJr)rArCrLrQrTrXrZr\r`rdrjrurzrrrrrrfloat32r&rrrrr__annotations__rrBrrrrrrr rGREENYELLOWRESETrr GREY_SQUARErrr YELLOW_SQUARE GREEN_SQUARErZr1r3r%r4rros $ 12.Icc !gNN I&?RV&W#&?RV&W#02&T   H % x H  h 8 C3sCTX3#cSV[_ 3 8   5<< H SSS   hshxhVSV VQYV      C TW \d  )<  Y\ afamamptat !],,%!]!]!]* !]  !]R(, ),,%))) ) * )  )X,,%* <(,,,%* >  X ><<>/4||>HM >bgbnbn>.2*.!!%(- ddLLdd d  d LL4' dd dd"&ddd \\DdV2*.(-<<LL<< <  < LL4' < ;;<"&<< \\<F2*. $$LL$$ $  $ LL4' $V2*. 66LL66 6  6 LL4' 66r - 8N7O4O u||  t@S @?KXM XM<<XMLL9,t3XMLL XM T\ XM ,,% XMTzXM 4 )D0#s :;XMLXMv?K)-(,)-o o<<oLL4'oLL o T\ o ,,% ooo $o \\I$oLod?K 26(,)- U U<<ULL4'U!<<$. U o U  $ U \\I$ULUp?K)-(,)-p p<<pLL4'pLL p T\ p ,,% pop $p \\I$pLpf?K )-)- L L<<LLL4'Lo L  $ L  \\I$ LLL^?K)-(,)-s s<<sLL4'sLL s T\ s ,,% sos $s \\I$sLsn):3*&?K)-(,)-@- @-<<@-LL4'@-LL @- T\ @- ,,% @-o@- $@-L@-J       D$(<.0 ug. 15<<1`c1h&ELL&r4