
    
i                        d Z ddlZddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZ ddlmZmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0m1Z1m2Z2 ddl3m4Z4m5Z5m6Z6  e(jn                  e8      Z9ee& G d de                    Z:e e&d       G d de$                    Z; G d de	jx                        Z=	 dMd e	jx                  d!ej|                  d"ej|                  d#ej|                  d$ej|                  dz  d%e?d&e?fd'Z@ G d( d)e	jx                        ZA G d* d+e	jx                        ZB G d, d-e      ZCe& G d. d/e             ZD G d0 d1e	jx                        ZE G d2 d3eD      ZF G d4 d5e	jx                        ZG G d6 d7e	jx                        ZH G d8 d9e	jx                        ZI G d: d;e	jx                        ZJ G d< d=e	jx                        ZK G d> d?e      ZL G d@ dAe	jx                        ZM G dB dCe	jx                        ZN G dD dEeD      ZO e&dF       G dG dHeD             ZP e&dI       G dJ dKeDe             ZQg dLZRy)NzPyTorch InstructBLIP model.    N)Callable)	dataclass)Any)nn   )initialization)ACT2FN)GenerationMixin)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsBaseModelOutputWithPooling,BaseModelOutputWithPoolingAndCrossAttentionsCausalLMOutputWithPastSeq2SeqLMOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging	torch_int)merge_with_config_defaults)OutputRecordercapture_outputs   )	AutoModelAutoModelForCausalLMAutoModelForSeq2SeqLM   )InstructBlipConfigInstructBlipQFormerConfigInstructBlipVisionConfigc                   :    e Zd ZU dZdZedz  ed<   dZedz  ed<   y)'BaseModelOutputWithVisionQformerOutputsz
    vision_outputs (`BaseModelOutputWithPooling`):
        Outputs of the vision encoder.
    qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
        Outputs of the Q-Former (Querying Transformer).
    Nvision_outputsqformer_outputs)	__name__
__module____qualname____doc__r*   r   __annotations__r+   r        r/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/models/instructblip/modeling_instructblip.pyr)   r)   2   s)     9=N.5<KOOADHOr2   r)   zQ
    Class defining the outputs of [`InstructBlipForConditionalGeneration`].
    )custom_introc                       e Zd ZU dZdZeej                     dz  ed<   dZ	eej                     dz  ed<   dZ
edz  ed<   dZedz  ed<   dZeez  dz  ed<   dee   fd	Zy)
/InstructBlipForConditionalGenerationModelOutputa  
    loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        Language modeling loss from the language model.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head of the language model.
    vision_outputs (`BaseModelOutputWithPooling`):
        Outputs of the vision encoder.
    qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
        Outputs of the Q-Former (Querying Transformer).
    language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`):
        Outputs of the language model.
    Nlosslogitsr*   r+   language_model_outputsreturnc                 H     t         fd j                         D              S )Nc              3   d   K   | ]'  }|d vr|   nt        |      j                          ) yw)r*   r+   r9   N)getattrto_tuple).0kselfs     r3   	<genexpr>zKInstructBlipForConditionalGenerationModelOutput.to_tuple.<locals>.<genexpr>\   sB      
  WW Gq!**,-
s   -0)tuplekeysrB   s   `r3   r?   z8InstructBlipForConditionalGenerationModelOutput.to_tuple[   s%     
 YY[	
 
 	
r2   )r,   r-   r.   r/   r7   rD   torchFloatTensorr0   r8   r*   r   r+   r   r9   r   r   r   r?   r1   r2   r3   r6   r6   @   s     -1D%!!
"T
)0.2FE%##$t+28<N.5<KOOADHONR2_DtKR
%* 
r2   r6   c                        e Zd Zdef fdZdej                  dededej                  fdZddej                  d	e
dej                  fd
Z xZS )InstructBlipVisionEmbeddingsconfigc                 p   t         |           || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  dd| j                              | _        t        j                  d| j                  | j                  | j                        | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j                  t        j                  d| j                  | j                              | _        y )Nr$   r   )in_channelsout_channelskernel_sizestrider    )super__init__rK   hidden_size	embed_dim
image_size
patch_sizer   	ParameterrG   randnclass_embeddingConv2dpatch_embeddingnum_patchesnum_positionsposition_embeddingrB   rK   	__class__s     r3   rR   z%InstructBlipVisionEmbeddings.__init__f   s    ++ ++ ++!||EKK1dnn,MN!yyDOO\`\k\k 
 !OOt>1D!--1"$,,u{{1d>P>PRVR`R`/a"br2   
embeddingsheightwidthr:   c                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t        j                  ||fd      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r$   Ng      ?r   r   r    bicubicF)sizemodealign_cornersdim)shaper^   rG   jit
is_tracingrV   r   reshapepermuter   
functionalinterpolateviewcat)rB   ra   rb   rc   r\   r]   class_pos_embedpatch_pos_embedrk   
new_height	new_widthsqrt_num_positionss               r3   interpolate_pos_encodingz5InstructBlipVisionEmbeddings.interpolate_pos_encodingx   s`    !&&q)A-//55a81< yy##%+*F6UZ?***11!RaR%811!QR%8r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy/?;CCr2   pixel_valuesrz   c                    |j                   \  }}}}| j                  j                  j                  }| j                  |j	                  |            }|j                  d      j                  dd      }| j                  j                  |dd      j	                  |      }	t        j                  |	|gd      }
|r| j                  |
||      }n| j                  }|
|d d d |
j                  d      d d f   j	                  |      z   }
|
S )Ndtyper    r$   re   rj   )rl   r[   weightr~   toflatten	transposerY   expandrG   rt   rz   r^   rg   )rB   r{   rz   
batch_size_rb   rc   target_dtypepatch_embedsclass_embedsra   r^   s               r3   forwardz$InstructBlipVisionEmbeddings.forward   s    '3'9'9$
Avu++2288++LOO,O,OP#++A.88A>++22:q"EHHVYYl;C
#!%!>!>z6SX!Y!%!8!8"4Q8L*//!:L8La5O"P"S"ST`"aa
r2   F)r,   r-   r.   r'   rR   rG   Tensorintrz   rH   boolr   __classcell__r`   s   @r3   rJ   rJ   e   sm    c7 c$&D5<< &D &DUX &D]b]i]i &DPE$5$5 QU bgbnbn r2   rJ   modulequerykeyvalueattention_maskscalingdropoutc                 p   t        j                  ||j                  dd            |z  }|||z   }t        j                  j                  |d      }t        j                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nre   rj   )ptrainingr$   r    )	rG   matmulr   r   rq   softmaxr   r   
contiguous)
r   r   r   r   r   r   r   kwargsattn_weightsattn_outputs
             r3   eager_attention_forwardr      s     <<s}}R'<=GL!#n4==((2(>L==((6??([L,,|U3K''1-88:K$$r2   c                        e Zd ZdZ fdZdej                  dedefdZdej                  de	ej                  ej                  d	z  e	ej                     d	z  f   fd
Z
 xZS )InstructBlipAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                    t         |           || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                   d| j                   d      | j                  dz  | _	        d| _
        |j                  | _        t        j                  | j                  d| j                  z  d      | _        |j                  ret        j                   t#        j$                  | j                              }t        j                   t#        j$                  | j                              }nd }d }|Qt#        j&                  |t#        j(                  |d      |f      }t        j                   |      | j                  _        t        j                  | j                  | j                        | _        y )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      Fr   )bias)requires_grad)rQ   rR   rK   rS   rT   num_attention_heads	num_headshead_dim
ValueErrorscale	is_causalattention_dropoutr   Linearqkvqkv_biasrW   rG   zerosrt   
zeros_liker   
projection)rB   rK   q_biasv_biasr   r`   s        r3   rR   zInstructBlipAttention.__init__   su   ++33$..8==4>>)T^^;MdnnM] ^NN#2'  ]]D(
!'!9!9 99T^^Q-?eL??\\%++dnn"=>F\\%++dnn"=>FFFyy&%*:*:6QV*WY_!`aHLL2DHHM))DNNDNNCr2   tensorseq_lenbszc                     |j                  ||| j                  | j                        j                  dd      j	                         S )Nr$   r    )rs   r   r   r   r   )rB   r   r   r   s       r3   _shapezInstructBlipAttention._shape   s7    {{3GQQRSUVWbbddr2   hidden_statesr:   Nc                    |j                         \  }}}| j                  |      }|j                  ||d| j                  || j                  z        j	                  ddddd      }|d   |d   |d   }	}}t        j                  | j                  j                  t              }
 |
| |||	fd| j                  sdn| j                  | j                  d|\  }}|j                  ||d	      j                         }| j                  |      }||fS )
z#Input shape: Batch x Time x Channelr   r    r   r$      N        )r   r   r   re   )rg   r   ro   r   rp   r   get_interfacerK   _attn_implementationr   r   r   r   r   r   )rB   r   r   r   tgt_lenrT   	mixed_qkvquery_states
key_statesvalue_statesattention_interfacer   r   s                r3   r   zInstructBlipAttention.forward   s#    #0"4"4"6WiHH]+	%%c7At~~yTXTbTbGbckkq!Q
	 2;1y|YWX\,j(?(M(MKK,,.E)
 %8		%

  #}}C$2H2HJJ	%
 	%
!\ "))#w;FFHook2L((r2   )r,   r-   r.   r/   rR   rG   r   r   r   rD   r   r   r   s   @r3   r   r      sp    GD>eU\\ eC ec e")||") 
u||U\\D0%2E2LL	M	")r2   r   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )InstructBlipMLPc                    t         |           || _        t        |j                     | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y N)rQ   rR   rK   r	   
hidden_actactivation_fnr   r   rS   intermediate_sizefc1fc2r_   s     r3   rR   zInstructBlipMLP.__init__  sd    #F$5$5699V//1I1IJ99V55v7I7IJr2   r   r:   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   rB   r   s     r3   r   zInstructBlipMLP.forward  s4    /**=9/r2   r,   r-   r.   rR   rG   r   r   r   r   s   @r3   r   r     s$    KU\\ ell r2   r   c                   p     e Zd Zdef fdZedej                  dee	   dej                  fd       Z xZS )InstructBlipEncoderLayerrK   c                 D   t         |           |j                  | _        t	        |      | _        t        j                  | j                  |j                        | _	        t        |      | _        t        j                  | j                  |j                        | _        y Neps)rQ   rR   rS   rT   r   	self_attnr   	LayerNormlayer_norm_epslayer_norm1r   mlplayer_norm2r_   s     r3   rR   z!InstructBlipEncoderLayer.__init__$  sm    ++.v6<<F<Q<QR"6*<<F<Q<QRr2   r   r   r:   c                     |}| j                  |      } | j                  dd|i|\  }}||z   }|}| j                  |      }| j                  |      }||z   }|S )Nr   r1   )r   r   r   r   )rB   r   r   residualr   s        r3   r   z InstructBlipEncoderLayer.forward,  s}     !((7)4>> 
'

q &0 ((7/%0r2   )r,   r-   r.   r%   rR   r   rG   r   r   r   rH   r   r   r   s   @r3   r   r   #  sR    S1 S || +, 
			 r2   r   c                   z     e Zd ZU eed<   dZdZdZdZdZ	dZ
dZdZg dZ ej                          fd       Z xZS )InstructBlipPreTrainedModelrK   blip)imagetextT)InstructBlipQFormerEmbeddingsr   %InstructBlipQFormerMultiHeadAttentionInstructBlipQFormerSelfOutputc                 :   t         |   |       | j                  j                  }t	        |t
              rEt        j                  |j                  d|       t        j                  |j                  d|       yt	        |t        t        f      r t        j                  |j                         yt	        |t              rZt        j                  |j                   t#        j$                  |j                   j&                  d         j)                  d             yy)zInitialize the weightsr   )meanstdre   r$   re   N)rQ   _init_weightsrK   initializer_range
isinstancerJ   inittrunc_normal_r^   rY   $InstructBlipForConditionalGenerationInstructBlipModelzeros_query_tokensr   copy_position_idsrG   arangerl   r   )rB   r   factorr`   s      r3   r   z)InstructBlipPreTrainedModel._init_weightsW  s     	f%..f:;v88sOv55CVL!EGX YZKK++, =>JJv**ELL9L9L9R9RSU9V,W,^,^_f,gh ?r2   )r,   r-   r.   r%   r0   base_model_prefixinput_modalitiessupports_gradient_checkpointing_supports_attention_backend_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_no_split_modulesrG   no_gradr   r   r   s   @r3   r   r   C  s^    (&*#"&N! U]]_
i 
ir2   r   c                   N     e Zd ZdZdef fdZedee   de	e
z  fd       Z xZS )InstructBlipEncodera  
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`InstructBlipEncoderLayer`].

    Args:
        config (`InstructBlipConfig`):
            The corresponding vision configuration for the `InstructBlipEncoder`.
    rK   c                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w NF)
rQ   rR   rK   r   
ModuleListrangenum_hidden_layersr   layersgradient_checkpointing)rB   rK   r   r`   s      r3   rR   zInstructBlipEncoder.__init__p  sP    mmuU[UmUmOn$o!%=f%E$op&+# %ps   A#r   r:   c                 R    |}| j                   D ]  } ||fi |} t        |      S )Nlast_hidden_state)r  r   )rB   inputs_embedsr   r   encoder_layers        r3   r   zInstructBlipEncoder.forwardv  s>     &![[ 	M)M	 ??r2   )r,   r-   r.   r/   r%   rR   r   r   r   rD   r   r   r   r   s   @r3   r  r  f  sL    ,1 , @ +,@ 
	 	@ @r2   r  c                        e Zd ZU dZdZeed<   eedZ	def fdZ
e ed      e	 	 ddej                  dz  d	ed
ee   deez  fd                     Zd Z xZS )InstructBlipVisionModelr{   )r   rK   )r   
attentionsc                     t         |   |       || _        |j                  }t	        |      | _        t        |      | _        t        j                  ||j                        | _        | j                          y r   )rQ   rR   rK   rS   rJ   ra   r  encoderr   r   r   post_layernorm	post_init)rB   rK   rT   r`   s      r3   rR   z InstructBlipVisionModel.__init__  s]     &&	6v>*62 ll9&:O:OPr2   F)tie_last_hidden_statesNrz   r   r:   c                     |t        d      | j                  ||      } | j                  dd|i|}|j                  }| j	                  |      }|d d dd d f   }| j	                  |      }t        ||      S )Nz You have to specify pixel_values)rz   r  r   r  pooler_outputr1   )r   ra   r  r  r  r   )rB   r{   rz   r   r   encoder_outputsr  pooled_outputs           r3   r   zInstructBlipVisionModel.forward  s     ?@@Ogh+74<< ,
',
,

 ,== //0AB)!Q'2++M:)/'
 	
r2   c                     | j                   S r   )ra   rF   s    r3   get_input_embeddingsz,InstructBlipVisionModel.get_input_embeddings  s    r2   r  )r,   r-   r.   main_input_namer   r'   r0   r   r   _can_record_outputsrR   r   r   r   rG   rH   r   r   r   rD   r   r   r   r   r   s   @r3   r  r    s    $O!$$1+
	7 	  E2 26).
''$.
 #'
 +,	

 
+	+
  3  
6r2   r  c                   X     e Zd Zd	 fd	Zd Zd Zd Zd Zd Z	 	 	 d
de	e
   fdZ xZS )r   c                    t         |           || _        |j                  |j                  z  dk7  r0t        |d      s$t        d|j                  |j                  fz        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        t        j                  |j                  | j                        | _        |r_t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        n^t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                   |j"                        | _        d| _        y )Nr   embedding_sizezLThe hidden size (%d) is not a multiple of the number of attention heads (%d)F)rQ   rR   rK   rS   r   hasattrr   r   attention_head_sizeall_head_sizer   r   r   encoder_hidden_sizer   r   Dropoutattention_probs_dropout_probr   save_attentionrB   rK   is_cross_attentionr`   s      r3   rR   z.InstructBlipQFormerMultiHeadAttention.__init__  sa    : ::a?PVXhHi^%%v'A'ABC 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
yy!;!;T=O=OPDH6#=#=t?Q?QRDJyy!3!3T5G5GHDH6#5#5t7I7IJDJzz&"E"EF#r2   c                     || _         y r   attn_gradients)rB   r1  s     r3   save_attn_gradientsz9InstructBlipQFormerMultiHeadAttention.save_attn_gradients  s
    ,r2   c                     | j                   S r   r0  rF   s    r3   get_attn_gradientsz8InstructBlipQFormerMultiHeadAttention.get_attn_gradients  s    """r2   c                     || _         y r   attention_map)rB   r7  s     r3   save_attention_mapz8InstructBlipQFormerMultiHeadAttention.save_attention_map  s
    *r2   c                     | j                   S r   r6  rF   s    r3   get_attention_mapz7InstructBlipQFormerMultiHeadAttention.get_attention_map  s    !!!r2   c                     |j                         d d | j                  | j                  fz   } |j                  | }|j	                  dddd      S )Nre   r   r    r$   r   )rg   r   r'  rs   rp   )rB   xnew_x_shapes      r3   transpose_for_scoresz:InstructBlipQFormerMultiHeadAttention.transpose_for_scores  sN    ffhsmt'?'?AYAY&ZZAFFK yyAq!$$r2   r   c                    |d u}|rC| j                  | j                  |            }| j                  | j                  |            }|}n@| j                  | j                  |            }| j                  | j                  |            }| j                  |      }	| j                  |	      }
t	        j
                  |
|j                  dd            }|t        j                  | j                        z  }|j                  }|||z   } t        j                  d      |      j                  |      }|r8| j                  r,| j                  |       |j!                  | j"                         | j%                  |      }t	        j
                  ||      }|j'                  dddd      j)                         }|j+                         d d | j,                  fz   } |j.                  | }||fS )Nre   r   rj   r   r    r$   r   )r>  r   r   r   rG   r   r   mathsqrtr'  r~   r   Softmaxr   r,  r8  register_hookr2  r   rp   r   rg   r(  rs   )rB   r   r   encoder_hidden_statesencoder_attention_maskr   r.  	key_layervalue_layermixed_query_layerquery_layerattention_scoresattention_scores_dtypeattention_probsattention_probs_droppedcontext_layernew_context_layer_shapes                    r3   r   z-InstructBlipQFormerMultiHeadAttention.forward  s    3$>11$((;P2QRI33DJJ?T4UVK3N11$((=2IJI33DJJ}4MNK JJ}5//0AB !<<Y5H5HR5PQ+dii8P8P.QQ!1!7!7%/.@ -"**,-=>AABXY$"5"5##O4))$*B*BC #',,"?%<kJ%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S***,CDo--r2   r   NNN)r,   r-   r.   rR   r2  r4  r8  r:  r>  r   r   r   r   r   s   @r3   r   r     sA    $0-#+"% "#4. +,4.r2   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )r   c                 (   t         |           t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                        | _
        y r   )rQ   rR   r   r   rS   denser   r   r*  hidden_dropout_probr   r_   s     r3   rR   z&InstructBlipQFormerSelfOutput.__init__  s`    YYv1163E3EF
f&8&8f>S>STzz&"<"<=r2   r   input_tensorr:   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r   rS  r   r   rB   r   rU  s      r3   r   z%InstructBlipQFormerSelfOutput.forward%  7    

=1]3}|'CDr2   r   r   s   @r3   r   r     1    >U\\  RWR^R^ r2   r   c                        e Zd Zd
 fd	Z	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dee   dej
                  fd	Z	 xZ
S )InstructBlipQFormerAttentionc                 d    t         |           t        ||      | _        t	        |      | _        y r   )rQ   rR   r   	attentionr   outputr-  s      r3   rR   z%InstructBlipQFormerAttention.__init__.  s)    >vGYZ3F;r2   Nr   r   rD  rE  r   r:   c                 ^     | j                   d||||d|\  }}| j                  ||      }|S )N)r   r   rD  rE  r1   )r^  r_  )	rB   r   r   rD  rE  r   r   r   attention_outputs	            r3   r   z$InstructBlipQFormerAttention.forward3  sL     ( 
')"7#9	

 
Q  ;;{MBr2   r   rP  )r,   r-   r.   rR   rG   r   rH   r   r   r   r   r   s   @r3   r\  r\  -  s    < 48:>;? ||  ))D0   %0047	 
 !& 1 1D 8  +,  
 r2   r\  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )InstructBlipQFormerIntermediatec                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )rQ   rR   r   r   rS   r   rS  r   r   strr	   intermediate_act_fnr_   s     r3   rR   z(InstructBlipQFormerIntermediate.__init__H  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r2   r   r:   c                 J    | j                  |      }| j                  |      }|S r   )rS  rf  r   s     r3   r   z'InstructBlipQFormerIntermediate.forwardP  s&    

=100?r2   r   r   s   @r3   rc  rc  G  s#    9U\\ ell r2   rc  c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )InstructBlipQFormerOutputc                 (   t         |           t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        y r   )rQ   rR   r   r   r   rS   rS  r   r   r*  rT  r   r_   s     r3   rR   z"InstructBlipQFormerOutput.__init__X  s`    YYv779K9KL
f&8&8f>S>STzz&"<"<=r2   r   rU  r:   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r   rW  rX  s      r3   r   z!InstructBlipQFormerOutput.forward^  rY  r2   r   r   s   @r3   ri  ri  W  rZ  r2   ri  c                   F     e Zd Z fdZ	 	 	 	 ddee   fdZd Zd Z xZ	S )InstructBlipQFormerLayerc                 f   t         |           |j                  | _        d| _        t	        |      | _        || _        ||j                  z  dk(  rt	        |d      | _        d| _	        nd| _	        t        |      | _        t        |      | _        t        |      | _        t        |      | _        y )Nr$   r   T)r.  F)rQ   rR   chunk_size_feed_forwardseq_len_dimr\  r^  	layer_idxcross_attention_frequencycrossattentionhas_cross_attentionrc  intermediateri  r_  intermediate_queryoutput_queryrB   rK   rq  r`   s      r3   rR   z!InstructBlipQFormerLayer.__init__f  s    '-'E'E$5f="v7771<">vZ^"_D'+D$',D$;FC/7"A&"I5f=r2   r   c           
      b    | j                   |fd|i|}|dkD  r|d d d |d d f   }| j                  r$|t        d       | j                  |f|||d|}t	        | j
                  | j                  | j                  |      }	|j                  d   |kD  rjt	        | j                  | j                  | j                  |d d |d d d f         j                  |	j                        }
t        j                  |	|
gd      }	|	S t	        | j                  | j                  | j                  |      }	|	S )Nr   r   z>encoder_hidden_states must be given for cross-attention layers)r   rD  rE  r$   rj   )r^  rt  r   rs  r   feed_forward_chunk_queryro  rp  rl   feed_forward_chunkr   devicerG   rt   )rB   r   r   rD  rE  query_lengthr   ra  query_attention_outputlayer_outputlayer_output_texts              r3   r   z InstructBlipQFormerLayer.forwardz  sp    *4>>
)
 
 !%5a,6I%J"''(0$%eff)<)<)<**#1*?+A	*
 *& 5--,,  &	L  %%a(<7$=++00$$$Qq%89	%
 "\(() "  %yy,8I)JPQR  5'',,   	L r2   c                 L    | j                  |      }| j                  ||      }|S r   )ru  r_  rB   ra  intermediate_outputr  s       r3   r{  z+InstructBlipQFormerLayer.feed_forward_chunk  s,    "//0@A{{#68HIr2   c                 L    | j                  |      }| j                  ||      }|S r   )rv  rw  r  s       r3   rz  z1InstructBlipQFormerLayer.feed_forward_chunk_query  s.    "556FG(()<>NOr2   NNNr   )
r,   r-   r.   rR   r   r   r   r{  rz  r   r   s   @r3   rm  rm  e  s6    >. "#3 +,3j
r2   rm  c                   D     e Zd Z fdZe	 	 	 	 ddee   fd       Z xZS )InstructBlipQFormerEncoderc           	          t         |           || _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        d| _	        y c c}w r  )
rQ   rR   rK   r   r  r	  r
  rm  layerr  rx  s      r3   rR   z#InstructBlipQFormerEncoder.__init__  sW    ]]JOPVPhPhJijY%fi8j

 ',# ks   A$r   c                     t        | j                  j                        D ]  }| j                  |   } ||||f||d|}! t	        |      S )N)rE  r}  r  )r	  rK   r
  r  r   )	rB   r   r   rD  rE  r}  r   ilayer_modules	            r3   r   z"InstructBlipQFormerEncoder.forward  sk     t{{445 
	A::a=L(% (>) M
	 9+
 	
r2   r  )	r,   r-   r.   rR   r   r   r   r   r   r   s   @r3   r  r    s:    ,  "#
 +,
 
r2   r  c                   2     e Zd ZdZ fdZ	 	 	 	 ddZ xZS )r   z;Construct the embeddings from word and position embeddings.c                 ,   t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j                  |j
                        | _	        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        | j!                  dt#        j$                  |j                        j'                  d      d       || _        y )N)padding_idxr   r   r   F)
persistent)rQ   rR   r   	Embedding
vocab_sizerS   pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsr   r   	layernormr*  rT  r   register_bufferrG   r   r   rK   r_   s     r3   rR   z&InstructBlipQFormerEmbeddings.__init__  s    !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c f&8&8f>S>STzz&"<"<= 	ELL)G)GHOOPWXej 	 	
 r2   c                    ||j                         d   }nd}|&| j                  d d |||z   f   j                         }|\| j                  |      }| j	                  |j                  |j                              }||z   }|t        j                  ||fd      }n|}|j                  | j                  j                  j                        }| j                  |      }| j                  |      }|S )Nr$   r   rj   )rg   r   cloner  r  r   r|  rG   rt   r  r   r~   r   )rB   	input_idsr   query_embedspast_key_values_length
seq_lengthra   r  s           r3   r   z%InstructBlipQFormerEmbeddings.forward  s      ")!,JJ,,Q0FVlIl0l-lmssuL --i8J"&":":<??:K\K\;]"^#&99J'"YYj'AqI
%J]]4>>#8#8#>#>?
^^J/
\\*-
r2   r  )r,   r-   r.   r/   rR   r   r   r   s   @r3   r   r     s    E"  r2   r   c                       e Zd ZdZdZdZdZdZe e	e
dd      g e	e
dd      gdZdef fd	Zd
 Zd Z	 ddej"                  dee   dej(                  dedej"                  f
dZeee	 	 	 	 	 ddej4                  dej6                  dz  dej4                  dz  dej"                  dz  dej6                  dz  dej6                  dz  dee   deej6                     ez  fd                     Z xZ S )InstructBlipQFormerModelz
    Querying Transformer (Q-Former), used in InstructBLIP. Slightly modified from BLIP-2 as it also takes the
    instruction as input.
    Fr$   z
.attention)index
layer_namez.crossattention)r   r  cross_attentionsrK   c                     t         |   |       || _        t        |      | _        t        |      | _        | j                          y r   )rQ   rR   rK   r   ra   r  r  r  r_   s     r3   rR   z!InstructBlipQFormerModel.__init__&  s9     7?1&9r2   c                 .    | j                   j                  S r   ra   r  rF   s    r3   r   z-InstructBlipQFormerModel.get_input_embeddings0  s    ...r2   c                 &    || j                   _        y r   r  rB   r   s     r3   set_input_embeddingsz-InstructBlipQFormerModel.set_input_embeddings3  s    */'r2   r   input_shaper|  	has_queryr:   c                    |j                         dk(  r|dddddddf   }n=|j                         dk(  r|ddddddf   }nt        d| d|j                   d      |j                  | j                        }d|z
  d	z  }|S )
a>  
        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.

        Arguments:
            attention_mask (`torch.Tensor`):
                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
            input_shape (`tuple[int]`):
                The shape of the input to the model.
            device: (`torch.device`):
                The device of the input to the model.

        Returns:
            `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
        r   Nr    z!Wrong shape for input_ids (shape z) or attention_mask (shape )r}   g      ?g     )rk   r   rl   r   r~   )rB   r   r  r|  r  extended_attention_masks         r3   get_extended_attention_maskz4InstructBlipQFormerModel.get_extended_attention_mask6  s    . 1$&4Qa]&C#!Q& '5QdA5E&F#3K=@[\j\p\p[qqrs  #:"<"<4::"<"N#&)@#@H"L&&r2   Nr  r   r  rD  rE  r   c                    ||t        d      ||j                  d   nd}| j                  |||      }	|	j                         dd }
|
\  }}|	j                  }|t        j                  ||f|      }| j                  ||
|      }|t        |t              r|d   j                         \  }}}n|j                         \  }}}||f}t        |t              r|D cg c]  }| j                  |       }}n?|)t        j                  ||      }| j                  |      }n| j                  |      }nd} | j                  |	f||||d|}|j                  }|dddddf   }t        ||	      S c c}w )
a$  
        query_embeds (`torch.FloatTensor`  of shape `(batch_size, sequence_length, hidden_size)`):
            Hidden states to be used in the attention computation. If cross-attention,
            will be used for the query (i.e., key and value will use the encoder_hidden_states).
        Nz7You have to specify query_embeds when input_ids is Noner$   r   )r  r   r  re   )r|  )r   rD  rE  r}  r  )r   rl   ra   rg   r|  rG   onesr  r   listinvert_attention_maskr  r  r   )rB   r  r   r   r  rD  rE  r   r}  embedding_outputr  r   r  r|  r  encoder_batch_sizeencoder_sequence_lengthr   encoder_hidden_shapemaskencoder_extended_attention_maskr  sequence_outputr  s                           r3   r   z InstructBlipQFormerModel.forwarda  s   $ !5VWW0<0H|))!,a??%% + 
 '++-cr2!,
J!((!"ZZ*j)A6RN #'"B"B>S^`f"g !,/6AVWXAYA^A^A`>"$;QAVA[A[A]>"$;Q$68O#P 0$7`v2wX\43M3Md3S2w/2w'/).4HQW)X&262L2LMc2d/262L2LMc2d/.2++74<<,
2"7#B%,
 ,
 *;;'1a0;-'
 	
) 3xs   E6r   )NNNNN)!r,   r-   r.   r/   r   r   r   r   rm  r   r   r"  r&   rR   r   r  rG   r   rD   r   r|  r   r  r   r   r   
LongTensorrH   r   r   r   r   r   r   s   @r3   r  r    s   
 #( N 2@Vbc
 @Vgh
8 /0  )')' 3Z)' 	)'
 )' 
)'V   4804,0:>;?F
##F
 ))D0F
 &&-	F

 llT)F
  %0047F
 !& 1 1D 8F
 +,F
 
u  	!$P	PF
    F
r2   r  z[
    InstructBLIP base Model consisting of language model, qformer and vision encoder.
    c                       e Zd ZdZdgZdef fdZd Zd Zd Z	de
j                  d	e
j                  fd
Zee	 	 	 	 	 	 	 dde
j                  de
j                  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  d	e
j                   dz  dedee   deez  fd              Z xZS )r   r{   r   rK   c                    t         |   |       t        |j                        | _        t        j                  t        j                  d|j                  |j                  j                              | _        t        |j                        | _        t        j                  |j                  j                  |j                   j                        | _        t%        j&                  |j                         | _        | j+                          y Nr$   )rQ   rR   r  vision_configvision_modelr   rW   rG   r   num_query_tokensqformer_configrS   r   r  qformerr   text_configlanguage_projectionr!   from_configlanguage_modelr  r_   s     r3   rR   zInstructBlipModel.__init__  s     3F4H4HILLQ8O8OQWQfQfQrQr)st/0E0EF#%99V-B-B-N-NPVPbPbPnPn#o '33F4F4FG 	r2   c                 6    | j                   j                         S r   r  r   rF   s    r3   r   z&InstructBlipModel.get_input_embeddings      ""7799r2   c                 :    | j                   j                  |       y r   r  r  r  s     r3   r  z&InstructBlipModel.set_input_embeddings      007r2   c                    | j                   }t        |      dkD  r:d|vr6t        j                  j	                         dkD  rt
        j                  d       t        | j                  d      rd| j                  j                  _
        yyz
        Some pre-processing hacks to make the model `accelerate` compatible. Check
        https://github.com/huggingface/transformers/pull/21707 for more details.
        r$   r  a  The `language_model` is not in the `hf_device_map` dictionary and you are running your script in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`. Please pass a `device_map` that contains `language_model` to remove this warning. Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for more details on creating a `device_map` for large models._hf_hookTNhf_device_maplenrG   cudadevice_countloggerwarningr&  r  r  io_same_devicerB   r  s     r3   _preprocess_acceleratez(InstructBlipModel._preprocess_accelerate  y    
 **}!&6m&KPUPZPZPgPgPilmPmNNM 4&&
3:>D((7 4r2   r  r  c                    |m| | j                         t        j                  | j                  j                  t        j
                  |j                              k(  }|j                  d      }n|| j                  j                  k(  }|j                  d      j                  |      j                  |j                        }|S zZ
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`.
        r~   r|  re   r   rG   r   rK   image_token_idlongr|  all	unsqueeze	expand_asr   rB   r  r  special_image_masks       r3   get_placeholder_maskz&InstructBlipModel.get_placeholder_mask       !.2M$2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*dkk.H.H!H/99"=GGVYYZgZnZno!!r2   Nqformer_input_idsqformer_attention_maskr   decoder_input_idsdecoder_attention_maskrz   r   r:   c
           	      *    | j                   d||	d|
}|d   }t        j                  |j                         dd t        j                  |j
                        }| j                  j                  |j                  d   dd      }t        j                  |j                         dd t        j                  |j
                        }|t        j                  |      }t        j                  ||gd      } | j                  d|||||d|
}|d   ddd|j                  d      ddf   }|7 | j                  j                         |      }|t        j                  |      }| j                  |      }|j                  |j
                  |j                         }| j#                  ||	      }|j%                  ||      }| j&                  j(                  r | j                  d||d
|
}n | j                  d||||d|
}t+        |||      S )a  
        qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
            to serve as text prompt, which the Q-Former model will encode.

            Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for
            details.

            [What are input IDs?](../glossary#input-ids)
        qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

            Only relevant in case an encoder-decoder language model (like T5) is used.
        )r{   rz   r   Nre   r  r$   rj   )r  r   r  rD  rE  r  r  r   )r  r   r  r  r=   r1   )r  rG   r  rg   r  r|  r   r   rl   	ones_likert   r  r  r   r  r   r~   r  masked_scatterrK   use_decoder_only_language_modelr6   )rB   r{   r  r  r  r   r  r  r  rz   r   r*   image_embedsimage_attention_maskr   query_attention_maskquery_outputsquery_outputlanguage_model_inputsr  outputss                        r3   r   zInstructBlipModel.forward  sP   P +** 
%%=
 

 &a(  %zz,*;*;*=cr*B%**]i]p]pq ((//0B0B10Er2N$zz,*;*;*=cr*B%**]i]p]pq!)%*__5F%G"!&,@BX+Y_`!a$ 
'1%".#7
 
 %Q'+A\->->q-A+A1(DE FD//DDFyQM%!&!; !% 8 8 F 5 8 89M9M}ObOb c!66yP]6^%445GI^_;;66)d)) +- G *d)) +-"3'=	
 G ?))#*
 	
r2   )NNNNNNF)r,   r-   r.   r!  _keep_in_fp32_modulesr%   rR   r   r  r  rG   r  rH   r  r   r   r   r   r   r   rD   r6   r   r   r   s   @r3   r   r     sR    %O+,1 :8?("e.>.> "uO`O` " 
 ;?.22659:>-1).^
''^
 !,,^
 !& 0 04 7	^

 $$t+^
 ((4/^
 !++d2^
 !& 0 04 7^
 ||d*^
 #'^
 -.^
 
@	@^
  ^
r2   r   a  
    InstructBLIP Model for generating text given an image and an optional text prompt. The model consists of a vision
    encoder, Querying Transformer (Q-Former) and a language model.

    One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue
    the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.
    c                       e Zd ZU eed<   dZdZdgZdef fdZd Z	d Z
d Zd	ej                  fd
Zd fd	Zd Zd Zee	 	 ddej*                  dej,                  dej,                  dz  dedz  dee   d	eez  fd              Zdej,                  dej*                  fdZee	 	 	 	 	 	 	 	 ddej*                  dej*                  dej,                  dz  dej*                  dz  dej,                  dz  dej,                  dz  dej,                  dz  dej*                  dz  dej,                  dz  dedee   d	eez  fd              Z ej@                         	 	 	 	 	 	 d dej*                  dej,                  dz  dej,                  dz  dej,                  dz  dej,                  dz  dej*                  dz  ded	ej,                  fd       Z! xZ"S )!r   rK   r{   Tr   c                    t         |   |       t        j                  |j                        | _        t        j                  t        j                  d|j                  |j                  j                              | _        t        j                  |j                        | _        t        j                   |j                  j                  |j"                  j                        | _        |j&                  r t)        j*                  |j"                        }nt-        j*                  |j"                        }|| _        | j1                          y r  )rQ   rR   r  _from_configr  r  r   rW   rG   r   r  r  rS   r   r  r  r   r  r  r  r"   r  r#   r  r  )rB   rK   r  r`   s      r3   rR   z-InstructBlipForConditionalGeneration.__init___  s     3@@AUAUVLLQ8O8OQWQfQfQrQr)st/<<V=R=RS#%99V-B-B-N-NPVPbPbPnPn#o 111==f>P>PQN2>>v?Q?QRN, 	r2   c                 6    | j                   j                         S r   r  rF   s    r3   r   z9InstructBlipForConditionalGeneration.get_input_embeddingss  r  r2   c                 :    | j                   j                  |       y r   r  r  s     r3   r  z9InstructBlipForConditionalGeneration.set_input_embeddingsv  r  r2   c                 :    | j                   j                  |       y r   )r  set_output_embeddings)rB   new_embeddingss     r3   r  z:InstructBlipForConditionalGeneration.set_output_embeddingsy  s    11.Ar2   r:   c                 6    | j                   j                         S r   )r  get_output_embeddingsrF   s    r3   r  z:InstructBlipForConditionalGeneration.get_output_embeddings|  s    ""88::r2   Nc                 \    || j                   j                         S t        |   |      S )N)modality)r  get_encoderrQ   )rB   r  r`   s     r3   r  z0InstructBlipForConditionalGeneration.get_encoder  s1    &&22447&&99r2   c                 6    | j                   j                         S r   )r  get_decoderrF   s    r3   r	  z0InstructBlipForConditionalGeneration.get_decoder  s    ""..00r2   c                    | j                   }t        |      dkD  r:d|vr6t        j                  j	                         dkD  rt
        j                  d       t        | j                  d      rd| j                  j                  _
        yyr  r  r  s     r3   r  z;InstructBlipForConditionalGeneration._preprocess_accelerate  r  r2   r  r  rz   r   c           
          | j                   d||dd|}t        di |d|i}|d   }t        j                  |j	                         dd t        j
                  |j                        }| j                  j                  |j                  d   dd      }	t        j                  |	j	                         dd t        j
                  |j                        }
|t        j                  |      }t        j                  |
|gd	      } | j                  d|||	||dd
|}||_        |d   ddd|	j	                  d      ddf   }| j                  |      }||_        |S )a  
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input images.
        qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
            to serve as text prompt, which the Q-Former model will encode.

            Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for
            details.

            [What are input IDs?](../glossary#input-ids)
        qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        T)r{   rz   return_dictr*   r   Nre   r  r$   rj   )r  r   r  rD  rE  r  r1   )r  r)   rG   r  rg   r  r|  r   r   rl   r  rt   r  r+   r  r  )rB   r{   r  r  rz   r   r*   r  r  r   r  r+   r  image_featuress                 r3   get_image_featuresz7InstructBlipForConditionalGeneration.get_image_features  s   > 6GT5F5F 6
%%=6
 	6
 Aq>qbpq%a(  %zz,*;*;*=cr*B%**]i]p]pq ((//0B0B10Er2N$zz,*;*;*=cr*B%**]i]p]pq!)%*__5F%G"!&,@BX+Y_`!a&$,, 
'1%".#7
 
 *9&&q)!-C|/@/@/C-CQ*FG 11,?'5$r2   r  r  c                    |m| | j                         t        j                  | j                  j                  t        j
                  |j                              k(  }|j                  d      }n|| j                  j                  k(  }|j                  d      j                  |      j                  |j                        }|S r  r  r  s       r3   r  z9InstructBlipForConditionalGeneration.get_placeholder_mask  r  r2   r   r  r  labelsc           	         | j                  ||||
d      }|j                  }|j                  }|j                  }| | j	                         |      }|t        j                  |      }|j                  |j                  |j                        }| j                  ||      }|j                  ||      }| j                  j                  rS | j                  d||d|}|d   }d}|	j | j                  d||	| j                  j                   j"                  d|}n5d|d<    | j                  d|||||	d	|}|j$                  }|j&                  }t)        |||||
      S )a  
        qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
            to serve as text prompt, which the Q-Former model will encode.

            Indices can be obtained using [`InstructBlipProcessor`]. See [`InstructBlipProcessor.__call__`] for
            details.

            [What are input IDs?](../glossary#input-ids)
        qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

            Only relevant in case an encoder-decoder language model (like T5) is used.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size -
            1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import InstructBlipProcessor, InstructBlipForConditionalGeneration
        >>> import torch
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> model = InstructBlipForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b")
        >>> processor = InstructBlipProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")

        >>> device = "cuda" if torch.cuda.is_available() else "cpu"
        >>> model.to(device)  # doctest: +IGNORE_RESULT

        >>> url = "https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read())).convert("RGB")
        >>> prompt = "What is unusual about this image?"
        >>> inputs = processor(images=image, text=prompt, return_tensors="pt").to(device)

        >>> outputs = model.generate(
        ...     **inputs,
        ...     do_sample=False,
        ...     num_beams=5,
        ...     max_length=256,
        ...     min_length=1,
        ...     top_p=0.9,
        ...     repetition_penalty=1.5,
        ...     length_penalty=1.0,
        ...     temperature=1,
        ... )
        >>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()
        >>> print(generated_text)
        The unusual aspect of this image is that a man is ironing clothes on the back of a yellow SUV, which is parked in the middle of a busy city street. This is an unconventional approach to ironing clothes, as it requires the man to balance himself and his ironing equipment on top of the vehicle while navigating through traffic. Additionally, the presence of taxis and other vehicles in the scene further emphasizes the unusual nature of this situation.
        ```Tr  r  rz   r  Nr  r  r   )r8   r  r  r  )r  r   r  r  r  )r7   r8   r*   r+   r9   r1   )r  r  r+   r*   r   rG   r  r   r|  r~   r  r  rK   r  r  loss_functionr  r  r7   r8   r6   )rB   r{   r  r  r  r   r  r  r  r  rz   r   r  r  r+   r*   r  r  r8   r7   s                       r3   r   z,InstructBlipForConditionalGeneration.forward  s   ^ CGBYBY/#9%= CZ C
 !/ < <(88'66 7D557	BM!"__Y7N 5 8 89M9M}ObOb c!66yP]6^%445GI^_;;66)d)) +- G
 QZFD!)t)) !&T[[=T=T=_=_ci
 %)F=!)d)) +-"3'= G <<D^^F>)+#*
 	
r2   c                 R   t        | d      r| j                          |j                  d   }	| j                  ||||d      }
|
j                  }||| j
                  j                  g| j
                  j                  z  }|| j
                  j                  j                  gz   }t        j                  |gt        j                  |j                        }|j                  |	d      } | j                         |      }|t        j                   |      }|j#                  |j                  |j$                        }| j'                  ||      }|j)                  ||      }||d}| j*                  j
                  j,                  s||d	<    | j*                  j.                  d
i ||}|S )a  
        Overrides `generate` function to be able to use the model as a conditional generator.

        Args:
            pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width)):
                Input images to be processed.
            qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                The sequence used as a prompt to be fed to the Q-Former module.
            qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                Mask to avoid performing attention on padding token indices.
            input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                The sequence used as a prompt for the generation.
            attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                Mask to avoid performing attention on padding token indices.
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Embedded representation of the inputs. Should be float, not int tokens.
            interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
                Whether to interpolate the positional encoding of the image embeddings.

        Returns:
            captions (list): A list of strings of length batch_size * num_captions.
        r  r   Tr  r  r$   r  r  r  r1   )r&  r  rl   r  r  rK   image_token_indexr  r  bos_token_idrG   r   r  r|  repeatr   r  r   r~   r  r  r  is_encoder_decodergenerate)rB   r{   r  r  r  r   r  rz   generate_kwargsr   r  r  image_tokensstart_tokensr  inputsr  s                    r3   r  z-InstructBlipForConditionalGeneration.generateu  s   D 4)'')!''*
BFBYBY/#9%= CZ C
 !/ < <   $ = =>A]A]]+t{{/F/F/S/S.TT!LL,uzzR^ReRef	%,,Z;	7D557	BM!"__Y7N 5 8 89M9M}ObOb c!66yP]6^%445GI^_#0NS""))<<"+F;.$%%..KK?Kr2   r   r  )NNNNNNNF)NNNNNF)#r,   r-   r.   r%   r0   r!  r  r  rR   r   r  r  r   Moduler  r  r	  r  r   r   rG   rH   r  r   r   r   rD   r)   r  r  r6   r   r  r  r   r   s   @r3   r   r   O  s    $O!+,1 (:8B;ryy ;:1?( 
 ;?05?''? !++? !& 0 04 7	?
 #'+? +,? 
8	8?  ?B"e.>.> "uO`O` " 
 ;?.22659:>26*.).B
''B
 !,,B
 !& 0 04 7	B

 $$t+B
 ((4/B
 !++d2B
 !& 0 04 7B
 ((4/B
   4'B
 #'B
 +,B
 
@	@B
  B
H U]]_ 6::>-12626).D''D !++d2D !& 0 04 7	D
 ##d*D ((4/D ((4/D #'D 
		D Dr2   r   )r  r   r   r   r  )r   )Sr/   r@  collections.abcr   dataclassesr   typingr   rG   r    r   r   activationsr	   
generationr
   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   r   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   utilsr   r   r   r   r   r   utils.genericr   utils.output_capturingr   r   autor!   r"   r#   configuration_instructblipr%   r&   r'   
get_loggerr,   r  r)   r6   r  rJ   r   floatr   r   r   r   r   r  r  r   r   r\  rc  ri  rm  r  r   r  r   r   __all__r1   r2   r3   <module>r3     s   "  $ !    & ! ) B 9  G & 6 j j 7 E I I o o 
		H	% 	P.H 	P  	P 
k 
 
<G299 Gd %II%<<% 
% <<	%
 LL4'% % %0G)BII G)Vbii  9 @ i/ i iD@")) @@39 3l^.BII ^.DBII  299  4bii  		 R9 Rl!
 !
H/BII /dY
: Y
x 
Z
3 Z

Z
z b+F bbJr2   