i dZddlZddlmZddlmZddlmZddlm Z ddl m Z dd l m Z mZmZmZdd lmZdd lmZmZd d lmZej0eZdZ Gddej>Z!Gddej>Z"Gddej>Z#Gd d!ej>Z$Gd"d#ej>Z%Gd$d%ej>Z&Gd&d'e Z'Gd(d)ej>Z(eGd*d+eZ)eGd,d-e)Z*ed./Gd0d1e)Z+Gd2d3ej>Z,Gd4d5ej>Z-Gd6d7ej>Z.ed8/Gd9d:e)Z/gd;Z0y)=zPyTorch MobileViTV2 model.N)nn)CrossEntropyLoss)initialization)ACT2FN)GradientCheckpointingLayer)BaseModelOutputWithNoAttention(BaseModelOutputWithPoolingAndNoAttention$ImageClassifierOutputWithNoAttentionSemanticSegmenterOutput)PreTrainedModel)auto_docstringlogging)MobileViTV2Configvaluedivisor min_valuereturnc|||}t|t||dz z|z|z}|d|zkr||z }t|S)zU Ensure that all layers have a channel count that is divisible by `divisor`. g?)maxint)rrr new_values p/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/transformers/models/mobilevitv2/modeling_mobilevitv2.pymake_divisibler(sS Is57Q;#677BWLMI3;W y>z-infinfmin_valmax_valc.t|t||SN)rminrrr s rclipr%5s wGU+ ,,rceZdZ ddedededededededed ed eezd d ffd Zdejd ejfdZ xZ S)MobileViTV2ConvLayerconfig in_channels out_channels kernel_sizestridegroupsbiasdilationuse_normalizationuse_activationrNc $t |t|dz dz |z} ||zdk7rtd|d|d||zdk7rtd|d|dt j ||||| |||d |_| r t j|d d d d |_nd|_| rdt| trt| |_ yt|jtrt|j|_ y|j|_ yd|_ y)NrrrzInput channels (z) are not divisible by z groups.zOutput channels (zeros) r)r*r+r,paddingr/r-r. padding_modegh㈵>g?T) num_featuresepsmomentumaffinetrack_running_stats)super__init__r ValueErrorrConv2d convolution BatchNorm2d normalization isinstancestrr activation hidden_act) selfr(r)r*r+r,r-r.r/r0r1r4 __class__s rr<zMobileViTV2ConvLayer.__init__;s* {Q!+,x7  1 $/ }UV\U]]efg g99#%#    !#)$( "D "&D  .#."("8F--s3"():):";"("3"3"DOrfeaturesc|j|}|j|j|}|j|j|}|Sr")r?rArD)rFrHs rforwardzMobileViTV2ConvLayer.forwardqsK##H-    )))(3H ?? &x0Hr)rrFrTT) __name__ __module__ __qualname__rrboolrCr<torchTensorrJ __classcell__rGs@rr'r':s"&%)4#!4#4# 4#  4#  4#4#4#4# 4#s 4# 4#l rr'c xeZdZdZ d dedededededdf fd Zd ejdejfd Z xZ S) MobileViTV2InvertedResidualzY Inverted residual block (MobileNetv2): https://huggingface.co/papers/1801.04381 r(r)r*r,r/rNc @t|ttt ||j zd}|dvrt d|d|dk(xr||k(|_t|||d|_ t|||d||||_ t|||dd |_ y) N)rrzInvalid stride .r)r)r*r+r)r)r*r+r,r-r/Fr)r*r+r1) r;r<rrround expand_ratior= use_residualr' expand_1x1conv_3x3 reduce_1x1)rFr(r)r*r,r/expanded_channelsrGs rr<z$MobileViTV2InvertedResidual.__init__s *3u[6CVCV5V/W+XZ[\  vha89 9#q[K{l/J.  :KYZ - )*$  / )%  rrHc|}|j|}|j|}|j|}|jr||zS|Sr")r\r]r^r[)rFrHresiduals rrJz#MobileViTV2InvertedResidual.forwardsI??8,==*??8,&*&7&7x("EXEr)r rKrLrM__doc__rrr<rOrPrJrQrRs@rrTrT{sc lm ' 69 IL VY eh  BF FFrrTc teZdZ d dedededededdf fd Zd ejdejfd ZxZ S) MobileViTV2MobileNetLayerr(r)r*r, num_stagesrNct|tj|_t |D]5}t ||||dk(r|nd}|jj||}7y)Nrr)r)r*r,)r;r<r ModuleListlayerrangerTappend) rFr(r)r*r,rfirirGs rr<z"MobileViTV2MobileNetLayer.__init__sh ]]_ z" 'A/')!"avQ E JJ  e $&K 'rrHc8|jD] }||} |Sr"ri)rFrH layer_modules rrJz!MobileViTV2MobileNetLayer.forwards$ JJ .L#H-H .r)rr rKrLrMrrr<rOrPrJrQrRs@rreresVqr'''69'IL'VY'kn' '  rrecheZdZdZdededdffd ZdejdejfdZ xZ S) MobileViTV2LinearSelfAttentionay This layer applies a self-attention with linear complexity, as described in MobileViTV2 paper: https://huggingface.co/papers/2206.02680 Args: config (`MobileVitv2Config`): Model configuration object embed_dim (`int`): `input_channels` from an expected input of size :math:`(batch_size, input_channels, height, width)` r( embed_dimrNc t|t||dd|zzdddd|_t j |j |_t|||dddd|_||_y)NrrTF)r(r)r*r.r+r0r1p) r;r<r'qkv_projrDropout attn_dropoutout_projrs)rFr(rsrGs rr<z'MobileViTV2LinearSelfAttention.__init__s{ ,!a)m,#   JJ)<)<=,!"#   #r hidden_statesc|j|}tj|d|j|jgd\}}}tjj j |d}|j|}||z}tj|dd}tjj j||j|z}|j|}|S)Nr)split_size_or_sectionsdimr~Tr~keepdim) rwrOsplitrsr functionalsoftmaxrysumrelu expand_asrz) rFr{qkvquerykeyrcontext_scorescontext_vectorouts rrJz&MobileViTV2LinearSelfAttention.forwardsmmM* "KKQX\XfXfDgmnosE,,44U4C**>:~->r4Hhh!!&&u-0H0H0OOmmC  rrbrRs@rrrrrs> #0#S#T#2U\\ellrrrc peZdZ d dededededdf fd Zdejdejfd Z xZ S) MobileViTV2FFNr(rsffn_latent_dim ffn_dropoutrNc t|t|||ddddd|_t j ||_t|||ddddd|_t j ||_y)NrTF)r(r)r*r+r,r.r0r1) r;r<r'conv1rrxdropout1conv2dropout2)rFr(rsrrrGs rr<zMobileViTV2FFN.__init__s| )!'#    ;/ )&"#     ;/ rr{c|j|}|j|}|j|}|j|}|Sr")rrrr)rFr{s rrJzMobileViTV2FFN.forward#s@ =1  m4  =1  m4 r rKrLrMrrfloatr<rOrPrJrQrRs@rrrsY ! 0!00 0  0  0@U\\ellrrc peZdZ d dededededdf fd Zdejdejfd Z xZ S) MobileViTV2TransformerLayerr(rsrdropoutrNcPt|tjd||j|_t |||_tj||_ tjd||j|_ t||||j|_ y)Nr num_groups num_channelsr7ru)r;r<r GroupNormlayer_norm_epslayernorm_beforerr attentionrxrlayernorm_afterrrffn)rFr(rsrrrGs rr<z$MobileViTV2TransformerLayer.__init__,s~  "  W]WlWl m7 J W- !||qyV\VkVkl!&)^VEWEWXrr{c|j|}|j|}||z}|j|}|j|}||z}|Sr")rrrr)rFr{layernorm_1_outattention_output layer_outputs rrJz#MobileViTV2TransformerLayer.forward:sY// >>>/:(=8 ++M: xx - #m3 rrrrRs@rrr+s^  Y! Y Y Y  Y  Y U\\ ell rrcheZdZdedededdffd ZdejdejfdZxZ S) MobileViTV2Transformerr(n_layersd_modelrNc<t ||j}||zg|z}|Dcgc]}t|dzdz}}t j |_t|D].}t||||}|j j|0ycc}w)N)rsr) r;r<ffn_multiplierrrrhrirjrrk) rFr(rrrffn_dimsd block_idxtransformer_layerrGs rr<zMobileViTV2Transformer.__init__Gs .."W,-82::ACbB'::]]_ x 1I ;'(9:M!  JJ  / 0  1;sBr{c8|jD] }||} |Sr"rn)rFr{ros rrJzMobileViTV2Transformer.forwardXs% JJ 8L(7M 8rrprRs@rrrFsA101C1#1RV1"U\\ellrrceZdZdZ ddededededededed d ffd Zd ejd e eje eefffd Z dejde eefd ejfdZ dejd ejfdZ xZ S)MobileViTV2LayerzE MobileViTV2 layer: https://huggingface.co/papers/2206.02680 r(r)r* attn_unit_dim n_attn_blocksr/r,rNct ||j|_|j|_|}|dk(r)t ||||dk(r|nd|dkDr|dznd|_|}nd|_t||||j||_ t|||ddd|_ t||||_ tjd||j|_t|||dd d|_y) Nrr)r)r*r,r/)r)r*r+r-F)r)r*r+r0r1)rrrT)r;r< patch_size patch_width patch_heightrTdownsampling_layerr'conv_kernel_sizeconv_kxkconv_1x1r transformerrrr layernormconv_projection) rFr(r)r*rrr/r, cnn_out_dimrGs rr<zMobileViTV2Layer.__init__cs  !,,"--# Q;&A')!)QvA*2Q,QA 'D #'K&*D #- #$//   - #$#   2&-ZghTZTiTij 4 #$"  r feature_mapc"|j\}}}}tjj||j|j f|j|j f}|j |||j|j zd}|||ffS)N)r+r,r)shaperrunfoldrrreshape)rFr batch_sizer) img_height img_widthpatchess r unfoldingzMobileViTV2Layer.unfoldings9D9J9J6 KY--&& **D,<,<=%%t'7'78'  //*k4;L;LtO_O_;_acdY///rr output_sizec|j\}}}}|j|||z|}tjj |||j |j f|j |j f}|S)N)rr+r,)rrrrfoldrr)rFrrrin_dimr n_patchesrs rfoldingzMobileViTV2Layer.foldingsz4;MM1 FJ //*fz.A9Mmm(( #**D,<,<=%%t'7'78 ) rrHc6|jr|j|}|j|}|j|}|j|\}}|j |}|j |}|j ||}|j|}|Sr")rrrrrrrr)rFrHrrs rrJzMobileViTV2Layer.forwards  " "..x8H==*==* $~~h7""7+..)<<5''1r)rrr)rKrLrMrcrrr<rOrPtuplerrrJrQrRs@rrr^s; !; ;  ;  ;  ; ; ;  ; z 0U\\ 0eELL%PSUXPX/z-MobileViTV2Encoder.forward..1sXq!-Xs)last_hidden_stater{) enumeraterirr )rFr{rrall_hidden_statesrlros rrJzMobileViTV2Encoder.forward"sq #7BD(4 IOA|(7M#$58H$H!  I X]4E$FXX X- ]noor)FT) rKrLrMrr<rOrPrNrr rJrQrRs@rrrs]O#0O#TO#h&+ p||p#p p / / prrczeZdZUeed<dZdZdZdZdgZ e jde jdd fd Zy ) MobileViTV2PreTrainedModelr( mobilevitv2 pixel_values)imageTrmodulerNct|tjtjtjfrt j |jd|jj|jt j|jt|dd^t j|jt j|jt j|j yyt|tj"r?t j|jt j|jyy)zInitialize the weightsr)meanstdN running_mean)rBrLinearr>r@initnormal_weightr(initializer_ranger.zeros_getattrr ones_ running_varnum_batches_trackedr)rFrs r _init_weightsz(MobileViTV2PreTrainedModel._init_weights?s fryy"))R^^D E LLSdkk6S6S T{{& FKK(v~t4@ F//0 6--. F667A - KK $ JJv}} %.r)rKrLrMr__annotations__base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modulesrOno_gradrModulerrrrrr6sT %$O!&*#+,U]]_ &BII &$ & &rrc eZdZd dedeffd Ze d dejdzdedzdedzde e zfd Z xZ S) MobileViTV2Modelr( expand_outputc t||||_||_t t d|j zdddd}t||j|ddd d |_ t||_ |jy ) a expand_output (`bool`, *optional*, defaults to `True`): Whether to expand the output of the model. If `True`, the model will output pooled features in addition to hidden states. If `False`, only the hidden states will be returned. rrrr$rVrrrTr)r*r+r,r0r1N) r;r<r(r!rr%rr'r conv_stemrencoder post_init)rFr(r!rrGs rr<zMobileViTV2Model.__init__Qs   *$ rF333R LVWce . ++$" *&1  rNrrrrc ||n|jj}||n|jj}| td|j |}|j |||}|j r |d}tj|ddgd}n|d}d}|s|||fn|f} | |ddzSt|||j S) Nz You have to specify pixel_valuesrrrrFrr)r pooler_outputr{) r(ruse_return_dictr=r$r%r!rOr r r{) rFrrrkwargsembedding_outputencoder_outputsr pooled_outputoutputs rrJzMobileViTV2Model.forwardms%9$D $++JjJj &1%>,7,, !5#'     / 2 "JJ'8r2hPUVM / 2  M;H;T'7[lZnFOAB// /7/')77  r)T)NNN) rKrLrMrrNr<rrOrPrr rJrQrRs@rr r Oss08-1,0#' ( llT)( #Tk( D[ ( 9 9 ( ( rr z MobileViTV2 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. ) custom_introc eZdZdeddffd Ze d dejdzdedzdejdzdedzde e zf d Z xZ S) !MobileViTV2ForImageClassificationr(rNcLt|||j|_t||_t d|j zd}|jdkDr!tj||jntj|_ |jy)NrrVrr) in_features out_features) r;r< num_labelsr rrrrr Identity classifierr&)rFr(r*rGs rr<z*MobileViTV2ForImageClassification.__init__s   +++F3%cF,C,C&CQO   1$ II,V=N=N O  rrrlabelsrc B||n|jj}|j|||}|r |jn|d}|j |}d} ||j |||j} |s|f|ddz} | | f| zS| St | ||jS)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr(rr)losslogitsr{)r(r+rr*r9 loss_functionr r{) rFrrr:rr,outputsr/r=r<r0s rrJz)MobileViTV2ForImageClassification.forwards&1%, #%"!  rrHc|jdd}|j|}|j|}tjj ||dd}|S)Nr)bilinearFsizemode align_corners)rrErrr interpolate)rFrH spatial_sizes rrJzMobileViTV2ASPPPooling.forwardsS~~bc* ##H-==*==,,XLzin,orrprRs@rrBrBsB  0  s  RU  Z^   rrBcdeZdZdZdeddffd ZdejdejfdZxZ S)MobileViTV2ASPPz ASPP module defined in DeepLab papers: https://huggingface.co/papers/1606.00915, https://huggingface.co/papers/1706.05587 r(rNc t|td|jzd}|}|j}t |j dk7r tdtj|_ t|||dd}|jj||jj|j Dcgc]}t|||d|d c}t|||}|jj|t|d |z|dd|_tj |j" |_ycc}w) NrrVrrz"Expected 3 values for atrous_ratesrrrX)r)r*r+r/r1ru)r;r<rraspp_out_channelslen atrous_ratesr=rrhconvsr'rkextendrBprojectrxaspp_dropout_probr) rFr(encoder_out_channelsr)r* in_projectionrate pool_layerrGs rr<zMobileViTV2ASPP.__init__s6 -cF4K4K.KUVW* // v"" #q (AB B]]_ , #%!   -( #// % +!- !!#)   ,FKN  *%+ L 0|YZkq  zzF$<$<= ) sErHcg}|jD]}|j||tj|d}|j |}|j |}|S)Nrr)rUrkrOcatrWr)rFrHpyramidconvpooled_featuress rrJzMobileViTV2ASPP.forward!s\JJ +D NN4> * +))G+,,w/,,7r rKrLrMrcrr<rOrPrJrQrRs@rrOrOs8*>0*>T*>X rrOcdeZdZdZdeddffd ZdejdejfdZxZ S)MobileViTV2DeepLabV3zJ DeepLabv3 architecture: https://huggingface.co/papers/1706.05587 r(rNc t|t||_t j |j |_t||j|jdddd|_ y)NrFT)r)r*r+r0r1r.) r;r<rOasppr Dropout2dclassifier_dropout_probrr'rRr7r9rFr(rGs rr<zMobileViTV2DeepLabV3.__init__2s] #F+ ||F$B$BC . 00**#  rr{cr|j|d}|j|}|j|}|S)Nr)rfrr9)rFr{rHs rrJzMobileViTV2DeepLabV3.forwardBs699]2./<<)??8,rrbrRs@rrdrd-s7 0 T U\\ellrrdzZ MobileViTV2 model with a semantic segmentation head on top, e.g. for Pascal VOC. c eZdZdeddffd Ze d dejdzdejdzdedzdedzde e zf d Z xZ S) "MobileViTV2ForSemanticSegmentationr(rNct|||j|_t|d|_t ||_|jy)NF)r!)r;r<r7r rrdsegmentation_headr&ris rr<z+MobileViTV2ForSemanticSegmentation.__init__OsE   +++F%H!5f!= rrr:rrc h||n|jj}||n|jj}|$|jjdk(r t d|j |d|}|r |j n|d}|j|}d} |Ytjj||jdddd } t|jj } | | |} |s|r |f|ddz} n |f|d dz} | | f| zS| St| ||r|j d Sdd S) a labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). Examples: ```python >>> import httpx >>> from io import BytesIO >>> import torch >>> from PIL import Image >>> from transformers import AutoImageProcessor, MobileViTV2ForSemanticSegmentation >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> image_processor = AutoImageProcessor.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") >>> model = MobileViTV2ForSemanticSegmentation.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") >>> inputs = image_processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) >>> # logits are of shape (batch_size, num_labels, height, width) >>> logits = outputs.logits ```Nrz/The number of labels should be greater than oneTr(r)rGFrH) ignore_indexr)r<r=r{ attentions)r(rr+r7r=rr{rnrrrLrrsemantic_loss_ignore_indexr ) rFrr:rrr,r?encoder_hidden_statesr=r<upsampled_logitsloss_fctr0s rrJz*MobileViTV2ForSemanticSegmentation.forwardYssN%9$D $++JjJj &1%  !}}88V\\"#.Zu 9  (T[[5[5[\H,f5D# WQR[0 WQR[0)-)9TGf$ Ev E&3G'//   NR   rr@) rKrLrMrr<rrOrPrNrr rJrQrRs@rrlrlIs 0T-1&*,0#' L llT)L  t#L #Tk L D[ L  ( (L L rrl)r3rlr r)rVN)1rcrOrtorch.nnrrr  activationsrmodeling_layersrmodeling_outputsr r r r modeling_utilsr utilsrrconfiguration_mobilevitv2r get_loggerrKloggerrrrr%rr'rTrerrrrrrrrr r3rBrOrdrl__all__rrrrs ! %&!9 .,8   H % #  C$J RU ).f e ----Y^- =299=B-F"))-Fb .<RYY<~&RYY&R"))6RYY0o1odcpcpL&&&0F 1F F R 5 (B5  5 rRYY09bii9z2998 X )CX  X v r