U 9%eP@sdZddlZddlmZmZmZmZmZddlZddl Zddlm Z ddl m Z m Z mZddlmZddlmZmZmZmZdd lmZdd lmZmZdd lmZmZmZmZm Z d d l!m"Z"e#e$Z%dZ&dZ'd dddgZ(dZ)dZ*ddddddgZ+dFe,e,ee,e,dddZ-Gddde j.Z/Gddde j.Z0Gdd d e j.Z1Gd!d"d"e j.Z2Gd#d$d$e j.Z3Gd%d&d&e j.Z4Gd'd(d(e j.Z5Gd)d*d*e j.Z6Gd+d,d,e j.Z7Gd-d.d.e j.Z8Gd/d0d0e j.Z9Gd1d2d2e j.Z:Gd3d4d4eZ;d5Zed:e<Gd;d<dd>e j.Z@Gd?d@d@e j.ZAGdAdBdBe j.ZBedCe<GdDdEdEe;ZCdS)Gz PyTorch MobileViT model.N)DictOptionalSetTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)BaseModelOutputWithNoAttention(BaseModelOutputWithPoolingAndNoAttention$ImageClassifierOutputWithNoAttentionSemanticSegmenterOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)MobileViTConfigrzapple/mobilevit-smalliztabby, tabby catzapple/mobilevit-x-smallzapple/mobilevit-xx-smallzapple/deeplabv3-mobilevit-smallz!apple/deeplabv3-mobilevit-x-smallz"apple/deeplabv3-mobilevit-xx-small)valuedivisor min_valuereturncCsF|dkr |}t|t||d||}|d|kr>||7}t|S)a Ensure that all layers have a channel count that is divisible by `divisor`. This function is taken from the original TensorFlow repo. It can be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py Ng?)maxint)rrr new_valuer$o/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/models/mobilevit/modeling_mobilevit.pymake_divisibleIs  r&c sTeZdZd eeeeeeeeeeeefdd fdd Ze j e j dd d Z Z S) MobileViTConvLayerrFTN) config in_channels out_channels kernel_sizestridegroupsbiasdilationuse_normalizationuse_activationrc stt|dd|} ||dkr@td|d|d||dkrbtd|d|dtj||||| |||dd |_| rtj|d d d d d |_nd|_| rt | t rt | |_ qt |j t rt |j |_ q|j |_ nd|_ dS)Nrr rzInput channels (z) are not divisible by z groups.zOutput channels (Zzeros) r)r*r+r,paddingr/r-r.Z padding_modegh㈵>g?T)Z num_featuresepsZmomentumZaffineZtrack_running_stats)super__init__r" ValueErrorrConv2d convolutionZ BatchNorm2d normalization isinstancestrr activation hidden_act) selfr(r)r*r+r,r-r.r/r0r1r2 __class__r$r%r5YsB         zMobileViTConvLayer.__init__featuresrcCs6||}|jdk r||}|jdk r2||}|SN)r8r9r<)r>rBr$r$r%forwards      zMobileViTConvLayer.forward)rrFrTT) __name__ __module__ __qualname__rr"boolrr;r5torchTensorrD __classcell__r$r$r?r%r'Xs( 6r'csFeZdZdZd eeeeeddfdd Zejejddd Z Z S) MobileViTInvertedResidualzQ Inverted residual block (MobileNetv2): https://arxiv.org/abs/1801.04381 rN)r(r)r*r,r/rc stttt||jd}|dkr:td|d|dkoH||k|_t|||dd|_ t|||d|||d|_ t|||dd d |_ dS) Nr)rr zInvalid stride .rr)r*r+r )r)r*r+r,r-r/Fr)r*r+r1) r4r5r&r"roundZ expand_ratior6 use_residualr' expand_1x1conv_3x3 reduce_1x1)r>r(r)r*r,r/Zexpanded_channelsr?r$r%r5s6  z"MobileViTInvertedResidual.__init__rAcCs4|}||}||}||}|jr0||S|SrC)rRrSrTrQ)r>rBresidualr$r$r%rDs    z!MobileViTInvertedResidual.forward)r) rErFrG__doc__rr"r5rIrJrDrKr$r$r?r%rLs!rLcsBeZdZd eeeeeddfdd ZejejdddZZ S) MobileViTMobileNetLayerrN)r(r)r*r, num_stagesrcsRtt|_t|D]0}t||||dkr4|ndd}|j||}qdS)Nrr)r)r*r,)r4r5r ModuleListlayerrangerLappend)r>r(r)r*r,rXirZr?r$r%r5s    z MobileViTMobileNetLayer.__init__rAcCs|jD] }||}q|SrCrZ)r>rB layer_moduler$r$r%rDs  zMobileViTMobileNetLayer.forward)rr rErFrGrr"r5rIrJrDrKr$r$r?r%rWsrWcsNeZdZeeddfdd ZejejdddZejejdd d Z Z S) MobileViTSelfAttentionNr( hidden_sizercst||jdkr2td|fd|jd|j|_t||j|_|j|j|_tj||j|j d|_ tj||j|j d|_ tj||j|j d|_ t |j|_dS)NrzThe hidden size z4 is not a multiple of the number of attention heads rM)r.)r4r5num_attention_headsr6r"attention_head_size all_head_sizerLinearZqkv_biasquerykeyrDropoutZattention_probs_dropout_probdropoutr>r(rcr?r$r%r5s zMobileViTSelfAttention.__init__)xrcCs6|dd|j|jf}|j|}|ddddS)Nrr rr )sizerdreviewpermute)r>rmZ new_x_shaper$r$r%transpose_for_scoress z+MobileViTSelfAttention.transpose_for_scores hidden_statesrc Cs||}|||}|||}||}t||dd}|t|j }t j j |dd}| |}t||}|dddd}|dd|jf} |j| }|S)Nrndimrr rr )rhrrrirrImatmul transposemathsqrtrer functionalZsoftmaxrkrq contiguousrorfrp) r>rtZmixed_query_layerZ key_layerZ value_layerZ query_layerZattention_scoresZattention_probsZ context_layerZnew_context_layer_shaper$r$r%rDs     zMobileViTSelfAttention.forward) rErFrGrr"r5rIrJrrrDrKr$r$r?r%rasracs:eZdZeeddfdd ZejejdddZZ S)MobileViTSelfOutputNrbcs*tt|||_t|j|_dSrCr4r5rrgdenserjZhidden_dropout_probrkrlr?r$r%r5s zMobileViTSelfOutput.__init__rscCs||}||}|SrCrrkr>rtr$r$r%rDs  zMobileViTSelfOutput.forwardr`r$r$r?r%r~sr~csNeZdZeeddfdd ZeeddddZej ej dd d Z Z S) MobileViTAttentionNrbcs.tt|||_t|||_t|_dSrC)r4r5ra attentionr~outputset pruned_headsrlr?r$r%r5s   zMobileViTAttention.__init__)headsrcCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrrv)lenrrrdrerrrhrirrrrfunion)r>rindexr$r$r% prune_heads%s zMobileViTAttention.prune_headsrscCs||}||}|SrC)rr)r>rtZ self_outputsattention_outputr$r$r%rD7s  zMobileViTAttention.forward) rErFrGrr"r5rrrIrJrDrKr$r$r?r%rsrcs<eZdZeeeddfdd ZejejdddZZ S)MobileViTIntermediateNr(rcintermediate_sizercs>tt|||_t|jtr2t|j|_ n|j|_ dSrC) r4r5rrgrr:r=r;r intermediate_act_fnr>r(rcrr?r$r%r5>s   zMobileViTIntermediate.__init__rscCs||}||}|SrC)rrrr$r$r%rDFs  zMobileViTIntermediate.forwardr`r$r$r?r%r=srcs@eZdZeeeddfdd ZejejejdddZZ S)MobileViTOutputNrcs*tt|||_t|j|_dSrCrrr?r$r%r5Ms zMobileViTOutput.__init__)rt input_tensorrcCs ||}||}||}|SrCr)r>rtrr$r$r%rDRs  zMobileViTOutput.forwardr`r$r$r?r%rLsrcs<eZdZeeeddfdd ZejejdddZZ S)MobileViTTransformerLayerNrcsZtt|||_t||||_t||||_tj ||j d|_ tj ||j d|_ dS)Nr3) r4r5rrr intermediaterrr LayerNormlayer_norm_epslayernorm_beforelayernorm_afterrr?r$r%r5Zs   z"MobileViTTransformerLayer.__init__rscCs<|||}||}||}||}|||}|SrC)rrrrr)r>rtrZ layer_outputr$r$r%rDbs    z!MobileViTTransformerLayer.forwardr`r$r$r?r%rYsrcs<eZdZeeeddfdd ZejejdddZZ S)MobileViTTransformerN)r(rcrXrcsJtt|_t|D](}t||t||jd}|j |qdS)N)rcr) r4r5rrYrZr[rr"Z mlp_ratior\)r>r(rcrX_transformer_layerr?r$r%r5ms    zMobileViTTransformer.__init__rscCs|jD] }||}q|SrCr^)r>rtr_r$r$r%rDys  zMobileViTTransformer.forwardr`r$r$r?r%rls rc s|eZdZdZdeeeeeeeddfdd Zeje eje fddd Z eje ejd d d Z ejejdd dZ ZS)MobileViTLayerz; MobileViT block: https://arxiv.org/abs/2110.02178 rN)r(r)r*r,rcrXr/rcst|j|_|j|_|dkrXt||||dkr6|nd|dkrH|dndd|_|}nd|_t||||jd|_ t|||dddd|_ t |||d|_ t j||jd|_t|||dd|_t|d|||jd|_dS) Nr r)r)r*r,r/rNF)r)r*r+r0r1)rcrXr)r4r5Z patch_size patch_width patch_heightrLdownsampling_layerr'Zconv_kernel_sizeconv_kxkconv_1x1r transformerrrr layernormconv_projectionfusion)r>r(r)r*r,rcrXr/r?r$r%r5sZ  zMobileViTLayer.__init__rAcCs|j|j}}t||}|j\}}}}tt|||} tt|||} d} | |ksh| |krtjj|| | fddd}d} | |} | |} | | }| ||| || |}| dd}| ||||}| dd}| |||d}||f||| || | d }||fS) NFbilinearromodeZ align_cornersTrr r rn) orig_size batch_sizechannels interpolate num_patchesnum_patches_widthnum_patches_height) rrr"shaperzceilrr|rreshapery)r>rBrr patch_arearrZ orig_heightZ orig_widthZ new_heightZ new_widthrnum_patch_widthnum_patch_heightrpatches info_dictr$r$r% unfoldingsH     zMobileViTLayer.unfolding)rrrc Cs|j|j}}t||}|d}|d}|d}|d} |d} ||||d} | dd} | ||| | ||} | dd } | ||| || |} |d rtjj | |d d d d} | S)Nrrrrrrnrr r rrrFr) rrr"r}rpryrrr|r) r>rrrrrrrrrrrBr$r$r%foldings<    zMobileViTLayer.foldingcCs||jr||}|}||}||}||\}}||}||}|||}||}|t j ||fdd}|SNrrv) rrrrrrrrrrIcat)r>rBrUrrr$r$r%rDs       zMobileViTLayer.forward)r)rErFrGrVrr"r5rIrJrrrrrDrKr$r$r?r%rs :+rcsDeZdZeddfdd Zd ejeeee e fddd Z Z S) MobileViTEncoderNr(rc sZt||_t|_d|_d}}|jdkrr(Zdilate_layer_4Zdilate_layer_5r/Zlayer_1Zlayer_2Zlayer_3Zlayer_4Zlayer_5r?r$r%r5!sx         zMobileViTEncoder.__init__FT)rtoutput_hidden_states return_dictrcCs|rdnd}t|jD]H\}}|jrH|jrHdd}tjj|||}n||}|r||f}q|sztdd||fDSt||dS)Nr$csfdd}|S)Ncs|SrCr$)inputsmoduler$r%custom_forwardwszOMobileViTEncoder.forward..create_custom_forward..custom_forwardr$)rrr$rr%create_custom_forwardvs z7MobileViTEncoder.forward..create_custom_forwardcss|]}|dk r|VqdSrCr$).0vr$r$r% sz+MobileViTEncoder.forward..)last_hidden_statert) enumeraterZrZtrainingrIutils checkpointtupler )r>rtrrZall_hidden_statesr]r_rr$r$r%rDks   zMobileViTEncoder.forward)FT) rErFrGrr5rIrJrHrrr rDrKr$r$r?r%r sM rc@sJeZdZdZeZdZdZdZe e j e j e j fddddZd d d ZdS) MobileViTPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. mobilevit pixel_valuesTN)rrcCsjt|tjtjfr@|jjjd|jjd|j dk rf|j j n&t|tj rf|j j |jj ddS)zInitialize the weightsg)meanZstdNg?) r:rrgr7weightdataZnormal_r(Zinitializer_ranger.Zzero_rZfill_)r>rr$r$r% _init_weightss   z&MobileViTPreTrainedModel._init_weightsFcCst|tr||_dSrC)r:rr)r>rrr$r$r%_set_gradient_checkpointings z4MobileViTPreTrainedModel._set_gradient_checkpointing)F)rErFrGrVr config_classZbase_model_prefixZmain_input_nameZsupports_gradient_checkpointingrrrgr7rrrr$r$r$r%rs raK This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`MobileViTConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. aF Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MobileViTImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zWThe bare MobileViT model outputting raw hidden-states without any specific head on top.c steZdZd eedfdd ZddZeee e e e de ddeejeeeeeee fd d d ZZS)MobileViTModelT)r( expand_outputcsnt|||_||_t||j|jdddd|_t||_ |jrbt||jd|jddd|_ | dS) Nrr r )r)r*r+r,rrrN) r4r5r(rr'Z num_channelsr conv_stemrencoder conv_1x1_exp post_init)r>r(rr?r$r%r5s&  zMobileViTModel.__init__cCsF|D]8\}}|jj|}t|tr|jjD]}|j|q.qdS)zPrunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrZr:rrrr)r>Zheads_to_pruneZ layer_indexrZmobilevit_layerrr$r$r% _prune_headss    zMobileViTModel._prune_headsZvision)r output_typerZmodalityexpected_outputN)rrrrc Cs|dk r |n|jj}|dk r |n|jj}|dkr8td||}|j|||d}|jr|||d}tj |ddgdd}n |d}d}|s|dk r||fn|f}||ddSt |||j d S) Nz You have to specify pixel_valuesrrrrurnF)rwZkeepdimr)r pooler_outputrt) r(ruse_return_dictr6rrrrrIrrrt) r>rrrZembedding_outputZencoder_outputsr pooled_outputrr$r$r%rDs0 zMobileViTModel.forward)T)NNN)rErFrGrrHr5rrMOBILEVIT_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPErrIrJrrrDrKr$r$r?r%rs&   rz MobileViT model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. c speZdZeddfdd Zeeeee e e dd e e je ee e je eeee fdddZZS) MobileViTForImageClassificationNrcsdt||j|_t||_tj|jdd|_|jdkrNt |j d|jnt |_ | dS)NT)Zinplacerrn)r4r5 num_labelsrrrrjclassifier_dropout_probrkrgrZIdentity classifierrr>r(r?r$r%r5#s  $z(MobileViTForImageClassification.__init__)rrrr)rrlabelsrrc Csr|dk r |n|jj}|j|||d}|r.|jn|d}|||}d}|dk r2|jjdkr|jdkrtd|j_n4|jdkr|jt j ks|jt j krd|j_nd|j_|jjdkrt } |jdkr| | | }n | ||}nN|jjdkrt} | |d|j|d}n|jjdkr2t} | ||}|sb|f|dd} |dk r^|f| S| St|||jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NrrZ regressionZsingle_label_classificationZmulti_label_classificationrnr )losslogitsrt)r(rrrrrkZ problem_typerZdtyperIlongr"r Zsqueezer rprrrt) r>rrrroutputsrrrloss_fctrr$r$r%rD2s>    "     z'MobileViTForImageClassification.forward)NNNN)rErFrGrr5rrr_IMAGE_CLASS_CHECKPOINTrr_IMAGE_CLASS_EXPECTED_OUTPUTrrIrJrHrrrDrKr$r$r?r%rs& rcs<eZdZeeeddfdd ZejejdddZZ S)MobileViTASPPPoolingN)r(r)r*rc s4ttjdd|_t|||ddddd|_dS)Nr)Z output_sizeTrelu)r)r*r+r,r0r1)r4r5rZAdaptiveAvgPool2d global_poolr'r)r>r(r)r*r?r$r%r5qs zMobileViTASPPPooling.__init__rAcCs:|jdd}||}||}tjj||ddd}|S)NrurFr)rrrrr|r)r>rBZ spatial_sizer$r$r%rDs   zMobileViTASPPPooling.forwardr`r$r$r?r%rpsrcs<eZdZdZeddfdd ZejejdddZZ S) MobileViTASPPzs ASPP module defined in DeepLab papers: https://arxiv.org/abs/1606.00915, https://arxiv.org/abs/1706.05587 Nrcstjdjtjdkr0tdt|_ t ddd}|j ||j fddjDt }|j |t d ddd|_tjjd |_dS) Nrur z"Expected 3 values for atrous_ratesrrrOc s g|]}td|ddqS)r r)r)r*r+r/r1)r')rZrater(r)r*r$r% s z*MobileViTASPP.__init__..r)p)r4r5raspp_out_channelsrZ atrous_ratesr6rrYconvsr'r\extendrprojectrjZaspp_dropout_probrk)r>r(Z in_projectionZ pool_layerr?rr%r5s:       zMobileViTASPP.__init__rAcCsDg}|jD]}|||q tj|dd}||}||}|Sr)rr\rIrrrk)r>rBZpyramidconvZpooled_featuresr$r$r%rDs   zMobileViTASPP.forward rErFrGrVrr5rIrJrDrKr$r$r?r%rs+rcs<eZdZdZeddfdd ZejejdddZZ S) MobileViTDeepLabV3zB DeepLabv3 architecture: https://arxiv.org/abs/1706.05587 Nrc sBtt||_t|j|_t||j |j ddddd|_ dS)NrFT)r)r*r+r0r1r.) r4r5raspprZ Dropout2drrkr'rrrrr?r$r%r5s  zMobileViTDeepLabV3.__init__rscCs&||d}||}||}|S)Nrn)r rkr)r>rtrBr$r$r%rDs  zMobileViTDeepLabV3.forwardr r$r$r?r%r sr zX MobileViT model with a semantic segmentation head on top, e.g. for Pascal VOC. c sleZdZeddfdd Zeeeee dd e e j e e j e e e e eeefdddZZS) MobileViTForSemanticSegmentationNrcs8t||j|_t|dd|_t||_|dS)NF)r)r4r5rrrr segmentation_headrrr?r$r%r5s   z)MobileViTForSemanticSegmentation.__init__)rr)rrrrrc Cs |dk r |n|jj}|dk r |n|jj}|j|d|d}|rB|jn|d}||}d}|dk r|jjdkrvtdn6tj j ||j ddddd } t |jj d } | | |}|s|r|f|dd} n|f|d d} |dk r|f| S| St|||r|jnddd S) a labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). Returns: Examples: ```python >>> import requests >>> import torch >>> from PIL import Image >>> from transformers import AutoImageProcessor, MobileViTForSemanticSegmentation >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> image_processor = AutoImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-small") >>> model = MobileViTForSemanticSegmentation.from_pretrained("apple/deeplabv3-mobilevit-small") >>> inputs = image_processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) >>> # logits are of shape (batch_size, num_labels, height, width) >>> logits = outputs.logits ```NTrrz/The number of labels should be greater than onerurFr)Z ignore_indexr )rrrtZ attentions)r(rrrrtrrr6rr|rrr Zsemantic_loss_ignore_indexr) r>rrrrrZencoder_hidden_statesrrZupsampled_logitsrrr$r$r%rDsB'     z(MobileViTForSemanticSegmentation.forward)NNNN)rErFrGrr5rrrrrrrIrJrHrrrDrKr$r$r?r%r s   r )rN)DrVrztypingrrrrrrIZtorch.utils.checkpointrZtorch.nnrr r Z activationsr Zmodeling_outputsr rrrZmodeling_utilsrZ pytorch_utilsrrrrrrrrZconfiguration_mobilevitrZ get_loggerrEloggerrrrrrZ'MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LISTr"r&Moduler'rLrWrar~rrrrrrrrZMOBILEVIT_START_DOCSTRINGrrrrrr r r$r$r$r%sx       @03  "l  WN;