U 9%eo@sdZddlZddlZddlmZmZmZmZddlZddl m m Z ddl Zddlm Z ddlmZmZmZddlmZddlmZmZdd lmZdd lmZmZdd lmZmZmZm Z d d l!m"Z"e #e$Z%dZ&dZ'd ddgZ(dZ)dZ*dgZ+d2ej,e-e.ej,dddZ/Gddde j0Z1Gddde j0Z2Gddde j0Z3Gddde j0Z4Gd d!d!e j0Z5Gd"d#d#e j0Z6Gd$d%d%e j0Z7Gd&d'd'e j0Z8Gd(d)d)eZ9d*Z:d+Z;ed,e:Gd-d.d.e9Z isinstance collectionsabcrr?r@rB num_patchesr Parameterr!Zrandnposition_embeddingsZzerosrDConv2d projection LayerNormlayer_norm_eps layer_normDropouthidden_dropout_probdropout) r+r>r?r@rArBrCrDrKr,r$r%r*es  zPvtPatchEmbeddings.__init__) embeddingsheightwidthrcCsr||}||jj|jjkr"|jS|d||ddddd}tj|||fdd}|dd||ddd}|S)Nrrr Zbilinear)sizemode)r>r?rMreshapepermuteFZ interpolate)r+rVrWrXrKZinterpolated_embeddingsr$r$r%interpolate_pos_encodingsz+PvtPatchEmbeddings.interpolate_pos_encoding) pixel_valuesrc Cs|j\}}}}||jkr td||}|j^}}}|ddd}||}|jdk r|j|dd} t j | |fdd}| |j ddddf||} t j |j ddddf| fdd} n| |j ||} | || }|||fS)NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rZrrYdim)rrB ValueErrorrOflatten transposerRrDexpandr!catr`rMrU) r+ra batch_sizerBrWrXZ patch_embed_rVrDrMr$r$r%r1s"     &zPvtPatchEmbeddings.forward)F)r4r5r6r7rrintrboolr*r!r9r`rr1r;r$r$r,r%r<^s r<cs8eZdZeedfdd ZejejdddZZ S) PvtSelfOutput)r>rCcs*tt|||_t|j|_dSr()r)r*rLineardenserSrTrU)r+r>rCr,r$r%r*s zPvtSelfOutput.__init__r.cCs||}||}|Sr()rorUr0r$r$r%r1s  zPvtSelfOutput.forward) r4r5r6rrkr*r!r9r1r;r$r$r,r%rmsrmcs^eZdZdZeeeedfdd Zeej dddZ d ej eee e ej d d d Z ZS) PvtEfficientSelfAttentionzpEfficient self-attention mechanism with reduction of the sequence [PvT paper](https://arxiv.org/abs/2102.12122).r>rCnum_attention_headssequences_reduction_ratiocst||_||_|j|jdkr@td|jd|jdt|j|j|_|j|j|_tj |j|j|j d|_ tj |j|j|j d|_ tj |j|j|j d|_ t|j|_||_|dkrtj||||d|_tj||jd|_dS) NrzThe hidden size (z6) is not a multiple of the number of attention heads ())biasrrErF)r)r*rCrrrdrkattention_head_size all_head_sizerrnZqkv_biasquerykeyvaluerSZattention_probs_dropout_probrUrsrNsequence_reductionrPrQrRr+r>rCrrrsr,r$r%r*s, z"PvtEfficientSelfAttention.__init__r.cCs6|dd|j|jf}||}|ddddS)NrYrrZrr )r[rrrvviewr^)r+r/Z new_shaper$r$r%transpose_for_scoress z.PvtEfficientSelfAttention.transpose_for_scoresFr/rWrXoutput_attentionsrcCs$|||}|jdkrl|j\}}}|ddd||||}||}|||dddd}||}|||} || |} t || dd} | t |j} tjj| dd} || } t | | } | dddd} | dd|jf}| |} |r| | fn| f}|S)NrrrZrYrbr )r~rxrsrr^r]r{rRryrzr!matmulrfmathsqrtrvr functionalZsoftmaxrU contiguousr[rwr})r+r/rWrXrZ query_layerriZseq_lenrBZ key_layerZ value_layerZattention_scoresZattention_probsZ context_layerZnew_context_layer_shapeoutputsr$r$r%r1s&       z!PvtEfficientSelfAttention.forward)F)r4r5r6r7rrkr8r*r!r9r~rlrr1r;r$r$r,r%rps rpcsPeZdZeeeedfdd ZddZd ej eee e ej ddd Z Z S) PvtAttentionrqcs6tt||||d|_t||d|_t|_dS)N)rCrrrs)rC)r)r*rpr+rmr#set pruned_headsr|r,r$r%r*s zPvtAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrrb)lenrr+rrrvrrrxryrzr#rorwunion)r+headsindexr$r$r% prune_heads s zPvtAttention.prune_headsFrcCs4|||||}||d}|f|dd}|S)Nrr)r+r#)r+r/rWrXrZ self_outputsattention_outputrr$r$r%r1szPvtAttention.forward)F)r4r5r6rrkr8r*rr!r9rlrr1r;r$r$r,r%rs rcsFeZdZdeeeeeedfdd ZejejdddZ Z S) PvtFFNN)r> in_featureshidden_features out_featurescsjt|dk r|n|}t|||_t|jtrBt|j|_ n|j|_ t|||_ t |j |_ dSr()r)r*rrndense1rHZ hidden_actr:r intermediate_act_fndense2rSrTrU)r+r>rrrr,r$r%r*)s  zPvtFFN.__init__r.cCs6||}||}||}||}||}|Sr()rrrUrr0r$r$r%r1:s      zPvtFFN.forward)NN) r4r5r6rrkrr*r!r9r1r;r$r$r,r%r(srcsDeZdZeeeeeedfdd Zdejeee dddZ Z S) PvtLayerr>rCrrr&rs mlp_ratiocszttj||jd|_t||||d|_|dkr>t|nt |_ tj||jd|_ t ||}t |||d|_dS)NrFrqr)r>rr)r)r*rrPrQ layer_norm_1r attentionr'Identityr& layer_norm_2rkrmlp)r+r>rCrrr&rsrZmlp_hidden_sizer,r$r%r*Ds  zPvtLayer.__init__Fr/rWrXrc Csn|j|||||d}|d}|dd}||}||}|||}||}||} | f|}|S)Nrrr)rrr&rr) r+r/rWrXrZself_attention_outputsrrZ mlp_outputZ layer_outputr$r$r%r1Zs    zPvtLayer.forward)F) r4r5r6rrkr8r*r!r9rlr1r;r$r$r,r%rCsrcsPeZdZedfdd Zd ejeeeeeee e e fdddZ Z S) PvtEncoderr>c stt||_td|jt|j}g}t |j D]r}| t ||dkrR|j n|jj d|d|j||j||dkr|jn |j|d|j|||j dkdq8t||_g}d}t |j D]}g}|dkr||j|d7}t |j|D]>}| t||j||j|||||j||j|dq| t|qt||_tj|jd|jd|_dS)NrrZrr=rrYrF)r)r*r>r!ZlinspaceZdrop_path_ratesumZdepthstolistrangeZnum_encoder_blocksappendr<r?Z patch_sizesstridesrB hidden_sizesrZ ModuleListpatch_embeddingsrrrZsequence_reduction_ratiosZ mlp_ratiosblockrPrQrR) r+r>Zdrop_path_decaysrViblockscurZlayersjr,r$r%r*rsJ       zPvtEncoder.__init__FTraroutput_hidden_states return_dictrcCs |rdnd}|rdnd}|jd}t|j}|} tt|j|jD]\} \} } | | \} } }| D]:}|| | ||}|d} |r||df}|r`|| f}q`| |dkrB| || |ddddd} qB| | } |r|| f}|st dd| ||fDSt | ||d S) Nr$rrrYr rZcss|]}|dk r|VqdSr(r$).0vr$r$r% sz%PvtEncoder.forward..Zlast_hidden_stater/ attentions) rrr enumerateziprr]r^rrRtupler )r+rarrrZall_hidden_statesZall_self_attentionsriZ num_blocksr/idxZembedding_layerZ block_layerrWrXrZ layer_outputsr$r$r%r1s4      "  zPvtEncoder.forward)FFT)r4r5r6rr*r! FloatTensorrrlrrr r1r;r$r$r,r%rqs5 rc@sNeZdZdZeZdZdZee j e j e j fddddZ d eed d d ZdS) PvtPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. pvtraN)modulercCst|tjrBtjj|jjd|jjd|j_|j dk r|j j nxt|tj rj|j j |jj dnPt|t rtjj|jjd|jjd|j_|jdk rtjj|jjd|jjd|j_dS)zInitialize the weightsr)ZmeanZstdNg?)rHrrninitZ trunc_normal_weightdatar>Zinitializer_rangeruZzero_rPZfill_r<rMrD)r+rr$r$r% _init_weightss&       z PvtPreTrainedModel._init_weightsF)rrzcCst|tr||_dSr()rHrZgradient_checkpointing)r+rrzr$r$r%_set_gradient_checkpointings z.PvtPreTrainedModel._set_gradient_checkpointing)F)r4r5r6r7r config_classZbase_model_prefixZmain_input_namerrrnrNrPrrrlrr$r$r$r%rs  raG This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`~PvtConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a  Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`PvtImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zSThe bare Pvt encoder outputting raw hidden-states without any specific head on top.c sxeZdZedfdd ZddZeede e e e de dd ejeeeeeeeee fd d d ZZS)PvtModelrcs(t|||_t||_|dSr()r)r*r>rencoder post_initr+r>r,r$r%r*s  zPvtModel.__init__cCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrlayerrr)r+Zheads_to_prunerrr$r$r% _prune_headsszPvtModel._prune_heads%(batch_size, channels, height, width)Zvision) checkpoint output_typerZmodalityexpected_outputNrcCs~|dk r |n|jj}|dk r |n|jj}|dk r4|n|jj}|j||||d}|d}|sl|f|ddSt||j|jdS)Nrarrrrrr)r>rruse_return_dictrr r/r)r+rarrrZencoder_outputssequence_outputr$r$r%r1#s$zPvtModel.forward)NNN)r4r5r6rr*rrPVT_INPUTS_DOCSTRINGr2r_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr!rrrlrrr1r;r$r$r,r%r s(    rz Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. c s|eZdZeddfdd Zeedee e e e dd e eje eje ee ee eeee fddd ZZS) PvtForImageClassificationN)r>rcsRt||j|_t||_|jdkr 1` a classification loss is computed (Cross-Entropy). NrrrZ regressionZsingle_label_classificationZmulti_label_classificationrY)losslogitsr/r)r>rrrZ problem_typerrr!longrkr Zsqueezerr}rr r/r) r+rarrrrrrrrZloss_fctr#r$r$r%r1`sJ    "     z!PvtForImageClassification.forward)NNNN)r4r5r6rr*rrr2r_IMAGE_CLASS_CHECKPOINTr r_IMAGE_CLASS_EXPECTED_OUTPUTrr!r9rlrrr1r;r$r$r,r%rJs(   r)rF)>r7rIrtypingrrrrr!Ztorch.nn.functionalrrr_Ztorch.utils.checkpointZtorch.nnrrr Z activationsr Zmodeling_outputsr r Zmodeling_utilsrZ pytorch_utilsrrutilsrrrrZconfiguration_pvtrZ get_loggerr4loggerrrrrrZ!PVT_PRETRAINED_MODEL_ARCHIVE_LISTr9r8rlr&Moduler'r<rmrprrrrrZPVT_START_DOCSTRINGrrrr$r$r$r%sX      B R*.Y' :