U 9%e@s\dZddlZddlZddlmZmZmZddlZddl Z ddl Z ddl m Z ddl m Z ddlmZddlmZdd lmZmZmZdd lmZdd lmZmZmZmZd d lmZee Z!d Z"dZ#dZ$d ddgZ%dZ&dZ'dZ(dZ)dZ*dddgZ+dBee,e,fe-e,ee j.e,ej/dddZ0Gddde j1Z2Gdd d e j1Z3Gd!d"d"e j1Z4Gd#d$d$e j1Z5Gd%d&d&e j1Z6Gd'd(d(e j1Z7Gd)d*d*e j1Z8Gd+d,d,e8Z9Gd-d.d.e j1Z:Gd/d0d0e j1Z;Gd1d2d2e j1Zd7Z?d8Z@ed9e?Gd:d;d;e>ZAedd>e>ZBed?e?Gd@dAdAe>ZCdS)Cz PyTorch SEW model.N)OptionalTupleUnion)nn)CrossEntropyLoss)ACT2FN)is_deepspeed_zero3_enabled)BaseModelOutputCausalLMOutputSequenceClassifierOutput)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) SEWConfigrzasapp/sew-tiny-100k-ft-ls100hi$iz_'MISTER QUILTER IS THE APPOSTILE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPOLLE'gzG?z(anton-l/sew-mid-100k-ft-keyword-spottingz '_unknown_'g ףp= #@zasapp/sew-tiny-100kzasapp/sew-small-100kzasapp/sew-mid-100k)shape mask_prob mask_lengthattention_mask min_masksreturncs|\}dkrtdkr6tdddtjdfdd}|dk rt|d nfd d t|D}tj |ft d }g} |} | d kr|S|D]v} || } tjj t | d| dd} t | d krd}n| d }t| tj| | tjd |g} | | qt| } t| dddddf|| f} | || } t ddddf}t||| f|| }| |} | dkr҈d| | dk<t|| dd |S)af Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `csXt|}t|}|kr2}|d|krTt|dd}|S)z;Given input length, compute how many spans should be maskedrr)intmax) input_lengthnum_masked_spanepsilonrrrsequence_lengthc/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/models/sew/modeling_sew.pycompute_num_masked_spanis  z6_compute_mask_indices..compute_num_masked_spanNcsg|]}qSr"r".0_)r!r"r# |sz)_compute_mask_indices..dtyperF)replace) ValueErrornprandomranditemsumdetachtolistrangezerosboolchoicearangelenZ concatenateonesZint32appendarrayZ broadcast_toreshaperZput_along_axis)rrrrr batch_sizer$ input_lengthsZ spec_aug_maskZspec_aug_mask_idxsZmax_num_masked_spanrrZspec_aug_mask_idxZdummy_mask_idxoffsetsr"rr#_compute_mask_indicesCs`    rBcs&eZdZdfdd ZddZZS)SEWNoLayerNormConvLayerrcsjt|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ t |j |_ dS)Nrr kernel_sizestridebias)super__init__conv_dim in_conv_dim out_conv_dimrConv1d conv_kernel conv_stride conv_biasconvrfeat_extract_activation activationselfconfiglayer_id __class__r"r#rIs  z SEWNoLayerNormConvLayer.__init__cCs||}||}|SN)rQrSrU hidden_statesr"r"r#forwards  zSEWNoLayerNormConvLayer.forward)r__name__ __module__ __qualname__rIr] __classcell__r"r"rXr#rCsrCcs&eZdZdfdd ZddZZS)SEWLayerNormConvLayerrcs|t|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ tj |jdd|_ t |j|_dS)NrrrDT)Zelementwise_affine)rHrIrJrKrLrrMrNrOrPrQ LayerNorm layer_normrrRrSrTrXr"r#rIs  zSEWLayerNormConvLayer.__init__cCs:||}|dd}||}|dd}||}|S)Nr%)rQ transposererSr[r"r"r#r]s      zSEWLayerNormConvLayer.forward)rr^r"r"rXr#rcsrccs&eZdZdfdd ZddZZS)SEWGroupNormConvLayerrcst|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ t |j |_ tj|j|jdd|_dS)NrrrDT)Z num_groupsZ num_channelsZaffine)rHrIrJrKrLrrMrNrOrPrQrrRrS GroupNormrerTrXr"r#rIs   zSEWGroupNormConvLayer.__init__cCs"||}||}||}|SrZ)rQrerSr[r"r"r#r]s   zSEWGroupNormConvLayer.forward)rr^r"r"rXr#rhsrhcs$eZdZfddZddZZS)SEWPositionalConvEmbeddingc sttj|j|j|j|jd|j|jd|_t rddl }|j j |jj ddtjj|jddd|_W5QRX|j ||jj|j ||jjntjj|jddd|_t|j|_t|j|_dS)N)rEpaddinggroupsrFrZ modifier_rankweight)namedim)rHrIrrM hidden_sizenum_conv_pos_embeddingsZnum_conv_pos_embedding_groupssqueeze_factorrQr deepspeedzeroGatheredParametersroutilsZ weight_normZregister_external_parameterweight_vweight_gSEWSamePadLayerrlrrRrS)rUrVrurXr"r#rIs$    z#SEWPositionalConvEmbedding.__init__cCs"||}||}||}|SrZ)rQrlrSr[r"r"r#r]s   z"SEWPositionalConvEmbedding.forwardr^r"r"rXr#rjs rjcs$eZdZfddZddZZS)r{cs$t|ddkrdnd|_dS)Nrkrr)rHrInum_pad_remove)rUrsrXr"r#rI(s zSEWSamePadLayer.__init__cCs,|jdkr(|ddddd|j f}|S)Nr)r|r[r"r"r#r],s zSEWSamePadLayer.forwardr^r"r"rXr#r{'s r{cs$eZdZfddZddZZS) SEWUpsamplingcs:tt|j|j|j|_t|j|_ |j|_dSrZ) rHrIrLinearrrrt projectionrrRrSrUrVrXr"r#rI3s  zSEWUpsampling.__init__cCsd||}||}|jdkr`|\}}}||j}||j}||||j|}||||}|S)Nr)rrSrtsizer>)rUr\bszsrc_lenZ src_embed_dimtgt_lenZ tgt_embed_dimr"r"r#r]9s     zSEWUpsampling.forwardr^r"r"rXr#r}2s r}cs0eZdZdZfddZddZddZZS)SEWFeatureEncoderz.Construct the features from raw audio waveformcstjdkr@tddgfddtjdD}n6jdkrdfddtjD}ntd jd t||_ d |_ d |_ dS) NgrouprrWcsg|]}t|ddqS)rr)rCr'irVr"r#r)Psz.SEWFeatureEncoder.__init__..rlayercsg|]}t|dqS)r)rcrrr"r#r)Tsz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT) rHrIZfeat_extract_normrhr5Znum_feat_extract_layersr-r ModuleList conv_layersgradient_checkpointing_requires_grad)rUrVrrXrr#rILs      zSEWFeatureEncoder.__init__cCs|D] }d|_qd|_dS)NF) parameters requires_gradrrUparamr"r"r#_freeze_parameters]s z$SEWFeatureEncoder._freeze_parameterscCsj|dddf}|jr"|jr"d|_|jD]<}|jr\|jr\|jr\dd}tjj|||}q(||}q(|S)NTcsfdd}|S)Ncs|SrZr"inputsmoduler"r#custom_forwardmszPSEWFeatureEncoder.forward..create_custom_forward..custom_forwardr"rrr"rr#create_custom_forwardls z8SEWFeatureEncoder.forward..create_custom_forward)rtrainingrrrtorchrx checkpoint)rU input_valuesr\Z conv_layerrr"r"r#r]bs   zSEWFeatureEncoder.forward)r_r`ra__doc__rIrr]rbr"r"rXr#rIs rcseZdZfddZZS)SEWFeatureExtractorcs8t|td|jjd|jjdjdtdS)Nz The class `zD` has been depreciated and will be removed in Transformers v5. Use `rz ` instead.)rHrIwarningswarnrYr_ __bases__ FutureWarningrrXr"r#rI}s  zSEWFeatureExtractor.__init__)r_r`rarIrbr"r"rXr#r|src seZdZdZdeeeeedfdd Zej eedd d Z dej e ej e e ej e ej e ej ee ej e ej e e ej fd d dZ ZS) SEWAttentionz=Multi-headed attention from 'Attention Is All You Need' paperFT) embed_dim num_headsdropout is_decoderrGcst||_||_||_|||_|j||jkrNtd|jd|d|jd|_||_t j |||d|_ t j |||d|_ t j |||d|_ t j |||d|_dS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g)rG)rHrIrrrhead_dimr-scalingrrr~k_projv_projq_projout_proj)rUrrrrrGrXr"r#rIs   zSEWAttention.__init__)tensorseq_lenrcCs ||||j|jddS)Nrrk)viewrrrg contiguous)rUrrrr"r"r#_shapeszSEWAttention._shapeN)r\key_value_statespast_key_valuerlayer_head_maskoutput_attentionsrcCsx|dk }|\}} } |||j} |r\|dk r\|djd|jdkr\|d} |d} n|r|||d|} |||d|} n|dk r|||d|} |||d|} tj|d| gdd} tj|d| gdd} n(|||d|} |||d|} |j r | | f}||j d|j f}|| | |j |} | j |} | j |} | d}t| | dd}|||j | |fkrtd||j | |fd||dk r ||d| |fkrtd |d| |fd|| ||j | ||}| ||j | |}tjj|dd}|dk r||j fkrhtd |j fd|| dddd| ||j | |}| ||j | |}|r| ||j | |}| ||j | |}nd}tjj||j|jd }t|| }|||j | |j fkr4td ||j | |j fd|| ||j | |j }|dd}| || |j}||}|||fS) z#Input shape: Batch x Time x ChannelNrrkrr%rqz$Attention weights should be of size z , but is z!Attention mask should be of size z/Head mask for a single layer should be of size )prz `attn_output` should be of size )rrrrrrrrcatrrrrr>Zbmmrgr-r functionalsoftmaxrrrr)rUr\rrrrrZis_cross_attentionrrr(Z query_statesZ key_statesZ value_statesZ proj_shaper attn_weightsZattn_weights_reshapedZ attn_probsZ attn_outputr"r"r#r]s~       "    zSEWAttention.forward)rFT)NNNNF)r_r`rarrfloatr7rIrTensorrrrr]rbr"r"rXr#rs4 rcs$eZdZfddZddZZS)SEWFeedForwardcsptt|j|_t|j|j|_ t |j t rDt |j |_n|j |_t|j|j|_t|j|_dSrZ)rHrIrDropoutZactivation_dropoutintermediate_dropoutr~rrZintermediate_sizeintermediate_dense isinstanceZ hidden_actstrrintermediate_act_fn output_densehidden_dropoutoutput_dropoutrrXr"r#rI$s  zSEWFeedForward.__init__cCs6||}||}||}||}||}|SrZ)rrrrrr[r"r"r#r]1s      zSEWFeedForward.forwardr^r"r"rXr#r#s rcs&eZdZfddZdddZZS)SEWEncoderLayercsftt|j|j|jdd|_t|j |_ tj |j|j d|_ t||_tj |j|j d|_dS)NF)rrrrZeps)rHrIrrrZnum_attention_headsZattention_dropout attentionrrrrrdlayer_norm_epsrer feed_forwardfinal_layer_normrrXr"r#rI=s  zSEWEncoderLayer.__init__NFcCsf|}|j|||d\}}}||}||}||}|||}||}|f}|rb||f7}|S)Nrr)rrrerr)rUr\rrZ attn_residualrr(outputsr"r"r#r]Js     zSEWEncoderLayer.forward)NFr^r"r"rXr#r<s rcs&eZdZfddZdddZZS) SEWEncodercst|_t|_tjj|_tj j j d|_ t j|_tfddtjD|_t|_d|_dS)Nrcsg|] }tqSr")rr&rr"r#r)fsz'SEWEncoder.__init__..F)rHrIrVrjpos_conv_embedrZ AvgPool1drtpoolrdrrrrerrrrr5num_hidden_layerslayersr}upsamplerrrXrr#rI_s    zSEWEncoder.__init__NFTc s|rdnd}rdnd}|dk rd||<|d}||jj} |jd|jj} tjd| | jddd | jdd} | | ddk}d|ddddddfj |j d}|t |j j }| |jdd|jd|jd}|jd} |dd }||} ||}t | d|d}|d d|f| d d|f}|dd }||}||}t}|jD]}|r||f}tg}|jr||jjkrd nd }|r|r |jr|jrfd d}tjj||||}n|||d}|d}|rd}rx||df}qx|r<||f}||}|jd| krvtj|ddd| |jdf}|st dd|||fDSt!|||dS)Nr"rr%rrdevice?r*rk.TFcsfdd}|S)Ncs|fSrZr"r)rrr"r#rszISEWEncoder.forward..create_custom_forward..custom_forwardr"rrrr#rsz1SEWEncoder.forward..create_custom_forwardr)NNcss|]}|dk r|VqdSrZr")r'vr"r"r# sz%SEWEncoder.forward..Zlast_hidden_stater\ attentions)"longr2rVrtrrr9rrexpandtor+Zfinfominrgrrrrerr rr0rZ layerdroprrxrrrrpadtupler )rUr\rroutput_hidden_states return_dictZall_hidden_statesZall_self_attentionsr@output_lengthsZmax_encoder_lengthZ attention_idsZn_input_timestepsZposition_embeddingsZpooled_hidden_statesZ min_lengthZdeepspeed_zero3_is_enabledrZdropout_probabilityZskip_the_layerrZ layer_outputsr"rr#r]js    &                zSEWEncoder.forward)NFFTr^r"r"rXr#r^s  rc@s\eZdZdZeZdZdZdZddZ ddd Z e e j efd d d Zee j d ddZdS)SEWPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. sewrTc Cst|trRtjj|jjddtd|jj d|jj dtj |jj dnt|tj rv|jjjd|jjdnt|tjtjfr|j j|jjdnt|tjrPtr@ddl}t|drt|d r|jj|j|jgdd tj|jjW5QRXn.|jj|jdd tj|jjW5QRXntj|jjt|tj tjfr||j dk r||j jdS) zInitialize the weightsrrkr)meanZstdrrNryrzrn)rrjrinitZnormal_rQromathsqrtrEZ in_channelsZ constant_rGr~datarVZinitializer_rangerdriZzero_Zfill_rMr ruhasattrrvrwryrzZkaiming_normal_)rUrrur"r"r# _init_weightss.     z SEWPreTrainedModel._init_weightsFcCst|ttfr||_dSrZ)rrrr)rUrvaluer"r"r#_set_gradient_checkpointingsz.SEWPreTrainedModel._set_gradient_checkpointing)r@cCs4dd}t|jj|jjD]\}}||||}q|S)zH Computes the output length of the convolutional layers cSstj|||dddS)Nfloor)Z rounding_moder)rdiv)rrErFr"r"r#_conv_out_lengthszMSEWPreTrainedModel._get_feat_extract_output_lengths.._conv_out_length)ziprVrNrO)rUr@rrErFr"r"r# _get_feat_extract_output_lengthssz3SEWPreTrainedModel._get_feat_extract_output_lengths)feature_vector_lengthrcCs~||dtj}|jd}tj||f|j|jd}d|tj |jd|jd|df<| dg d dg }|S)Nr%r)r+rrr) rr2rrrrr6r+rr9flipZcumsumr7)rUrrrr?r"r"r#"_get_feature_vector_attention_mask s "z5SEWPreTrainedModel._get_feature_vector_attention_maskN)F)r_r`rarr config_classZbase_model_prefixZmain_input_nameZsupports_gradient_checkpointingrrrr LongTensorrrrr"r"r"r#rs ra SEW was proposed in [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav Artzi. This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving etc.). This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`SEWConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z]The bare SEW Model transformer outputting raw hidden-states without any specific head on top.c seZdZedfdd Zd ejeejeejdddZ e e e e eeded deejeejeejeeeeeeeeefd d d ZZS)SEWModelrcst|||_t||_tj|jd|jd|_ |jd|j k|_ |j rbt |jd|j |_ t|j|_|jdks|jdkrtt|j |_t||_|dS)Nr%rr)rHrIrVrfeature_extractorrrdrJrrerrproject_featuresr~feature_projectionrZfeat_proj_dropoutfeature_dropoutmask_time_probmask_feature_prob Parameterr FloatTensorZuniform_masked_spec_embedrencoder post_initrrXr"r#rIJs   zSEWModel.__init__N)r\mask_time_indicesrcCst|jdds|S|\}}}|dk r<|j|j||<nZ|jjdkr|jrt||f|jj|jj ||jj d}t j ||j t jd}|j|j||<|jjdkr|jrt||f|jj|jj|jjd}t j ||j t jd}|dddfd|d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://arxiv.org/abs/1904.08779). Zapply_spec_augmentTNr)rrrr)rr+)rrrr%)getattrrVrr rr+rrrBZmask_time_lengthZmask_time_min_masksrrrr7rZmask_feature_lengthZmask_feature_min_masksr)rUr\r rr?r!rrZmask_feature_indicesr"r"r#_mask_hidden_states^s4 zSEWModel._mask_hidden_statesaudio)r output_typermodalityexpected_output)rrr rrrrc Cs|dk r |n|jj}|dk r |n|jj}|dk r4|n|jj}||}|dd}||}|jrl||}| |}|dk r| |j d|}|j ||d}|j |||||d} | d}|s|f| ddSt|| j| jdS)Nrrk)r rrrrrr)rVrruse_return_dictrrgrerrrrrrr r r\r) rUrrr rrrZextract_featuresr\Zencoder_outputsr"r"r#r]s8     zSEWModel.forward)NN)NNNNN)r_r`rarrIrr rrrrSEW_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPErr7rrr]rbr"r"rXr#rEs< .  rzaSEW Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).c seZdZdeedfdd ZddZddZd d Zd d Z e e e e eeeed deejeejeeeeeeeejeeefdddZZS) SEWForCTCN) target_langcs~t|t||_t|j|_||_|j dkrFt d|j dt |dr\|j r\|jn|j}t||j |_|dS)NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `SEWForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration. add_adapter)rHrIrrrrZ final_dropoutrr vocab_sizer-rYrroutput_hidden_sizerrr~lm_headr )rUrVrrrXr"r#rIs    zSEWForCTC.__init__cCsr|j}|dk r2t|jdddkr2td|dn<|dkrXt|jdddk rXtdn|dk rn|j|dddS)a' This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when passing `target_lang=...` to `from_pretrained(...)`. This method is **not** supposed to be called by the user and is prone to be changed in the future. NZadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)Z force_load)rrrVr-loggerinfoZ load_adapter)rUrr"r"r# tie_weightss  zSEWForCTC.tie_weightscCstdt|dS) Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5.Please use the equivalent `freeze_feature_encoder` method instead.Nrrrfreeze_feature_encoderrUr"r"r#freeze_feature_extractors z"SEWForCTC.freeze_feature_extractorcCs|jjdSr#Nrrrr'r"r"r#r&sz SEWForCTC.freeze_feature_encodercCs|jD] }d|_q dSz Calling this function will disable the gradient computation for the base model so that its parameters will not be updated during training. Only the classification head will be updated. FNrrrrr"r"r#freeze_base_model szSEWForCTC.freeze_base_model)rrrr expected_lossrrrrrlabelsrc Csf|dk r |n|jj}|j|||||d}|d}||}||} d} |dk r"||jjkrttd|jj|dk r|ntj |tj d}| | d tj } |dk} | d} || }tjj| dtjddd}tjjjd d ,tjj||| | |jj|jj|jjd } W5QRX|sR| f|td}| dk rN| f|S|St| | |j|jd S) a labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`. Nrrz$Label values must be <= vocab_size: r*r%)rqr+rF)enabled)blankZ reductionZ zero_infinitylosslogitsr\r)rVrrrrrrr-rZ ones_likerrr2rZ masked_selectrrZ log_softmaxZfloat32rgbackendsZcudnnflagsZctc_lossZ pad_token_idZctc_loss_reductionZctc_zero_infinity_HIDDEN_STATES_START_POSITIONr r\r)rUrrrrrr0rr\r5r4r@Z labels_maskZtarget_lengthsZflattened_targetsZ log_probsoutputr"r"r#r]sR      zSEWForCTC.forward)N)NNNNN)r_r`rarrrIr"r(r&r-rrrrr r_CTC_EXPECTED_OUTPUT_CTC_EXPECTED_LOSSrrr7rrr]rbr"r"rXr#rs6   rz SEW Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. c seZdZfddZddZddZddZeee e e e d e ed deejeejeeeeeeeejeee fd d dZZS)SEWForSequenceClassificationcst|t|dr$|jr$tdt||_|jd}|jrTt t |||_ t |j|j|_t |j|j|_|dS)NrzZSequence classification does not support the use of SEW adapters (config.add_adapter=True)r)rHrIrrr-rrruse_weighted_layer_sumrrrr; layer_weightsr~rrZclassifier_proj_size projector num_labels classifierr )rUrVZ num_layersrXr"r#rIks   z%SEWForSequenceClassification.__init__cCstdt|dS)z Calling this function will disable the gradient computation for the feature encoder so that its parameters will not be updated during training. r$Nr%r'r"r"r#r(|s z5SEWForSequenceClassification.freeze_feature_extractorcCs|jjdSr)r*r'r"r"r#r&sz3SEWForSequenceClassification.freeze_feature_encodercCs|jD] }d|_q dSr+r,rr"r"r#r-sz.SEWForSequenceClassification.freeze_base_modelr)rrrrrr.Nr/cCsf|dk r |n|jj}|jjr dn|}|j|||||d}|jjr|t}tj|dd}tjj |j dd} || dddj dd}n|d}| |}|dkr|jdd} n<||jd|} d|| <|j dd| j dd dd} || } d} |dk r"t}|| d|jj| d} |sR| f|td}| dk rN| f|S|St| | |j|jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NTrrrr%rrr3)rVrr=rr8rstackrrrr>rr2r?rrrrArr@r r\r)rUrrrrrr0rr\Z norm_weightsZ pooled_outputZ padding_maskr5r4Zloss_fctr9r"r"r#r]sF     z$SEWForSequenceClassification.forward)NNNNN)r_r`rarIr(r&r-rrr_SEQ_CLASS_CHECKPOINTr r_SEQ_CLASS_EXPECTED_OUTPUT_SEQ_CLASS_EXPECTED_LOSSrrrr7rrr]rbr"r"rXr#r<bs6    r<)Nr)Drrrtypingrrrnumpyr.rZtorch.utils.checkpointrZtorch.nnrZ activationsrZintegrations.deepspeedr Zmodeling_outputsr r r Zmodeling_utilsr rxrrrrZconfiguration_sewrZ get_loggerr_r r8rrrr:r;rCrDrEZ!SEW_PRETRAINED_MODEL_ARCHIVE_LISTrrrZndarrayrBModulerCrcrhrjr{r}rrrrrrrZSEW_START_DOCSTRINGrrrr<r"r"r"r#s           x" 3 "nK|