U 9%es>@sdZddlZddlZddlmZmZmZmZddlZ ddl Z ddl Z ddl m Z ddl mZmZmZddlmZddlmZdd lmZmZmZmZmZdd lmZdd lmZmZm Z m!Z!d d l"m#Z#m$Z$e %e&Z'd Z(dZ)dddgZ*e j+e,e,dddZ-d}e j+e,dddZ.d~e j/e j0e j1e,dddZ2de j+e j0ee,dddZ3dee,e,fe4e,ee j5e,e j6ddd Z7Gd!d"d"e j8Z9Gd#d$d$e j8Z:Gd%d&d&e j8Z;Gd'd(d(e j8ZGd-d.d.e j j8Z?Gd/d0d0e j8Z@Gd1d2d2e j8ZAGd3d4d4e j8ZBGd5d6d6e j8ZCGd7d8d8e j8ZDGd9d:d:e j8ZEGd;d<dd>e j8ZGGd?d@d@e j8ZHGdAdBdBe j8ZIGdCdDdDe j8ZJGdEdFdFe j8ZKGdGdHdHe j8ZLGdIdJdJe j8ZMGdKdLdLeZNGdMdNdNeNZOGdOdPdPeNZPGdQdRdReNZQGdSdTdTeNZRGdUdVdVeNZSGdWdXdXeNZTGdYdZdZeNZUGd[d\d\eNZVGd]d^d^e j8ZWGd_d`d`e j8ZXdaZYdbZZdcZ[eddeYGdedfdfeNZ\edgeZGdhdidieNZ]deNe j^ee j^e4e4e4ee j8e_ee j^ee j^e j^ffdn dodpZ`edqeZGdrdsdseNZaedteZGdudvdveNZbdwZcGdxdydye j8ZdedzecGd{d|d|eZedS)z PyTorch SpeechT5 model.N)ListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossL1Loss)ACT2FN)is_deepspeed_zero3_enabled)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutputSeq2SeqSpectrogramOutput)PreTrainedModel)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)SpeechT5ConfigSpeechT5HifiGanConfigrzmicrosoft/speecht5_asrzmicrosoft/speecht5_ttszmicrosoft/speecht5_vc) input_ids pad_token_iddecoder_start_token_idcCsh||j}|ddddf|ddddf<||dddf<|dkrTtd||dk||S)z1 Shift input ids one token to the right. Nrrz1self.model.config.pad_token_id has to be defined.i) new_zerosshapeclone ValueError masked_fill_)rrrZshifted_input_idsr#m/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/models/speecht5/modeling_speecht5.pyshift_tokens_right;s (r%) input_valuesreduction_factorcCsj|dkr"|dd|dd|f}||j}|ddddf|ddddf<||dkd|S)zw Shift input spectrograms one timestep to the right. Also applies the reduction factor to the sequence length. rNrY)rrr r")r&r'Zshifted_input_valuesr#r#r$shift_spectrograms_rightKs  (r*)input_ids_shapedtypedevicepast_key_values_lengthcCs|\}}tj||ft|j|d}tj|d|d}|||d|ddkd||}|dkrtj tj ||||d|gdd}|ddddddf |d|||S)zB Make causal mask used for bi-directional self-attention. r-rrrr,r-dimN) torchfullfinfominarangesizer"viewtocatzerosexpand)r+r,r-r.bsztgt_lenmaskZ mask_condr#r#r$_make_causal_mask]s"  rA)r@r,r?cCsj|\}}|dk r|n|}|ddddddf|d|||}d|}||tjt|jS)z_ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. Nr?)r8r=r:Z masked_fillr3boolr5r6)r@r,r?r>src_lenZ expanded_maskZ inverted_maskr#r#r$ _expand_maskos  *rE)r mask_prob mask_lengthattention_mask min_masksreturncs|\}dkrtdkr6tdddtjdfdd}|dk rt|d nfd d t|D}tj |ft d }g} |} | d kr|S|D]v} || } tjj t | d| dd} t | d krd}n| d }t| tj| | tjd |g} | | qt| } t| dddddf|| f} | || } t ddddf}t||| f|| }| |} | dkr҈d| | dk<t|| dd |S)af Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `csXt|}t|}|kr2}|d|krTt|dd}|S)z;Given input length, compute how many spans should be maskedrr)intmax) input_lengthnum_masked_spanepsilonrGrFrIsequence_lengthr#r$compute_num_masked_spans  z6_compute_mask_indices..compute_num_masked_spanNrcsg|]}qSr#r#.0_)rRr#r$ sz)_compute_mask_indices..r,rF)replace)r!nprandomranditemsumdetachtolistranger<rCchoicer7lenZ concatenateonesZint32appendarrayZ broadcast_toreshaperMZput_along_axis)rrFrGrHrI batch_sizerS input_lengthsZ spec_aug_maskZspec_aug_mask_idxsZmax_num_masked_spanrNrOZspec_aug_mask_idxZdummy_mask_idxoffsetsr#rPr$_compute_mask_indices~s`    rkcs&eZdZdfdd ZddZZS)SpeechT5NoLayerNormConvLayerrcsjt|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ t |j |_ dS)Nrr kernel_sizestridebias)super__init__conv_dim in_conv_dim out_conv_dimrConv1d conv_kernel conv_stride conv_biasconvr feat_extract_activation activationselfconfiglayer_id __class__r#r$rrs  z%SpeechT5NoLayerNormConvLayer.__init__cCs||}||}|SN)rzr|r~ hidden_statesr#r#r$forwards  z$SpeechT5NoLayerNormConvLayer.forward)r__name__ __module__ __qualname__rrr __classcell__r#r#rr$rlsrlcs&eZdZdfdd ZddZZS)SpeechT5LayerNormConvLayerrcs|t|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ tj |jdd|_ t |j|_dS)NrrrmT)Zelementwise_affine)rqrrrsrtrurrvrwrxryrz LayerNorm layer_normr r{r|r}rr#r$rr s  z#SpeechT5LayerNormConvLayer.__init__cCs:||}|dd}||}|dd}||}|S)Nr)rz transposerr|rr#r#r$rs      z"SpeechT5LayerNormConvLayer.forward)rrr#r#rr$r srcs&eZdZdfdd ZddZZS)SpeechT5GroupNormConvLayerrcst|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ t |j |_ tj|j|jdd|_dS)NrrrmT)Z num_groupsZ num_channelsZaffine)rqrrrsrtrurrvrwrxryrzr r{r| GroupNormrr}rr#r$rr)s   z#SpeechT5GroupNormConvLayer.__init__cCs"||}||}||}|Sr)rzrr|rr#r#r$r9s   z"SpeechT5GroupNormConvLayer.forward)rrr#r#rr$r(srcseZdZdZdeeeedfdd ZdeeeedddZedeeeedd d Z e de j ed d dZ de j eeedddZZS)%SpeechT5SinusoidalPositionalEmbeddingzDThis module produces sinusoidal positional embeddings of any length.N) num_positions embedding_dim padding_idxcs4td|_||_||_|||j||dSN)rqrroffsetrr make_weights)r~rrrrr#r$rrDs  z.SpeechT5SinusoidalPositionalEmbedding.__init__)num_embeddingsrrcCsP||||}t|dr.|j|jj|jjd}t||_d|j_|j dS)Nweightsr0F) get_embeddinghasattrr:rr,r-r Parameter requires_gradZdetach_)r~rrrZ emb_weightsr#r#r$rKs   z2SpeechT5SinusoidalPositionalEmbedding.make_weightscCs|d}td|d}ttj|tjd| }tj|tjdd|d}tjt|t |gdd |d}|ddkrtj|t |dgdd}|dk rd||ddf<| t S) z Build sinusoidal embeddings. This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of "Attention Is All You Need". ri'rrXrr1rN)mathlogr3expr7float unsqueezer;sincosr9r<r:Zget_default_dtype)rrrZhalf_dimembr#r#r$rUs & z3SpeechT5SinusoidalPositionalEmbedding.get_embeddingr)rr.cCs||\}}|||j||j}|jd|}||jdkrZ|||j|j|j|j d| d ||d S)Nrrr) r8"create_position_ids_from_input_idsrr:r-rrrr index_selectr9r_)r~rr.r>seq_lenZ position_idsZmax_posr#r#r$rgs z-SpeechT5SinusoidalPositionalEmbedding.forward)rrr.cCs6||}tj|dd|||}||S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rr1)nerLr3cumsumZtype_aslong)r~rrr.r@Zincremental_indicesr#r#r$rvs zHSpeechT5SinusoidalPositionalEmbedding.create_position_ids_from_input_ids)N)N)N)r)r)rrr__doc__rLrrrr staticmethodrr3no_gradTensorrrrr#r#rr$rAs rcs$eZdZfddZddZZS)SpeechT5PositionalConvEmbeddingc sttj|j|j|j|jd|jd|_tjj }t tjj drNtjj j }t rddl }|jj|jjdd||jddd|_W5QRX|j||jj|j||jjn||jddd|_t|j|_t|j|_dS)Nr)rnpaddinggroups weight_normr)Z modifier_rankweight)namer2)rqrrrrv hidden_sizenum_conv_pos_embeddingsZnum_conv_pos_embedding_groupsrzutilsrrZparametrizationsr deepspeedzeroZGatheredParametersrZregister_external_parameterZweight_vZweight_gSpeechT5SamePadLayerrr r{r|)r~rrrrr#r$rrs(   z(SpeechT5PositionalConvEmbedding.__init__cCs:|dd}||}||}||}|dd}|SNrr)rrzrr|rr#r#r$rs      z'SpeechT5PositionalConvEmbedding.forwardrr#r#rr$rs rcs*eZdZdZdfdd ZddZZS) SpeechT5ScaledPositionalEncodinguS Scaled positional encoding, see §3.2 in https://arxiv.org/abs/1809.08895 cst||}td|d}ttjd|dtjdtd| }t|||dddddf<t |||dddddf<|d}t |j d|ddt j|d |_||_tj td |_dS) NrrrrXg@peF) persistent)prB)r3r<r7rrrrrrrrqrrregister_bufferrDropoutdropoutr2rtensoralpha)r~rr2max_lenrpositionZdiv_termrr#r$rrs *$$  z)SpeechT5ScaledPositionalEncoding.__init__cCs4||j|jddd|df}||}|S)Nr)rrr8r)r~rr#r#r$rs& z(SpeechT5ScaledPositionalEncoding.forward)r)rrrrrrrrr#r#rr$rs rcs&eZdZdfdd ZddZZS)"SpeechT5RelativePositionalEncodingcs.t||_||_tjd|||_dSr)rqrrr2 max_lengthr3r Embeddingpe_k)r~r2rrr#r$rrs z+SpeechT5RelativePositionalEncoding.__init__cCs~|jd}td||j}|dddf|dddf}|j |||j k<|jd|||jk<||j}||S)Nrr)rr3r7rr:r-rr)r~rrZpos_seqr#r#r$rs   z*SpeechT5RelativePositionalEncoding.forward)rrr#r#rr$rsrcs$eZdZfddZddZZS)rcs$t|ddkrdnd|_dS)Nrrr)rqrrnum_pad_remove)r~rrr#r$rrs zSpeechT5SamePadLayer.__init__cCs,|jdkr(|ddddd|j f}|S)Nr)rrr#r#r$rs zSpeechT5SamePadLayer.forwardrr#r#rr$rs rcs0eZdZdZfddZddZddZZS)SpeechT5FeatureEncoderz.Construct the features from raw audio waveformcstjdkr@tddgfddtjdD}n6jdkrdfddtjD}ntd jd t||_ d |_ d |_ dS) Ngrouprrcsg|]}t|ddqS)rr)rlrUirr#r$rWsz3SpeechT5FeatureEncoder.__init__..rlayercsg|]}t|dqS)r)rrrr#r$rWsz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT) rqrrZfeat_extract_normrraZnum_feat_extract_layersr!r ModuleList conv_layersgradient_checkpointing_requires_grad)r~rrrrr$rrs       zSpeechT5FeatureEncoder.__init__cCs|D] }d|_qd|_dSNF) parametersrr)r~paramr#r#r$_freeze_parameterss z)SpeechT5FeatureEncoder._freeze_parameterscCsj|dddf}|jr"|jr"d|_|jD]<}|jr\|jr\|jr\dd}tjj|||}q(||}q(|S)NTcsfdd}|S)Ncs|Srr#inputsmoduler#r$custom_forward szUSpeechT5FeatureEncoder.forward..create_custom_forward..custom_forwardr#rrr#rr$create_custom_forward s z=SpeechT5FeatureEncoder.forward..create_custom_forward)rtrainingrrrr3r checkpoint)r~r&rZ conv_layerrr#r#r$rs   zSpeechT5FeatureEncoder.forward)rrrrrrrrrr#r#rr$rs rcs$eZdZfddZddZZS)SpeechT5FeatureProjectioncsJttj|jd|jd|_t|jd|j|_ t |j |_ dS)NrZeps) rqrrrrrslayer_norm_epsrLinearr projectionrZfeat_proj_dropoutrr~rrr#r$rrs z"SpeechT5FeatureProjection.__init__cCs&||}||}||}||fSr)rrr)r~rZnorm_hidden_statesr#r#r$r$s   z!SpeechT5FeatureProjection.forwardrr#r#rr$rs rcseZdZfddZddZdejeejeej dddZ e ejd d d Z e eje fd d dZdej eej eejdddZZS)SpeechT5SpeechEncoderPrenetcs|t||_t||_t||_|jdks8|jdkrPt t |j |_t||_t|j|jd|j |j|_dS)Nr)r)rqrrrrfeature_encoderrfeature_projectionmask_time_probmask_feature_probrrr3 FloatTensorruniform_masked_spec_embedrpos_conv_embedrmax_speech_positionsrpos_sinusoidal_embedrrr#r$rr-s    z$SpeechT5SpeechEncoderPrenet.__init__cCs|jdSr)rrr~r#r#r$freeze_feature_encoder>sz2SpeechT5SpeechEncoderPrenet.freeze_feature_encoderN)r&rHmask_time_indicesc Cs||}|dd}|dk r0||jd|}||\}}|j|||d}||}||}|dk rx|d}nt j |jddt j|j d}| |}||}||fS)Nrr)rrHr0) rr"_get_feature_vector_attention_maskrr_mask_hidden_statesrrrr3r<r-r) r~r&rHrZextract_featuresrZpositional_conv_embedding padding_maskZ positional_sinusoidal_embeddingsr#r#r$rAs*    z#SpeechT5SpeechEncoderPrenet.forward)feature_vector_lengthrHcCs|jdddddf}||tj}|jd}tj||f|j|jd}d|tj |jd|jd|df<| dgd dg }|S)Nrr1rr0rr/) r _get_feat_extract_output_lengthsr:r3rrr<r,r-r7fliprC)r~rrHZnon_padded_lengthsoutput_lengthsrhr#r#r$rds "z>SpeechT5SpeechEncoderPrenet._get_feature_vector_attention_mask)ricCs4dd}t|jj|jjD]\}}||||}q|S)zH Computes the output length of the convolutional layers cSstj|||dddS)Nfloor)Z rounding_moder)r3div)rNrnror#r#r$_conv_out_lengthyszVSpeechT5SpeechEncoderPrenet._get_feat_extract_output_lengths.._conv_out_length)ziprrwrx)r~rirrnror#r#r$rtsz.)rqrrrrrraZspeech_decoder_prenet_layerslayersrr r final_layerrpositional_dropoutrencode_positionsZspeaker_embedding_dimspeaker_embeds_layerrrrr$rrs   z$SpeechT5SpeechDecoderPrenet.__init__N)r&speaker_embeddingscCs|}|jD]*}tj||}tjj||jjdd}q ||}||}|dk rtj |}| d}| d| dd}t j||gdd}tj||}|S)NT)rrrr1)rr functionalZrelurrZspeech_decoder_prenet_dropoutrr normalizerr=r8r3r;r)r~r&r inputs_embedsrr#r#r$rs"     z#SpeechT5SpeechDecoderPrenet.forward)N) rrrrrr3rrrrr#r#rr$r s  r cs&eZdZdfdd ZddZZS)SpeechT5BatchNormConvLayerrcst|dkr|j}n|j}||jdkr6|j}n|j}tj|||jd|jdddd|_t ||_ ||jdkrt |_ nd|_ t |j|_dS)NrrrF)rnrorrp)rqrrr Zspeech_decoder_postnet_unitsspeech_decoder_postnet_layersrrvZspeech_decoder_postnet_kernelrzZ BatchNorm1d batch_normZTanhr|rZspeech_decoder_postnet_dropoutr)r~rrrtrurr#r$rrs(    z#SpeechT5BatchNormConvLayer.__init__cCs6||}||}|jdk r(||}||}|Sr)rzrr|rrr#r#r$rs      z"SpeechT5BatchNormConvLayer.forward)rrr#r#rr$rsrcs<eZdZfddZejdddZejdddZZS)SpeechT5SpeechDecoderPostnetcs^t|_tjjj|_tjj|_ t fddt j D|_ dS)Ncsg|]}t|qSr#)rrrr#r$rWsz9SpeechT5SpeechDecoderPostnet.__init__..)rqrrrrrrr r'feat_outprob_outrrarrrrrr$rrs z%SpeechT5SpeechDecoderPostnet.__init__rcCsJ|||dd|jj}||}|||dd}|||fS)Nrr)rr9r8rr postnetr)r~routputs_before_postnetoutputs_after_postnetlogitsr#r#r$rs z$SpeechT5SpeechDecoderPostnet.forwardcCs0|dd}|jD] }||}q||ddSr)rr)r~rZ layer_outputrr#r#r$r!s   z$SpeechT5SpeechDecoderPostnet.postnet) rrrrrr3rrrrr#r#rr$rs rcs<eZdZfddZddZddZejddd ZZ S) SpeechT5TextEncoderPrenetcs>t||_t|j|j|j|_t |j |j|j |_ dSr) rqrrrrr vocab_sizerr embed_tokensrrmax_text_positionsrrrr#r$rr)s z"SpeechT5TextEncoderPrenet.__init__cCs|jSrr$rr#r#r$get_input_embeddings3sz.SpeechT5TextEncoderPrenet.get_input_embeddingscCs ||_dSrr&r~valuer#r#r$set_input_embeddings6sz.SpeechT5TextEncoderPrenet.set_input_embeddings)rcCs||}||}|Sr)r$r)r~rrr#r#r$r9s  z!SpeechT5TextEncoderPrenet.forward) rrrrrr'r*r3rrrr#r#rr$r"(s r"csReZdZfddZddZddZd ejeej ee ej dd d Z Z S) SpeechT5TextDecoderPrenetcsnt||_t|j|_|jr0t |j nd|_ t |j |j |j|_t|j|jd|j |j|_dS)NrBr)rqrrrrrrrZscale_embeddingrsqrtr embed_scalerr#rr$rr%embed_positionsrrr#r$rr@s z"SpeechT5TextDecoderPrenet.__init__cCs|jSrr&rr#r#r$r'Nsz.SpeechT5TextDecoderPrenet.get_input_embeddingscCs ||_dSrr&r(r#r#r$r*Qsz.SpeechT5TextDecoderPrenet.set_input_embeddingsN)rrHpast_key_valuescCs~|dk r"|}|d|d}ntd|dk rD|ddjdnd}|||}|||j}||7}||}||fS)Nrz'You have to specify `decoder_input_ids`rr)r8r9r!rr.r$r-r)r~rrHr/ input_shaper.Z positionsrr#r#r$rTs  z!SpeechT5TextDecoderPrenet.forward)NN)rrrrrr'r*r3rrr rrrrr#r#rr$r+?s  r+cs<eZdZfddZejdddZddZdd ZZ S) SpeechT5TextDecoderPostnetcs*t||_tj|j|jdd|_dS)NFrp)rqrrrrrrr#lm_headrrr#r$rrks z#SpeechT5TextDecoderPostnet.__init__rcCs ||Srr3rr#r#r$rpsz"SpeechT5TextDecoderPostnet.forwardcCs|jSrr4rr#r#r$get_output_embeddingsssz0SpeechT5TextDecoderPostnet.get_output_embeddingscCs ||_dSrr4r~Znew_embeddingsr#r#r$set_output_embeddingsvsz0SpeechT5TextDecoderPostnet.set_output_embeddings) rrrrrr3rrr5r7rr#r#rr$r1js r1cseZdZdZdeeeeedfdd Zej eedd d Z dej e ej e e ej e ej e ej e ej ee ej e ej e e ej fd d dZ ZS)SpeechT5Attentionz Multi-headed attention from 'Attention Is All You Need' paper with relative position bias (see https://aclanthology.org/N18-2074.pdf) r)FT) embed_dim num_headsr is_decoderrpcst||_||_||_|||_|j||jkrNtd|jd|d|jd|_||_t j |||d|_ t j |||d|_ t j |||d|_ t j |||d|_dS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gr2)rqrrr9r:rhead_dimr!scalingr;rrk_projv_projq_projout_proj)r~r9r:rr;rprr#r$rrs   zSpeechT5Attention.__init__)rrr>cCs ||||j|jddSr)r9r:r<r contiguous)r~rrr>r#r#r$_shapeszSpeechT5Attention._shapeN)rkey_value_statespast_key_valuerHlayer_head_mask position_biasoutput_attentionsrJcCs|dk }|\} } } |||j} |rD|dk rD|d} |d}n|rr|||d| } |||d| }n|dk r|||d| } |||d| }tj|d| gdd} tj|d|gdd}n(|||d| } |||d| }|jr| |f}| |j d|j f}|| | | j |} | j |} |j |}| d}t | | dd}|| |j | |fkrtd| |j | |fd||dk r|  | |j d|j  dd}t|| d d}| dd | |j |d|d}||7}|dk rx|| d| |fkrNtd | d| |fd|| | |j | ||}| | |j | |}tjj|dd}|dk r||j fkrtd |j fd|| dddd| | |j | |}| | |j | |}|r$| | |j | |}| | |j | |}nd}tjj||j|jd }t ||}|| |j | |j fkrtd | |j | |j fd|| | |j | |j }| dd}|| | |j}||}|||fS)z#Input shape: Batch x Time x ChannelNrrrrr1z$Attention weights should be of size z , but is rz!Attention mask should be of size z/Head mask for a single layer should be of size )rrz `attn_output` should be of size )r8r@r=rCr>r?r3r;r;r:r<r9Zbmmrr!rBmatmulrrZsoftmaxrrrgr9rA)r~rrDrErHrFrGrHZis_cross_attentionr>r?rVZ query_statesZ key_statesZ value_statesZ proj_shaperD attn_weightsZ reshape_qZ rel_pos_biasZattn_weights_reshapedZ attn_probsZ attn_outputr#r#r$rs      "   "   zSpeechT5Attention.forward)r)FT)NNNNNF)rrrrrLrrCrrr3rrCrrrrr#r#rr$r8zs8  r8cs$eZdZfddZddZZS)SpeechT5FeedForwardcsltt|j|_t|j||_t |j t rBt |j |_ n|j |_ t||j|_t|j|_dSr)rqrrrrZactivation_dropoutintermediate_dropoutrrintermediate_dense isinstanceZ hidden_actstrr intermediate_act_fn output_densehidden_dropoutoutput_dropout)r~rZintermediate_sizerr#r$rrs  zSpeechT5FeedForward.__init__cCs6||}||}||}||}||}|Sr)rMrPrLrQrSrr#r#r$r(s      zSpeechT5FeedForward.forwardrr#r#rr$rKs rKcsNeZdZedfdd Zd ejeejeejeejedddZ Z S) SpeechT5EncoderLayerrcsjtt|j|j|jdd|_t|j |_ tj |j|j d|_ t||j|_tj |j|j d|_dS)NFr9r:rr;r)rqrrr8rencoder_attention_headsattention_dropout attentionrrrRrrrrrKZencoder_ffn_dim feed_forwardfinal_layer_normrrr#r$rr3s zSpeechT5EncoderLayer.__init__NFrrHrFrGrHc Csj|}|j|||||d\}}}||}||}||}|||}||}|f} |rf| |f7} | S)as Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, hidden_size)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(config.encoder_attention_heads,)`. position_bias (`torch.FloatTensor`): relative position embeddings of size `(seq_len, seq_len, hidden_size // encoder_attention_heads)` output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. r[)rXrrrYrZ) r~rrHrFrGrHresidualrJrVoutputsr#r#r$r@s"     zSpeechT5EncoderLayer.forward)NNNF) rrrrrrr3rrrCrrr#r#rr$rT2srTc steZdZedfdd Zd ejeejeejeejeejeejeeejee ee d dd Z Z S) SpeechT5DecoderLayerrcstt|j|j|jdd|_t|j |_ tj |j|j d|_ t|j|j|jdd|_tj |j|j d|_t||j|_tj |j|j d|_dS)NTrUr)rr;)rqrrr8rZdecoder_attention_headsrW self_attnrrrRrrrself_attn_layer_norm encoder_attnencoder_attn_layer_normrKZdecoder_ffn_dimrYrZrrr#r$rrps$ zSpeechT5DecoderLayer.__init__NFT) rrHencoder_hidden_statesencoder_attention_maskrFcross_attn_layer_head_maskrErH use_cachec Cs|} |dk r|ddnd} |j|| |||d\}} } ||}| |}||}d}d}|dk r|} |dk rz|ddnd}|j||||||d\}}}||}| |}||}| |} |||}||}|f}|r|| |f7}| r|| f7}|S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, hidden_size)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(batch, seq_len, hidden_size)` encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of size `(decoder_attention_heads,)`. past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. Nr)rrErHrFrHr)rrDrHrFrErH)r_rr`rarbrYrZ)r~rrHrcrdrFrerErHrfr\Zself_attn_past_key_valueZself_attn_weightsZpresent_key_valueZcross_attn_present_key_valueZcross_attn_weightsZcross_attn_past_key_valuer]r#r#r$rsJ         zSpeechT5DecoderLayer.forward)NNNNNNFT) rrrrrrr3rrrrCrrr#r#rr$r^os( r^c@s2eZdZdZeZdZdZdZddZ d dd Z d S) SpeechT5PreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. speecht5r&Tc Cst|trTtjj|jjddtd|jj d|jj dtj |jj dnXt|t rtd|jj}tjj|jj| |dtjj|jj | |dnt|tjr|jjjd|jjd|j dk r|j jnt|tjtjfr|j j|jjdnt|tjrjtj|j|j dk rt|j|j |j d}tjj|j | |dnBt|tjr|jjjd|jjd|jdk r|jj|jdS) zInitialize the weightsrrrmeanZstd)abr)NrB)rNrrinitnormal_rzrrr,rnZ in_channelsZ constant_rprrZ in_featuresrrdatarinitializer_rangezero_rrZfill_rvZkaiming_normal_rrr)r~rkr#r#r$ _init_weightss6        z%SpeechT5PreTrainedModel._init_weightsFcCst|tttfr||_dSr)rNSpeechT5EncoderSpeechT5Decoderrr)r~rr)r#r#r$_set_gradient_checkpointingsz3SpeechT5PreTrainedModel._set_gradient_checkpointingN)F) rrrrr config_classZbase_model_prefixmain_input_nameZsupports_gradient_checkpointingrsrvr#r#r#r$rgsrgc sdeZdZdZedfdd Zd ejeej eej ee ee ee e e e fdddZZS) rtzu Transformer encoder consisting of *config.encoder_layers* layers. Each layer is a [`SpeechT5EncoderLayer`]. rcs~ttjjjd|_tj|_ j |_ t fddt jD|_tjjj|_d|_|dS)Nrcsg|] }tqSr#)rTrTrr#r$rWsz,SpeechT5Encoder.__init__..F)rqrrrrrrrrrRrZencoder_layerdrop layerdroprraZencoder_layersrrrVZencoder_max_relative_positionr.r post_initrrrr$rrs   zSpeechT5Encoder.__init__N)rrH head_maskrHoutput_hidden_states return_dictrJcsdk r n|jj|dk r |n|jj}|dk r4|n|jj}|dk rPt||j}||}||}||}t }|r|dnd} rdnd} |dk r| dt |j krt dt |j d| ddt|j D]\} } |r| |f} d} |jrtg}||jk} | r|r|jrd|jrdfdd }tjj|| |||dk rZ|| nd|}n$| ||||dk r~|| ndd }|d}| rd }r| |d f} q|r| |f} |std d|| | fDSt|| | dS)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`): Features extracted from the speech or text input by the encoder prenet. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr#rz&The head_mask should be specified for  layers, but it is for .Fcsfdd}|S)Ncs|fSrr#r)rrHr#r$roszNSpeechT5Encoder.forward..create_custom_forward..custom_forwardr#rrHrr$rnsz6SpeechT5Encoder.forward..create_custom_forward)rHrGrFrH)NNrcss|]}|dk r|VqdSrr#rUvr#r#r$ sz*SpeechT5Encoder.forward..last_hidden_stater attentions)rrHr|use_return_dictrEr,rrr.r r8rcrr! enumeraterr3r\ryrrrtupler )r~rrHr{rHr|r}rGdeepspeed_zero3_is_enabledall_hidden_statesall_self_attentionsidxZ encoder_layerskip_the_layerdropout_probabilityr layer_outputsr#rr$rsn$            zSpeechT5Encoder.forward)NNNNNrrrrrrrr3rrrrCrrr rrr#r#rr$rt s  rtc sdeZdZdZedfdd Zd ejeej eej ee ee ee e e e fdddZZS) SpeechT5EncoderWithSpeechPrenetz Wrapper around SpeechT5Encoder that applies SpeechT5SpeechEncoderPrenet to convert the audio waveform data to hidden features. rcs2t|t||_t||_d|_|dSr)rqrrrprenetrtwrapped_encoderrrzrrr#r$rrs    z(SpeechT5EncoderWithSpeechPrenet.__init__Nr&rHr{rHr|r}rJc Cs*|||\}}|j||||||d}|SN)rrHr{rHr|r}rr r~r&rHr{rHr|r}rr]r#r#r$rs  z'SpeechT5EncoderWithSpeechPrenet.forward)NNNNNrr#r#rr$rs   rc steZdZdZedfdd ZddZddZd ej e ej e ej e e e e e e e eefd d d ZZS)SpeechT5EncoderWithTextPrenetz| Wrapper around SpeechT5Encoder that applies SpeechT5TextEncoderPrenet to convert the input_ids to hidden features. rcs2t|t||_t||_d|_|dSr)rqrrr"rrtrrrzrrr#r$rrs    z&SpeechT5EncoderWithTextPrenet.__init__cCs |jSrrr'rr#r#r$r'sz2SpeechT5EncoderWithTextPrenet.get_input_embeddingscCs|j|dSrrr*r(r#r#r$r*sz2SpeechT5EncoderWithTextPrenet.set_input_embeddingsNrc Cs$||}|j||||||d}|Srrrr#r#r$rs  z%SpeechT5EncoderWithTextPrenet.forward)NNNNN)rrrrrrrr'r*r3rrrrCrrr rrr#r#rr$rs$  rc sdeZdZdZedfdd Zd ejeej eej ee ee ee e e e fdddZZS) SpeechT5EncoderWithoutPrenet This wrapper class is a helper class to correctly load pretrained checkpoints when used in combination with [`SpeechT5Model`]. rcs(t|t||_d|_|dSr)rqrrrtrrrzrrr#r$rrs  z%SpeechT5EncoderWithoutPrenet.__init__NrcCs|j||||||dSr)r)r~r&rHr{rHr|r}r#r#r$rs z$SpeechT5EncoderWithoutPrenet.forward)NNNNNrr#r#rr$rs   rcseZdZdZedfdd ZddZd eej eej eej eej eej eej ee ej ee ee ee ee eeefd d d ZZS) ruzt Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`SpeechT5DecoderLayer`] rcsFtj|_tfddtjD|_d|_ | dS)Ncsg|] }tqSr#)r^rTrr#r$rWsz,SpeechT5Decoder.__init__..F) rqrrZdecoder_layerdropryrrraZdecoder_layersrrrzrrrr$rrs   zSpeechT5Decoder.__init__cCs`d}|ddkr$t||j|j|d}|dk r\t||j|dd|j}|dkrT|n||}|S)Nrr)r-r.r?)rAr,r-rEr:)r~rHr0rr.Zcombined_attention_maskZexpanded_attn_maskr#r#r$_prepare_decoder_attention_masks z/SpeechT5Decoder._prepare_decoder_attention_maskN) rrHrcrdr{cross_attn_head_maskr/rfrHr|r}rJc  s.dk r n|jj| dk r | n|jj} dk r4n|jj| dk rH| n|jj} |dd} |dk rz|ddjdnd} ||| || }|dk r|dk rt||j | dd}t }|j r|j rֈrt dd| rdnd}rdnd}r|dk rdnd}r dnd}t||gd d gD]V\}}|dk r"|dt|jkr"td |d t|jd |ddq"t|jD]T\}}| r||f}d}|j rtg}||jk}|r|sΐq|dk r||nd}|j rH|j rHfdd}tjj|||||||dk r*||nd|dk r>||ndd}n>||||||dk rd||nd|dk rx||nd|d }|d}r||rdndf7}r||df}|dk r||df}q| r||f}r|nd}| stdd|||||fDSt|||||dS)a  Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`): Features extracted from the speech or text input by the decoder prenet. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing cross-attention on hidden heads. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. NrrrrzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr#r{rzThe `z` should be specified for r~rcsfdd}|S)Ncs|fSrr#r)rrHrfr#r$rszNSpeechT5Decoder.forward..create_custom_forward..custom_forwardr#rrHrfrr$rsz6SpeechT5Decoder.forward..create_custom_forward)rHrcrdrFrerErHrfr rcss|]}|dk r|VqdSrr#rr#r#r$rsz*SpeechT5Decoder.forward..)rr/rrcross_attentions)rrHr|rfrr8rrrEr,r rrloggerZ warning_oncerrcrr!rr3r\ryrrrr)r~rrHrcrdr{rr/rfrHr|r}r0r.rrrZall_cross_attentionsZnext_decoder_cacheZ attn_maskZ mask_namerZ decoder_layerrrrErrZ next_cacher#rr$r2sI    $         zSpeechT5Decoder.forward) NNNNNNNNNNN)rrrrrrrrrr3rr rrrCrrrrrr#r#rr$ru s8   rucseZdZdZedfdd Zd eejeej eejeej eej eej eej ee ejee ee ee ee e eefd ddZZS) SpeechT5DecoderWithSpeechPrenetz Wrapper around SpeechT5Decoder that applies SpeechT5SpeechDecoderPrenet to convert log-mel filterbanks to hidden features. rcs2t|t||_t||_d|_|dSr)rqrrr rruwrapped_decoderrrzrrr#r$rrs    z(SpeechT5DecoderWithSpeechPrenet.__init__N) r&rHrcrdrr{rr/rfrHr|r}rJc  Cs0|||} |j| ||||||| | | | d }|SN) rrHrcrdr{rr/rfrHr|r}rr)r~r&rHrcrdrr{rr/rfrHr|r}decoder_hidden_statesr]r#r#r$rs z'SpeechT5DecoderWithSpeechPrenet.forward) NNNNNNNNNNNNrrrrrrrrr3rr rrrCrrrrrr#r#rr$rs:   rcseZdZdZedfdd ZddZddZd ee j ee j ee j ee j ee j ee j ee e j eeeeeeeeeeefd d d ZZS)SpeechT5DecoderWithTextPrenetz{ Wrapper around SpeechT5Decoder that applies SpeechT5TextDecoderPrenet to convert input tokens to hidden features. rcs2t|t||_t||_d|_|dSr)rqrrr+rrurrrzrrr#r$rr+s    z&SpeechT5DecoderWithTextPrenet.__init__cCs |jSrrrr#r#r$r'4sz2SpeechT5DecoderWithTextPrenet.get_input_embeddingscCs|j|dSrrr(r#r#r$r*7sz2SpeechT5DecoderWithTextPrenet.set_input_embeddingsN r&rHrcrdr{rr/rfrHr|r}rJc  Cs6||||\} }|j| |||||||| | | d } | Srr)r~r&rHrcrdr{rr/rfrHr|r}rr]r#r#r$r:sz%SpeechT5DecoderWithTextPrenet.forward) NNNNNNNNNNN)rrrrrrrr'r*rr3rr rrrCrrrrrr#r#rr$r&s:   rcseZdZdZedfdd Zd eejeej eejeej eej eej ee ejee ee ee ee e eefd ddZZS) SpeechT5DecoderWithoutPrenetrrcs(t|t||_d|_|dSr)rqrrrurrrzrrr#r$rras  z%SpeechT5DecoderWithoutPrenet.__init__Nrc Cs$|j||||||||| | | d } | Sr)r) r~r&rHrcrdr{rr/rfrHr|r}r]r#r#r$ris z$SpeechT5DecoderWithoutPrenet.forward) NNNNNNNNNNNrr#r#rr$r[s6   rcsVeZdZdZedfdd Zejejejej dddZ dd Z e d d Z ZS) $SpeechT5GuidedMultiheadAttentionLossz Guided attention loss from the paper [Efficiently Trainable Text-to-Speech System Based on Deep Convolutional Networks with Guided Attention](https://arxiv.org/abs/1710.08969), adapted for multi-head attention. rcst|j|_|j|_dSr)rqrrZguided_attention_loss_sigmasigmaZguided_attention_loss_scalescalerrr#r$rrs z-SpeechT5GuidedMultiheadAttentionLoss.__init__)r input_masks output_masksrJcCsX||||j}|d|d@}||jd}||}t||}|j|S)aY Compute the attention loss. Args: attentions (`torch.FloatTensor` of shape `(batch_size, layers * heads, output_sequence_length, input_sequence_length)`): Batch of multi-head attention weights input_masks (`torch.BoolTensor` of shape `(batch_size, input_sequence_length)`): Input attention mask as booleans. output_masks (`torch.BoolTensor` of shape `(batch_size, output_sequence_length)`): Target attention mask as booleans. Returns: `torch.Tensor` with the loss value rrr)_make_guided_attention_masksr-rr:r3rj masked_selectr)r~rrrguided_attn_masksmasksZlosseslossr#r#r$rs z,SpeechT5GuidedMultiheadAttentionLoss.forwardc Cs|d}|d}tjt||jd|jdf|d}tt||D]0\}\}} ||| |j|||d| d|f<qF| dS)Nrrr/) r^r3r<rcrrr_make_guided_attention_maskrr) r~rrr-rirrrZilenZolenr#r#r$rs   $&zASpeechT5GuidedMultiheadAttentionLoss._make_guided_attention_maskscCsdtjtj||dtj||ddd\}}||}||}dt||d d|dS)Nr/Zxy)ZindexingrBr)r3Zmeshgridr7rr)rNZ output_lengthrr-Zgrid_yZgrid_xr#r#r$rs     z@SpeechT5GuidedMultiheadAttentionLoss._make_guided_attention_mask)rrrrrrrr3rZ BoolTensorrrrrrrr#r#rr$rs  rc sTeZdZdZedfdd Zd ejejejejeje ejej dddZ Z S) SpeechT5SpectrogramLossz; Loss computation used by SpeechT5ForTextToSpeech. rcsPt|j|_|j|_|j|_t|_tt dd|_ |jrLt ||_ dS)Ng@)Z pos_weight) rqrruse_guided_attention_lossguided_attention_loss_num_headsr'r l1_criterionrr3r bce_criterionrattn_criterionrrr#r$rrs z SpeechT5SpectrogramLoss.__init__N)rHrr r!labelsrrJcs@|dk}||}||}||}||||}|dddddf} tj| dt| dd| jgdd} | ddddf| } || }|| } || } j r.) rrr3r;rdr8r:r-rrr'r)r~rHrr r!rrrZl1_lossr stop_labelsZbce_lossrZattnrrZ attn_lossr#rr$rs(    .   zSpeechT5SpectrogramLoss.forward)N) rrrrrrrr3r rrrrrr#r#rr$rsra This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`SpeechT5Config`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. encoder ([`SpeechT5EncoderWithSpeechPrenet`] or [`SpeechT5EncoderWithTextPrenet`] or `None`): The Transformer encoder module that applies the appropiate speech or text encoder prenet. If `None`, [`SpeechT5EncoderWithoutPrenet`] will be used and the `input_values` are assumed to be hidden states. decoder ([`SpeechT5DecoderWithSpeechPrenet`] or [`SpeechT5DecoderWithTextPrenet`] or `None`): The Transformer decoder module that applies the appropiate speech or text decoder prenet. If `None`, [`SpeechT5DecoderWithoutPrenet`] will be used and the `decoder_input_values` are assumed to be hidden states. aM This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`SpeechT5Config`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should **not** be passed to avoid degraded performance when doing batched inference. For such models `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly different results depending on whether `input_values` is padded or not. decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_values`. Causal mask will also be used by default. If you want to change padding behavior, you should read [`SpeechT5Decoder._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. head_mask (`torch.FloatTensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. decoder_head_mask (`torch.FloatTensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_values` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_values` of shape `(batch_size, sequence_length)`. decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `decoder_input_values` you can choose to directly pass an embedded representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be input (see `past_key_values`). This is useful if you want more control over how to convert `decoder_input_values` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zlThe bare SpeechT5 Encoder-Decoder Model outputting raw hidden-states without any specific pre- or post-nets.cseZdZdeeejeejdfdd ZddZddZ d d Z d d Z d dZ e eeeeddeejeejeejeejeejeejeejeeeejeeeejeeeejeeeeeeeeejefdddZZS) SpeechT5ModelN)rencoderdecodercsJt|||_|dkr"t|n||_|dkr8t|n||_|dSr)rqrrrrrrrrz)r~rrrrr#r$rrs  zSpeechT5Model.__init__cCs0t|jtr|jSt|jtr,|jSdSr)rNrrr'rrrr#r#r$r's     z"SpeechT5Model.get_input_embeddingscCs4t|jtr|j|t|jtr0|j|dSr)rNrrr*rrr(r#r#r$r*s   z"SpeechT5Model.set_input_embeddingscCs|jSr)rrr#r#r$ get_encoderszSpeechT5Model.get_encodercCs|jSr)rrr#r#r$ get_decoderszSpeechT5Model.get_decodercCst|jtr|jjdSz Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. N)rNrrrrrr#r#r$rs z$SpeechT5Model.freeze_feature_encoder output_typerw)r&rHdecoder_input_valuesdecoder_attention_maskr{decoder_head_maskrencoder_outputsr/rfrrHr|r}rJcCsj| dk r | n|jj} | dk r | n|jj} | dk r4| n|jj} |dk rH|n|jj}|dkrp|j|||| | |d}nH|rt|tst|dt|dkr|dndt|dkr|dndd}|dk rt|jt r|jj |dj d|}n|}t|j trd| i}ni}|j f|||d|||| | | | |d |}|sB||St|j|j|j|j|j|j|j|jd S) au input_values (`torch.Tensor` of shape `(batch_size, sequence_length)`): Depending on which encoder is being used, the `input_values` are either: float values of the input raw speech waveform, or indices of input sequence tokens in the vocabulary, or hidden states. decoder_input_values (`torch.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Depending on which decoder is being used, the `decoder_input_values` are either: float values of log-mel filterbank features extracted from the raw speech waveform, or indices of decoder input sequence tokens in the vocabulary, or hidden states. speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. Returns: N)r&rHr{rHr|r}rrrrr) r&rHrcrdr{rr/rfrHr|r})rr/rdecoder_attentionsrencoder_last_hidden_statercencoder_attentions)rrHr|rfrrrNr rcrrrrrrrrr/rrr)r~r&rHrrr{rrrr/rfrrHr|r}rdZ decoder_argsZdecoder_outputsr#r#r$rsp"    zSpeechT5Model.forward)NN)NNNNNNNNNNNNNN)rrrrrrModulerrr'r*rrrrSPEECHT5_INPUTS_DOCSTRINGrr_CONFIG_FOR_DOCr3rr rrrCrrrr#r#rr$r~sZ rz8SpeechT5 Model with a speech encoder and a text decoder.cs eZdZdgZedfdd ZddZddZd d Zd d Z d dZ e e e eeddeejeejeejeejeejeejeejeeeejeeeejeeeeeeeeeejeeefdddZdddZeddZZS)SpeechT5ForSpeechToTextz#text_decoder_postnet.lm_head.weightrcs\t||jdkr(td|jdt|}t|}t||||_t ||_ | dS)NYou are trying to instantiate a with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `SpeechT5ForSpeechToText.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.) rqrrr#r!rrrrrhr1text_decoder_postnetrz)r~rspeech_encoderZ text_decoderrr#r$rr s    z SpeechT5ForSpeechToText.__init__cCs |jSrrhrrr#r#r$r- sz#SpeechT5ForSpeechToText.get_encodercCs |jSrrhrrr#r#r$r0 sz#SpeechT5ForSpeechToText.get_decodercCs|jdSrrrrrr#r#r$r3 sz.SpeechT5ForSpeechToText.freeze_feature_encodercCs |jSr)rr5rr#r#r$r5: sz-SpeechT5ForSpeechToText.get_output_embeddingscCs|j|dSr)rr7r6r#r#r$r7= sz-SpeechT5ForSpeechToText.set_output_embeddingsrN)r&rHdecoder_input_idsrr{rrrr/rfrHr|r}rrJcCs| dk r | n|jj} |dk r8|dkr8t||jj|jj}|j||||||||| | | | dd }||d}d}|dk rt}||d|jj |d}| s|f|dd}|dk r|f|S|St |||j |j |j |j|j|j|jd S)ay input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (*pip install soundfile*). To prepare the array into `input_values`, the [`SpeechT5Processor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`SpeechT5Processor.__call__`] for details. decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`SpeechT5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) SpeechT5 uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Label indices can be obtained using [`SpeechT5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. Returns: Example: ```python >>> from transformers import SpeechT5Processor, SpeechT5ForSpeechToText >>> from datasets import load_dataset >>> dataset = load_dataset( ... "hf-internal-testing/librispeech_asr_demo", "clean", split="validation" ... ) # doctest: +IGNORE_RESULT >>> dataset = dataset.sort("id") >>> sampling_rate = dataset.features["audio"].sampling_rate >>> processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_asr") >>> model = SpeechT5ForSpeechToText.from_pretrained("microsoft/speecht5_asr") >>> # audio file is decoded on the fly >>> inputs = processor(audio=dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt") >>> predicted_ids = model.generate(**inputs, max_length=100) >>> # transcribe speech >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True) >>> transcription[0] 'mister quilter is the apostle of the middle classes and we are glad to welcome his gospel' ``` ```python >>> inputs["labels"] = processor(text_target=dataset[0]["text"], return_tensors="pt").input_ids >>> # compute loss >>> loss = model(**inputs).loss >>> round(loss.item(), 2) 19.68 ``` NT) r&rHrrr{rrrr/rfrHr|r}rrr) rr!r/rrrrrcr)rrr%rrrhrrr9r#rr/rrrrrcr)r~r&rHrrr{rrrr/rfrHr|r}rr]r!rZloss_fctoutputr#r#r$r@ sTRzSpeechT5ForSpeechToText.forwardc Ks2|dk r|ddddf}||||||||dS)Nr)rr/rrHr{rrrfr#) r~rr/rHr{rrrfrkwargsr#r#r$prepare_inputs_for_generation s z5SpeechT5ForSpeechToText.prepare_inputs_for_generationcs.d}|D] }|tfdd|Df7}q|S)Nr#c3s"|]}|d|jVqdS)rN)rr:r-)rUZ past_statebeam_idxr#r$r sz9SpeechT5ForSpeechToText._reorder_cache..)r)r/rZreordered_pastZ layer_pastr#rr$_reorder_cache s z&SpeechT5ForSpeechToText._reorder_cache)NNNNNNNNNNNNNN)NNNNNNN)rrrZ_tied_weights_keysrrrrrrr5r7rrrrrrr3rr rrrCrrrrrrr#r#rr$r sf   r?r)4@F) modelr&r threshold minlenratio maxlenratiovocoderoutput_cross_attentionsrJc Cst|}|jj||dd} | j} t|jjtrL|jjj| dj d|}t | d||j j } t | d||j j } | dd|j j} g}g}d}d}|d7}|jj| |}|jjj|ddddfd| ||d|dd}|r|tj|jdd|jd}|j}|j|}||j j |j j}||tj| |ddd|j jfdd} t|j|}|| krt t||kdks|| krtj|ddd}|j|}|d}qq|dk r||}n|}|rtj|d d}||f}|S) NT)r&rHr}rrr)rrHrcrdr/rfrHr}r1)rrr)r3Z ones_likerhrrrNrrrrrLr8rr'rr rrrer;rr/speech_decoder_postnetrr9Zsigmoidrr^rrsqueeze)rr&rrrrrrrdZ encoder_outrmaxlenZminlenZoutput_sequence spectrogramrr/rrZ decoder_outZlast_decoder_outputZspectrumZprobr]r#r#r$_generate_speech sh       $&    rz8SpeechT5 Model with a text encoder and a speech decoder.cs~eZdZdZedfdd ZddZddZee e e e d de eje eje eje eje eje eje eje eeeje eeeje ee ee ee ee eje eje ejeee fd d d Zedeje ejeeee ejeeejeejejffdddZedeje ejeeee ejeeejeejejffdddZZS)SpeechT5ForTextToSpeechrrcs\t||jdkr(td|jdt|}t|}t||||_t ||_ | dS)Nra with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `SpeechT5ForTextToSpeech.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.) rqrrr#r!rrrrrhrrrz)r~rZ text_encoderspeech_decoderrr#r$rrH s    z SpeechT5ForTextToSpeech.__init__cCs |jSrrrr#r#r$r\ sz#SpeechT5ForTextToSpeech.get_encodercCs |jSrrrr#r#r$r_ sz#SpeechT5ForTextToSpeech.get_decoderrN)rrHrrr{rrrr/rfrHr|r}rrrrJcCs | dk r | n|jj} |dk r(tdt|dk rR|dkrFt||jj}|jjrRd} |j||||||||| | || | dd}| |d\}}}d}|dk rt |j}|||||||j }| s|f|dd}|dk r|f|S|St |||j |j|j|j |j|j|jd S)a input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. The `batch_size` should be 1 currently. Indices can be obtained using [`SpeechT5Tokenizer`]. See [`~PreTrainedTokenizer.encode`] and [`~PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) decoder_input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_mel_bins)`): Float values of input mel spectrogram. SpeechT5 uses an all-zero spectrum as the starting token for `decoder_input_values` generation. If `past_key_values` is used, optionally only the last `decoder_input_values` have to be input (see `past_key_values`). speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. labels (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_mel_bins)`, *optional*): Float values of target mel spectrogram. Timesteps set to `-100.0` are ignored (masked) for the loss computation. Spectrograms can be obtained using [`SpeechT5Processor`]. See [`SpeechT5Processor.__call__`] for details. Returns: Example: ```python >>> from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan, set_seed >>> import torch >>> processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts") >>> model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts") >>> vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan") >>> inputs = processor(text="Hello, my dog is cute", return_tensors="pt") >>> speaker_embeddings = torch.zeros((1, 512)) # or load xvectors from a file >>> set_seed(555) # make deterministic >>> # generate speech >>> speech = model.generate(inputs["input_ids"], speaker_embeddings, vocoder=vocoder) >>> speech.shape torch.Size([15872]) ``` N^The argument `stop_labels` is deprecated and will be removed in version 4.30.0 of TransformersTr&rHrrr{rrrr/rfrrHr|r}rr rrr/rrrrrcr)rrwarningswarn FutureWarningr*r'rrhrrrrr/rrrrcr)r~rrHrrr{rrrr/rfrHr|r}rrrr]rr r!r criterionrr#r#r$rb sjA  zSpeechT5ForTextToSpeech.forwardrr)rF)rrrrrrrrJc Kst||||||||S)a Converts a sequence of input tokens into a sequence of mel spectrograms, which are subsequently turned into a speech waveform using a vocoder. Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. The `batch_size` should be 1 currently. Indices can be obtained using [`SpeechT5Tokenizer`]. See [`~PreTrainedTokenizer.encode`] and [`~PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. threshold (`float`, *optional*, defaults to 0.5): The generated sequence ends when the predicted stop token probability exceeds this value. minlenratio (`float`, *optional*, defaults to 0.0): Used to calculate the minimum required length for the output sequence. maxlenratio (`float`, *optional*, defaults to 20.0): Used to calculate the maximum allowed length for the output sequence. vocoder (`nn.Module`, *optional*): The vocoder that converts the mel spectrogram into a speech waveform. If `None`, the output is the mel spectrogram. output_cross_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of the decoder's cross-attention layers. Returns: `tuple(torch.FloatTensor)` comprising various elements depending on the inputs: - **spectrogram** (*optional*, returned when no `vocoder` is provided) `torch.FloatTensor` of shape `(output_sequence_length, config.num_mel_bins)` -- The predicted log-mel spectrogram. - **waveform** (*optional*, returned when a `vocoder` is provided) `torch.FloatTensor` of shape `(num_frames,)` -- The predicted speech waveform. - **cross_attentions** (*optional*, returned when `output_cross_attentions` is `True`) `torch.FloatTensor` of shape `(config.decoder_layers, config.decoder_attention_heads, output_sequence_length, input_sequence_length)` -- The outputs of the decoder's cross-attention layers. r) r~rrrrrrrrr#r#r$generate s0z SpeechT5ForTextToSpeech.generatec Cst||||||||S)a Converts a sequence of input tokens into a sequence of mel spectrograms, which are subsequently turned into a speech waveform using a vocoder. Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. The `batch_size` should be 1 currently. Indices can be obtained using [`SpeechT5Tokenizer`]. See [`~PreTrainedTokenizer.encode`] and [`~PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. threshold (`float`, *optional*, defaults to 0.5): The generated sequence ends when the predicted stop token probability exceeds this value. minlenratio (`float`, *optional*, defaults to 0.0): Used to calculate the minimum required length for the output sequence. maxlenratio (`float`, *optional*, defaults to 20.0): Used to calculate the maximum allowed length for the output sequence. vocoder (`nn.Module`, *optional*, defaults to `None`): The vocoder that converts the mel spectrogram into a speech waveform. If `None`, the output is the mel spectrogram. output_cross_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of the decoder's cross-attention layers. Returns: `tuple(torch.FloatTensor)` comprising various elements depending on the inputs: - **spectrogram** (*optional*, returned when no `vocoder` is provided) `torch.FloatTensor` of shape `(output_sequence_length, config.num_mel_bins)` -- The predicted log-mel spectrogram. - **waveform** (*optional*, returned when a `vocoder` is provided) `torch.FloatTensor` of shape `(num_frames,)` -- The predicted speech waveform. - **cross_attentions** (*optional*, returned when `output_cross_attentions` is `True`) `torch.FloatTensor` of shape `(config.decoder_layers, config.decoder_attention_heads, output_sequence_length, input_sequence_length)` -- The outputs of the decoder's cross-attention layers. r)r~rrrrrrrr#r#r$generate_speech s/z'SpeechT5ForTextToSpeech.generate_speech)NNNNNNNNNNNNNNNN)Nrr)rNF)Nrr)rNF)rrrrxrrrrrrrrrrrr3r rrrrCrrrrrrrrrr#r#rr$rA s  |:rz:SpeechT5 Model with a speech encoder and a speech decoder.cs$eZdZedfdd ZddZddZdd Zee e e e d de eje eje eje eje eje eje eje eeeje eeeje ee ee ee ee eje eje ejeee fd d dZedeje ejeeee ejeejdddZZS)SpeechT5ForSpeechToSpeechrcs@t|t|}t|}t||||_t||_|dSr) rqrrrrrrhrrrz)r~rrrrr#r$rr[ s   z"SpeechT5ForSpeechToSpeech.__init__cCs |jSrrrr#r#r$rg sz%SpeechT5ForSpeechToSpeech.get_encodercCs |jSrrrr#r#r$rj sz%SpeechT5ForSpeechToSpeech.get_decodercCs|jdSrrrr#r#r$rm sz0SpeechT5ForSpeechToSpeech.freeze_feature_encoderrN)r&rHrrr{rrrr/rfrHr|r}rrrrJcCs| dk r | n|jj} |dk r(tdt|dk rF|dkrFt||jj}|j||||||||| | || | dd}||d\}}}d}| s|f|dd}|dk r|f|S|St |||j |j |j |j |j|j|jd S)a input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (*pip install soundfile*). To prepare the array into `input_values`, the [`SpeechT5Processor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`SpeechT5Processor.__call__`] for details. decoder_input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_mel_bins)`): Float values of input mel spectrogram. SpeechT5 uses an all-zero spectrum as the starting token for `decoder_input_values` generation. If `past_key_values` is used, optionally only the last `decoder_input_values` have to be input (see `past_key_values`). speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. labels (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_mel_bins)`, *optional*): Float values of target mel spectrogram. Spectrograms can be obtained using [`SpeechT5Processor`]. See [`SpeechT5Processor.__call__`] for details. Returns: Example: ```python >>> from transformers import SpeechT5Processor, SpeechT5ForSpeechToSpeech, SpeechT5HifiGan, set_seed >>> from datasets import load_dataset >>> import torch >>> dataset = load_dataset( ... "hf-internal-testing/librispeech_asr_demo", "clean", split="validation" ... ) # doctest: +IGNORE_RESULT >>> dataset = dataset.sort("id") >>> sampling_rate = dataset.features["audio"].sampling_rate >>> processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_vc") >>> model = SpeechT5ForSpeechToSpeech.from_pretrained("microsoft/speecht5_vc") >>> vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan") >>> # audio file is decoded on the fly >>> inputs = processor(audio=dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt") >>> speaker_embeddings = torch.zeros((1, 512)) # or load xvectors from a file >>> set_seed(555) # make deterministic >>> # generate speech >>> speech = model.generate_speech(inputs["input_values"], speaker_embeddings, vocoder=vocoder) >>> speech.shape torch.Size([77824]) ``` NrTrrrr)rrrrrr*r'rhrrr/rrrrrcr)r~r&rHrrr{rrrr/rfrHr|r}rrrr]rVrr!rrr#r#r$rt sRGz!SpeechT5ForSpeechToSpeech.forwardrr)rF)r&rrrrrrrJc Cs.|dkrtjd|jd}t||||||||S)av Converts a raw speech waveform into a sequence of mel spectrograms, which are subsequently turned back into a speech waveform using a vocoder. Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. The `batch_size` should be 1 currently. Values can be obtained by loading a *.flac* or *.wav* audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (*pip install soundfile*). To prepare the array into `input_values`, the [`SpeechT5Processor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`SpeechT5Processor.__call__`] for details. speaker_embeddings (`torch.FloatTensor` of shape `(batch_size, config.speaker_embedding_dim)`, *optional*): Tensor containing the speaker embeddings. threshold (`float`, *optional*, defaults to 0.5): The generated sequence ends when the predicted stop token probability exceeds this value. minlenratio (`float`, *optional*, defaults to 0.0): Used to calculate the minimum required length for the output sequence. maxlenratio (`float`, *optional*, defaults to 20.0): Used to calculate the maximum allowed length for the output sequence. vocoder (`nn.Module`, *optional*, defaults to `None`): The vocoder that converts the mel spectrogram into a speech waveform. If `None`, the output is the mel spectrogram. output_cross_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of the decoder's cross-attention layers. Returns: `tuple(torch.FloatTensor)` comprising various elements depending on the inputs: - **spectrogram** (*optional*, returned when no `vocoder` is provided) `torch.FloatTensor` of shape `(output_sequence_length, config.num_mel_bins)` -- The predicted log-mel spectrogram. - **waveform** (*optional*, returned when a `vocoder` is provided) `torch.FloatTensor` of shape `(num_frames,)` -- The predicted speech waveform. - **cross_attentions** (*optional*, returned when `output_cross_attentions` is `True`) `torch.FloatTensor` of shape `(config.decoder_layers, config.decoder_attention_heads, output_sequence_length, input_sequence_length)` -- The outputs of the decoder's cross-attention layers. N)rir/)r3r<r-r)r~r&rrrrrrr#r#r$r s/z)SpeechT5ForSpeechToSpeech.generate_speech)NNNNNNNNNNNNNNNN)Nrr)rNF)rrrrrrrrrrrrrrrr3rr rrrCrrrrrrrrr#r#rr$rV st   vraT This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`SpeechT5HifiGanConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. cs@eZdZdfdd ZdddZd d Zd d Zd dZZS)HifiGanResidualBlockr rr 皙?csbt|_tfddttD_tfddttD_dS)Nc s2g|]*}tjd||dqSr)rodilationrrrv get_paddingrchannelsrrnr~r#r$rWA s z1HifiGanResidualBlock.__init__..c s*g|]"}tjddddqSrrrT)rrnr~r#r$rWN s  ) rqrrleaky_relu_sloperrrarcconvs1convs2)r~rrnrrrrr$rr< s    zHifiGanResidualBlock.__init__rcCs|||dSrr#)r~rnrr#r#r$r[ sz HifiGanResidualBlock.get_paddingcCs4|jD]}tj|q|jD]}tj|qdSr)rrrrrr~rr#r#r$apply_weight_norm^ s  z&HifiGanResidualBlock.apply_weight_normcCs4|jD]}tj|q|jD]}tj|qdSr)rrrremove_weight_normrrr#r#r$r d s  z'HifiGanResidualBlock.remove_weight_normcCsXt|j|jD]D\}}|}tj||j}||}tj||j}||}||}q|Sr)rrrrr leaky_relur)r~rZconv1Zconv2r\r#r#r$rj s zHifiGanResidualBlock.forward)r rr)r) rrrrrrr r rrr#r#rr$r; s  rzHiFi-GAN vocoder.csVeZdZeZdZedfdd ZddZddZd d Z e j e j d d d Z Z S)SpeechT5HifiGanrrc sNt|t|j|_t|j|_tj|j |j dddd|_ t |_ tt|j|jD]H\}\}}|j tj|j d||j d|d||||ddqZt |_tt|j D]F}|j d|d}t|j|jD] \}}|jt||||jqqtj|ddddd|_|dt|j |dt|j |dS)Nrr )rnrorrrjr)rqrrrcZresblock_kernel_sizes num_kernelsZupsample_rates num_upsamplesrrvZ model_in_dimZupsample_initial_channelconv_prer upsamplerrrZupsample_kernel_sizesreZConvTranspose1d resblocksraZresblock_dilation_sizesrr conv_postrr3r<rdrz)r~rrZ upsample_raternrrrr#r$rr} s<       zSpeechT5HifiGan.__init__cCsBt|tjtjfr>|jjjd|jjd|j dk r>|j j dS)zInitialize the weights.r)riN) rNrrrvrrornrrprprq)r~rr#r#r$rs s zSpeechT5HifiGan._init_weightscCsLtj|j|jD]}tj|q|jD] }|q,tj|jdSr)rrrrrrr rrr#r#r$r  s    z!SpeechT5HifiGan.apply_weight_normcCsLtj|j|jD]}tj|q|jD] }|q,tj|jdSr)rrr rrrrrr#r#r$r  s    z"SpeechT5HifiGan.remove_weight_norm)rrJcCs|jjr||j|j}|dk}|s2|d}|dd}||}t|j D]p}t j ||jj }|j||}|j||j|}td|jD] }||j||j||7}q||j}qRt j |}||}t|}|s|dddd}n |d}|S)a Converts a log-mel spectrogram into a speech waveform. Passing a batch of log-mel spectrograms returns a batch of speech waveforms. Passing a single, un-batched log-mel spectrogram returns a single, un-batched speech waveform. Args: spectrogram (`torch.FloatTensor`): Tensor containing the log-mel spectrograms. Can be batched and of shape `(batch_size, sequence_length, config.model_in_dim)`, or un-batched and of shape `(sequence_length, config.model_in_dim)`. Returns: `torch.FloatTensor`: Tensor containing the speech waveform. If the input spectrogram is batched, will be of shape `(batch_size, num_frames,)`. If un-batched, will be of shape `(num_frames,)`. r rrrr)rZnormalize_beforerjrr2rrrrarrrr rrrrrr3tanhrr9)r~rZ is_batchedrrZ res_statejZwaveformr#r#r$r s*         zSpeechT5HifiGan.forward)rrrrrwrxrrrsr r r3rrrr#r#rr$r u s&r )r)r)N)Nr)Nrr)rNF)frrrtypingrrrrnumpyrZr3Ztorch.utils.checkpointrZtorch.nnrrr Z activationsr Zintegrations.deepspeedr Zmodeling_outputsr rrrrZmodeling_utilsrrrrrrZconfiguration_speecht5rrZ get_loggerrrZ_HIDDEN_STATES_START_POSITIONrZ&SPEECHT5_PRETRAINED_MODEL_ARCHIVE_LISTrrLr%r*Sizer,r-rArErr Zndarrayrkrrlrrrrrrrrrrr rrr"r+r1r8rKrTr^rgrtrrrrurrrrrZSPEECHT5_BASE_START_DOCSTRINGZSPEECHT5_START_DOCSTRINGrrrrrCrrrZHIFIGAN_START_DOCSTRINGrr r#r#r#r$s         xG' 64(+!=l.&+!l15,;=VT [Q: