U ,-eR@szdZddlZddlZddlmZddlmZmZmZm Z m Z ddl Z ddl m mZddl m Z ddlmZddlmZmZmZdd lmZdd lmZmZmZmZdd lmZmZm Z m!Z!m"Z"d d l#m$Z$m%Z%m&Z&e!'e(Z)dZ*ddgZ+ddZ,ddZ-ddZ.d_ddZ/e j0e j0dddZ1eGdddeZ2eGdddeZ3eGdd d eZ4Gd!d"d"e j5Z6Gd#d$d$e j5Z7Gd%d&d&e j5Z8Gd'd(d(e j5Z9Gd)d*d*e j5Z:Gd+d,d,e j5Z;Gd-d.d.e j5ZGd3d4d4e j5Z?Gd5d6d6e j5Z@Gd7d8d8e j5ZAd9ZBd:ZCd;ZDdd>e j5ZFGd?d@d@e j5ZGGdAdBdBe j5ZHGdCdDdDe j5ZIGdEdFdFe j5ZJGdGdHdHe j5ZKGdIdJdJe j5ZLGdKdLdLe j5ZMGdMdNdNe j5ZNGdOdPdPe j5ZOGdQdRdReZPGdSdTdTePZQGdUdVdVePZReeBGdWdXdXePZSedYeBGdZd[d[ePZTed\eBGd]d^d^ePZUdS)`z PyTorch CLAP model.N) dataclass)AnyListOptionalTupleUnion)nn)ACT2FN))BaseModelOutputWithPastAndCrossAttentionsBaseModelOutputWithPooling,BaseModelOutputWithPoolingAndCrossAttentions)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesmeshgridprune_linear_layer) ModelOutputadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)ClapAudioConfig ClapConfigClapTextConfigzlaion/clap-htsat-fusedzlaion/clap-htsat-unfusedcCsJ|j\}}}|dddddddfdd|d}|||||}|S)ae Interpolate data in time domain. This is used to compensate the resolution reduction in downsampling of a CNN. Args: hidden_states (`torch.FloatTensor` of shape (batch_size, time_length, classes_num)): Input hidden states ratio (`int`): The ratio of the length of the output to the length of the input. Nr)shaperepeatreshape) hidden_statesratio batch_size time_lengthZ classes_numZ upsampledr#g/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/clap/modeling_clap.py interpolate7s (r%cCsR|j\}}}}|||||||||}|ddddddd|||}|S)aR Returns the resized hidden states. The output shape should be `(batch_size * num_windows, window_size, window_size, num_channels)` Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, height, width, num_channels)`): Input hidden states window_size (`int`): Window size rrr )rviewpermute contiguous)r window_sizer!heightwidth num_channelswindowsr#r#r$window_partitionHs $r2cCsbt|jd||||}||||||||d}|dddddd|||d}|S)aQ Args: windows (`torch.FloatTensor` of shape `(num_windows * batch_size, window_size, window_size, num_channels)`): Input windows window_size (`int`): Window size height (`int`): Height of the resized audio width (`int`): Width of the resized audio rr)rr r&r'r()intrr*r+r,)r1r-r.r/r!rr#r#r$window_reverse]s $r4cCs6||}tj|dd|||}||S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rdim)ner3torchZcumsumZtype_aslong) input_ids padding_idxpast_key_values_lengthmaskZincremental_indicesr#r#r$"create_position_ids_from_input_idsqs r>)logitsreturncCs"tjt||jd}tj||S)Ndevice)r8arangelenrBr functionalZ cross_entropy)r?labelsr#r#r$contrastive_losssrGc@s^eZdZUdZdZeejed<dZ ejed<dZ ee ejed<dZ ee ejed<dS)ClapTextModelOutputa Base class for text model's outputs that also contains a pooling of the last hidden states. Args: text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): The text embeddings obtained by applying the projection layer to the pooler_output. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. N text_embedslast_hidden_stater attentions) __name__ __module__ __qualname____doc__rIrr8 FloatTensor__annotations__rJrrrKr#r#r#r$rHs rHc@s^eZdZUdZdZeejed<dZ ejed<dZ ee ejed<dZ ee ejed<dS)ClapAudioModelOutputak ClapAudio model output to mimic the output of the original implementation. Args: audio_embeds (`torch.FloatTensor` of shape `(batch_size, hidden_size)`): The Audio embeddings obtained by applying the projection layer to the pooler_output. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. N audio_embedsrJrrK) rLrMrNrOrSrr8rPrQrJrrrKr#r#r#r$rRs rRc@seZdZUdZdZeejed<dZ ejed<dZ ejed<dZ ejed<dZ ejed<dZ eed<dZeed <eed d d ZdS) ClapOutputa Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`): Contrastive loss for audio-text similarity. logits_per_audio:(`torch.FloatTensor` of shape `(audio_batch_size, text_batch_size)`): The scaled dot product scores between `audio_embeds` and `text_embeds`. This represents the audio-text similarity scores. logits_per_text:(`torch.FloatTensor` of shape `(text_batch_size, audio_batch_size)`): The scaled dot product scores between `text_embeds` and `audio_embeds`. This represents the text-audio similarity scores. text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`ClapTextModel`]. audio_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`): The audio embeddings obtained by applying the projection layer to the pooled output of [`ClapAudioModel`]. text_model_output(`BaseModelOutputWithPooling`): The output of the [`ClapTextModel`]. audio_model_output(`BaseModelOutputWithPooling`): The output of the [`ClapAudioModel`]. Nlosslogits_per_audiologits_per_textrIrStext_model_outputaudio_model_outputr@cstfddDS)Nc3s,|]$}|dkr|n t|VqdS))rXrYN)getattrto_tuple).0kselfr#r$ sz&ClapOutput.to_tuple..)tuplekeysr_r#r_r$r\s zClapOutput.to_tuple)rLrMrNrOrUrr8rPrQrVrWrIrSrXr rYrrr\r#r#r#r$rTs   rTcs*eZdZdZdfdd ZddZZS) ClapDropPathz Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). This is a slightly refactored version of the `SwinDropPath` implementation. Ncst||_dSN)super__init__ drop_prob)r`rh __class__r#r$rgs zClapDropPath.__init__cCsj|jdks|js|Sd|j}|jdfd|jd}|tj||j|jd}|| ||}|S)Nrr)rdtyperB) rhtrainingrndimr8ZrandrmrBZfloor_div)r`rZ keep_probrZ random_tensoroutputr#r#r$forwards zClapDropPath.forward)N)rLrMrNrOrgrr __classcell__r#r#rir$rdsrdcs.eZdZdZedfdd ZddZZS)ClapAudioAFFBlockz ATTENTIONAL FEATURE FUSION Block from CLAP, since in CLAP we are always in 2D mode, it is not needed to implement the 1D version. configc st|j}|j}t||}ttj||ddddt|tj ddtj||ddddt||_ tt dtj||ddddt|tj ddtj||ddddt||_ t |_dS)NrrZ kernel_sizeZstridepaddingT)Zinplace)rfrgpatch_embeds_hidden_sizeZ aff_block_rr3rZ SequentialConv2d BatchNorm2dZReLU local_attZAdaptiveAvgPool2d global_attZSigmoidsigmoid)r`rvchannelsZdownsize_ratioZinter_channelsrir#r$rg s(     zClapAudioAFFBlock.__init__cCsF||}||||}||}d||d|d|}|S)Nr&r)r|r}r~)r`rZresidualZattention_inputZfused_layer_outputrqr#r#r$rr!s  zClapAudioAFFBlock.forwardrLrMrNrOrrgrrrsr#r#rir$rtsrtcs0eZdZdZedfdd ZdddZZS) ClapAudioPatchEmbedz This module converts the hidden states reshaped as an image to patch embeddings ready to be passed to the Transformer block. rucstt|jtr"|j|jfn|j}t|jtr@|j|jfn|j}t|jtr^|j|jfn|j}||_||_|d|d|d|df|_|jd|jd|_ |j |_ |j |_ |d|dd|d|ddf}|j r|j dkrdnd}tj|j||j|||d|_|jr*t|jnt|_|j rt||_tj|j|j|d|ddf|d|ddf|d|_dS)Nrrr&Z channel_mapr'rwr )rfrg isinstance spec_sizer3 patch_size patch_strideimg_size grid_sizeZ num_patchesZflatten_patch_embedsflatten enable_fusionZ fusion_typerrzZpatch_embed_input_channelsryprojZenable_patch_layer_norm LayerNormIdentitynormrt fusion_model mel_conv2d)r`rvrrrrxZ scale_factorrir#r$rg1s> "( zClapAudioPatchEmbed.__init__Nc Cs|jrb|ddddddddf}|j\}}}}||jdksR||jdkrtd|d|d|jdd|jdd ||}|d}t|dkr\||ddddddf} | j\}}}}| ||d||} | | } | j\} } }}| ||| ||} | d d } | d} t j j| d|| fd d} |||| ||<|}nf|j\} } }}||jdks||jdkrtd|d|d|jdd|jdd ||}|j r| d dd }||}|S) NrrzInput audio size (*z) doesn't match model (z).r))rr&r rr'r Zconstantr&)rrr ValueErrorrsizerDr,r*rr+rr8rrEpadr transposer) r`rZ is_longer_idxZglobal_hidden_statesr!r0r.r/Z output_widthZlocal_hidden_states_featuresZ local_widthr#r#r$rr[sN (       (  zClapAudioPatchEmbed.forward)Nrr#r#rir$r+s*rcsTeZdZfddZddZd ejeejeejee e ejddd Z Z S) ClapAudioSelfAttentionc s t||dkr,td|d|d||_t|||_|j|j|_t|tj j r`|n||f|_ t td|j ddd|j dd||_t|j d}t|j d}tt||gdd}t|d}|dddddf|dddddf} | ddd} | dddddf|j dd7<| dddddf|j dd7<| dddddfd|j dd9<| d } |d | t j|j|j|jd |_t j|j|j|jd |_t j|j|j|jd |_t |j|_ dS) NrThe hidden size (6) is not a multiple of the number of attention heads ()r&rZij)Zindexingr)relative_position_indexbias)!rfrgrnum_attention_headsr3attention_head_size all_head_sizer collectionsabcIterabler-r Parameterr8zerosrelative_position_bias_tablerCstackrrr+r,sumregister_bufferLinearZqkv_biasquerykeyvalueDropoutattention_probs_dropout_probdropout) r`rvr6 num_headsr-Zcoords_hZcoords_wZcoordsZcoords_flattenZrelative_coordsrrir#r$rgs8  * ,((,  zClapAudioSelfAttention.__init__cCs6|dd|j|jf}||}|ddddSNr)rr&rr rrrr*r+r`xZ new_x_shaper#r#r$transpose_for_scoress z+ClapAudioSelfAttention.transpose_for_scoresNFrattention_mask head_maskoutput_attentionsr@cCs|j\}}}||}|||} |||} ||} t| | dd} | t |j } |j |j d} | |jd|jd|jd|jdd} | ddd} | | d} |dk r|jd}| ||||j||} | |dd} | d|j||} tjj| dd}||}|dk rB||}t|| }|dddd}|dd|jf}| |}|r||fn|f}|S)Nr)rrr&r5r )rrrrrr8matmulrmathsqrtrrrr*r-r+r, unsqueezerrrEsoftmaxrrr)r`rrrrr!r6r0mixed_query_layer key_layer value_layer query_layerattention_scoresZrelative_position_biasZ mask_shapeattention_probs context_layernew_context_layer_shapeoutputsr#r#r$rrsH         zClapAudioSelfAttention.forward)NNF) rLrMrNrgrr8TensorrrPboolrrrrsr#r#rir$rs %rcs4eZdZfddZejejejdddZZS)ClapAudioSelfOutputcs*tt|||_t|j|_dSre)rfrgrrdenserrrr`rvr6rir#r$rgs zClapAudioSelfOutput.__init__r input_tensorr@cCs||}||}|Srerrr`rrr#r#r$rrs  zClapAudioSelfOutput.forwardrLrMrNrgr8rrrrsr#r#rir$rs rcsTeZdZfddZddZd ejeejeejee e ejddd Z Z S) ClapAudioAttentioncs2tt|||||_t|||_t|_dSre)rfrgrr`rrqset pruned_heads)r`rvr6rr-rir#r$rgs  zClapAudioAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dSNrrr5rDrr`rrrrrrrrqrrunionr`Zheadsindexr#r#r$ prune_headss zClapAudioAttention.prune_headsNFrcCs6|||||}||d|}|f|dd}|SNrrr`rq)r`rrrr self_outputsattention_outputrr#r#r$rrszClapAudioAttention.forward)NNF) rLrMrNrgrr8rrrPrrrrrsr#r#rir$rs rcs0eZdZfddZejejdddZZS)ClapAudioIntermediatecsHtt|t|j||_t|jt rs  zClapAudioOutput.forwardrr#r#rir$r8s rc steZdZdfdd ZddZddZdd Zdeje e e fe ej e e e e e ejejfd d dZZS)ClapAudioLayerrcst|j|_||_|j|_||_tj||jd|_ t ||||jd|_ |j dkr`t |j nt|_tj||jd|_t|||_t|||_dS)NZeps)r-rk)rfrgchunk_size_feed_forward shift_sizer-input_resolutionrrlayer_norm_epslayernorm_beforer attentiondrop_path_raterdr drop_pathlayernorm_afterr intermediaterrq)r`rvr6rrrrir#r$rgFs  zClapAudioLayer.__init__cCs"t||jkrd|_t||_dSNr)minr-r)r`rr#r#r$set_shift_and_window_sizeSsz(ClapAudioLayer.set_shift_and_window_sizec Cs|jdkr tjd||df|d}td|j t|j |j t|j df}td|j t|j |j t|j df}d}|D].}|D]$} ||dd|| ddf<|d7}qqt||j} | d|j|j} | d| d} | | dkt d| dkt d} nd} | S)Nrrrmr)r&gYrk) rr8rslicer-r2r*rZ masked_fillfloat) r`r.r/rmZimg_maskZ height_slicesZ width_slicescountZ height_sliceZ width_sliceZ mask_windows attn_maskr#r#r$ get_attn_maskYs*       &zClapAudioLayer.get_attn_maskcCsR|j||j|j}|j||j|j}ddd|d|f}tj||}||fSr)r-rrEr)r`rr.r/ pad_rightZ pad_bottom pad_valuesr#r#r$ maybe_padus zClapAudioLayer.maybe_padNFrinput_dimensionsrralways_partitionr@cCs|s||n|\}}|\}} } |} ||}||||| }||||\}} |j\} } }} |jdkrtj||j |j fdd}n|}t ||j }|d|j |j | }|j | ||j d}|dk r| |j}|j||||d}|d}|d|j |j | }t||j | |}|jdkrsz+ClapAudioStage.__init__..)r6 norm_layerF) rfrgrvr6r ModuleListrangeblocksr downsampleZpointing)r`rvr6rdepthrrrrir r$rgs zClapAudioStage.__init__NFrcCs|\}}t|jD]4\}} |dk r*||nd} | ||| ||} | d}q|} |jdk r|dd|dd} }||| |f}|| |}n ||||f}|| |f}|r|| dd7}|S)Nrrr&) enumeraterr)r`rrrrrr.r/r  layer_modulelayer_head_maskr !hidden_states_before_downsamplingZheight_downsampledZwidth_downsampledoutput_dimensionsZ stage_outputsr#r#r$rrs*     zClapAudioStage.forward)NFF) rLrMrNrgr8rrr3rrPrrrrsr#r#rir$r s  r cs^eZdZdZejfeeeejddfdd Z ddZ e j eeefe j dd d Z ZS) ClapAudioPatchMerginga' Patch Merging Layer. Args: input_resolution (`Tuple[int]`): Resolution of input feature. dim (`int`): Number of input channels. norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`): Normalization layer class. N)rr6rr@csBt||_||_tjd|d|dd|_|d||_dS)Nr'r&Fr)rfrgrr6rr reductionr)r`rr6rrir#r$rg s  zClapAudioPatchMerging.__init__cCsF|ddkp|ddk}|rBddd|dd|df}tj||}|S)Nr&rr)rrEr)r` input_featurer.r/Z should_padrr#r#r$rs zClapAudioPatchMerging.maybe_pad)rrr@c Cs|\}}|j\}}}|||||}||||}|ddddddddddf}|ddddddddddf} |ddddddddddf} |ddddddddddf} t|| | | gd}||dd|}||}||}|S)Nrr&rr)r')rr*rr8catrr) r`rrr.r/r!r6r0Zinput_feature_0Zinput_feature_1Zinput_feature_2Zinput_feature_3r#r#r$rrs $$$$  zClapAudioPatchMerging.forward)rLrMrNrOrrrr3Modulergrr8rrrrsr#r#rir$rs $rc sjeZdZfddZddZd eejeejeeeeeeeeeee e e fdd d Z Z S) ClapAudioEncodercsttj__t_j_jj _ j _ j j _ t jdjd_ddtdjtjDjjfddtjD_tfddtjD_d_tj _tj_j_td_ dS) Nr&rcSsg|] }|qSr#)item)r]rr#r#r$rAsz-ClapAudioEncoder.__init__..rcs,g|]$}dd|dd|fqS)rr&rr#r )rr#r$rDscs|g|]t}ttjd|j|j|j|tjd|tjd|d|jdkrptnddqS)r&Nr)rvr6rrrrr) r r3ryinput_resolutionsdepthsrr num_layersr)r]Zi_layer)rvrr`r#r$rGs *F)!rfrgrDr"r#rvr patch_embedrrrZ num_mel_bins freq_ratior3ryZ num_featuresr8Zlinspacerrrrr!rrlayersgradient_checkpointingr{ batch_normrrZAdaptiveAvgPool1davgpoolr`rvri)rvrrr`r$rg4s,      zClapAudioEncoder.__init__c Cs|j\}}}}t|j|j}|j|j}||ks:||krBtd||krbtjj|||fddd}||krtjj|||fddd}|j\}}} } ||||j| |j| }| dddd }|||| |j| |j}|S) z The input is 4 normalized log mel spectrograms. It is reshape to the common shape of images. Each channel should represent 1 of the 4 crops of the spectrogram. For more details, refer to the [`ClapFeatureExtractor`]. z@the wav size should be less than or equal to the swin input sizeZbicubicT)modeZ align_cornersrrr r&) rr3rr%rrrEr%rr+r,) r`normalized_input_featuresrr"Z freq_lengthZ spec_widthZ spec_heigthbatchrtimefreqr#r#r$reshape_mel2img\sD z ClapAudioEncoder.reshape_mel2imgNFT) is_longerrroutput_hidden_states(output_hidden_states_before_downsamplingr return_dictr@c %sX|dd}||} | dd} d} |jrJ||j} t| dkd} || } | jd} | | | } |rrdnd}|r~dnd}rdnd}|j d}|r| j\}}}| j |f||f}| dddd}|| f7}||f7}t |jD]X\}}|dk r||nd}|j |}|jrJ|jrJfdd}tjj||| ||}n|| |||}|d} |d}|d}|d|d f}|r|r|j\}}}|j |f|d|df|f}| dddd}||f7}||f7}nP|r0|s0| j\}}}| j |f||f}| dddd}|| f7}||f7}r||dd7}q|| }|j\}}}| dt|jd|jd}| dt|jd|jd} | ddd|||| }|j\}}}!}"|!|j}#||||!|#|#|"}| ddddd |||#d }|t|d}$t|$d}$|sHtd d ||$||fDSt||$||d S)Nrr rr&r#csfdd}|S)Ncs|fSrer#inputs)modulerr#r$custom_forwardszOClapAudioEncoder.forward..create_custom_forward..custom_forwardr#r7r8rr7r$create_custom_forwardsz7ClapAudioEncoder.forward..create_custom_forwardrr)r'css|]}|dk r|VqdSrer#r]vr#r#r$rasz+ClapAudioEncoder.forward..)rJ pooler_outputrrK)rr(rrrBr8wherer0rr$r!r*r+rr&r'rnutils checkpointrrDr"rr,rr%r)rrbr )%r`input_featuresr1rrr2r3rr4r,Zis_longer_list_idxZis_longer_listrZ frames_numall_hidden_statesZall_reshaped_hidden_statesall_self_attentionsrr!r hidden_sizeZreshaped_hidden_stater rrr;r rrrJZ n_channelsZ freq_shapeZtemporal_shapeZ n_frequenciesZn_tempZ c_freq_binZ latent_outputr#rr$rrs                                zClapAudioEncoder.forward)NNFFFFT)rLrMrNrgr0rr8rPrrrrRrrrsr#r#rir$r3s& (' ra= This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`ClapConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. a6 Args: input_features (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Input audio features. This should be returnes by the [`ClapFeatureExtractor`] class that you can also retrieve from [`AutoFeatureExtractor`]. See [`ClapFeatureExtractor.__call__`] for details. is_longer (`torch.FloatTensor`, of shape `(batch_size, 1)`, *optional*): Whether the audio clip is longer than `max_length`. If `True`, a feature fusion will be enabled to enhance the features. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. a$ Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) input_features (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Input audio features. This should be returnes by the [`ClapFeatureExtractor`] class that you can also retrieve from [`AutoFeatureExtractor`]. See [`ClapFeatureExtractor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cs2eZdZeeefdfdd ZddZZS)ClapProjectionLayerrucsHt||_|j}|j}t|||_t|j |_ t|||_ dSre) rfrgrvrEprojection_dimrrlinear1r Zprojection_hidden_act activationlinear2)r`rvrErGrir#r$rgls  zClapProjectionLayer.__init__cCs"||}||}||}|Sre)rHrIrJrr#r#r$rrvs   zClapProjectionLayer.forward) rLrMrNrrrrgrrrsr#r#rir$rFks rFcs2eZdZdZfddZd ddZdd ZZS) ClapTextEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. csttj|j|j|jd|_t|j|j|_ t|j |j|_ tj |j|j d|_ t|j|_t|dd|_|jdt|jddd|jd tj|jtjd dd|j|_tj|j|j|jd|_ dS) N)r;rposition_embedding_typeabsolute position_ids)rr)T) persistenttoken_type_idsr)rfrgr EmbeddingZ vocab_sizerEZ pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsZtype_vocab_sizetoken_type_embeddingsrrrrrr[rLrr8rCexpandrrNrr9r;r*rir#r$rgs. zClapTextEmbeddings.__init__Nrc Cs|dkr*|dk r t||j|}n ||}|dk r<|}n|dd}|d}|dkrt|dr|jddd|f}||d|} | }ntj|tj |j j d}|dkr| |}| |} || } |jdkr||} | | 7} || } || } | S)Nr)rrPrrlrM)r>r;&create_position_ids_from_inputs_embedsrhasattrrPrVr8rr9rNrBrRrUrLrTrr) r`r:rPrN inputs_embedsr< input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedrU embeddingsrTr#r#r$rrs0         zClapTextEmbeddings.forwardcCsN|dd}|d}tj|jd||jdtj|jd}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr)rrlr)rr8rCr;r9rBrrV)r`rYrZZsequence_lengthrNr#r#r$rWs  z9ClapTextEmbeddings.create_position_ids_from_inputs_embeds)NNNNr)rLrMrNrOrgrrrWrsr#r#rir$rK~s  (rKc seZdZd fdd ZejejdddZd ejeejeejeejeejee e ejee e ejdd d Z Z S) ClapTextSelfAttentionNcst|j|jdkr>t|ds>td|jd|jd|j|_t|j|j|_|j|j|_t |j|j|_ t |j|j|_ t |j|j|_ t |j|_|pt|dd|_|jdks|jd kr|j|_t d |jd |j|_|j|_dS) NrZembedding_sizerrrrLrM relative_keyrelative_key_queryr&r)rfrgrErrXrr3rrrrrrrrrrr[rLrSrQdistance_embedding is_decoderr`rvrLrir#r$rgs* zClapTextSelfAttention.__init__)rr@cCs6|dd|j|jf}||}|ddddSrrrr#r#r$rs z*ClapTextSelfAttention.transpose_for_scoresFrrrencoder_hidden_statesencoder_attention_maskpast_key_valuerr@cCs||}|dk } | r4|dk r4|d} |d} |}n| r^|||} |||} |}nv|dk r|||} |||} tj|d| gdd} tj|d| gdd} n |||} |||} ||} |dk } |jr| | f}t| | dd}|j dks |j dkr| j d| j d}}| r^tj |dtj |j d dd}ntj|tj |j d dd}tj|tj |j d dd}||}|||jd}|j| jd }|j dkrtd | |}||}n4|j dkrtd | |}td | |}|||}|t|j}|dk r:||}tjj|dd}||}|dk rf||}t|| }|dddd }|dd|jf}||}|r||fn|f}|jr||f}|S)Nrrr&r5r)rr`rarlrzbhld,lrd->bhlrzbhrd,lrd->bhlrr ) rrrrr8rrcrrrLrtensorr9rBr*rCrbrSrrmZeinsumrrrrrErrr+r,rr)r`rrrrfrgrhrrZis_cross_attentionrrr use_cacherZ query_lengthZ key_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyrrrrr#r#r$rrsp             zClapTextSelfAttention.forward)N)NNNNNF) rLrMrNrgr8rrrrPrrrrrsr#r#rir$r_s$r_cs4eZdZfddZejejejdddZZS)ClapTextSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNr) rfrgrrrErrrrrrr*rir#r$rg`s zClapTextSelfOutput.__init__rcCs&||}||}|||}|Srerrrrr#r#r$rrfs  zClapTextSelfOutput.forwardrr#r#rir$rk_s rkc sveZdZd fdd ZddZd ejeejeejeejeejee e ejee e ejddd Z Z S) ClapTextAttentionNcs.tt||d|_t||_t|_dS)NrL)rfrgr_r`rkrqrrrdrir#r$rgos  zClapTextAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dSrrrr#r#r$rus zClapTextAttention.prune_headsFrec Cs<||||||||}||d|} | f|dd} | Srr) r`rrrrfrgrhrrrrr#r#r$rrs  zClapTextAttention.forward)N)NNNNNF) rLrMrNrgrr8rrrPrrrrrsr#r#rir$rnns$rncs0eZdZfddZejejdddZZS)ClapTextIntermediatecsBtt|j|j|_t|jt r6t |j|_ n|j|_ dSre) rfrgrrrEintermediate_sizerrrrr rr*rir#r$rgs   zClapTextIntermediate.__init__rcCs||}||}|Srerrr#r#r$rrs  zClapTextIntermediate.forwardrr#r#rir$rps rpcs4eZdZfddZejejejdddZZS)ClapTextOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSrl) rfrgrrrqrErrrrrrr*rir#r$rgs zClapTextOutput.__init__rcCs&||}||}|||}|Srermrr#r#r$rrs  zClapTextOutput.forwardrr#r#rir$rrs rrc steZdZfddZd ejeejeejeejeejeeeejee eejdddZ dd Z Z S) ClapTextLayercsrt|j|_d|_t||_|j|_|j|_|jrZ|jsLt|dt|dd|_ t ||_ t ||_ dS)Nrz> should be used as a decoder model if cross attention is addedrMro)rfrgr seq_len_dimrnrrcadd_cross_attentionrcrossattentionrprrrrqr*rir#r$rgs   zClapTextLayer.__init__NFrec Cs|dk r|ddnd}|j|||||d} | d} |jrP| dd} | d} n | dd} d} |jr|dk rt|dstd|d|dk r|d dnd}|| ||||||}|d} | |dd} |d} | | } t|j|j|j| }|f| } |jr| | f} | S) Nr&rrhrrr)rvz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) rrcrXrrvrfeed_forward_chunkrrt)r`rrrrfrgrhrZself_attn_past_key_valueZself_attention_outputsrrZpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputsrr#r#r$rrsV         zClapTextLayer.forwardcCs||}|||}|Sre)rrq)r`rZintermediate_outputrr#r#r$rxs  z ClapTextLayer.feed_forward_chunk)NNNNNF) rLrMrNrgr8rrrPrrrrrxrsr#r#rir$rss$  ArscseZdZfddZd ejeejeejeejeejeeeejee ee ee ee e eeje fd ddZ Z S) ClapTextEncodercs:t|_tfddtjD|_d|_dS)Ncsg|] }tqSr#)rs)r]rrur#r$rsz,ClapTextEncoder.__init__..F) rfrgrvrrrnum_hidden_layerslayerr'r*rirur$rgs  zClapTextEncoder.__init__NFT) rrrrfrgpast_key_valuesrjrr2r4r@c  st| rdnd} rdnd} r(|jjr(dnd} |jrJ|jrJ|rJtdd}|rRdnd}t|jD]\}}| rv| |f} |dk r||nd}|dk r||nd|jr|jrևfdd}tj j |||||||}n||||||}|d}|r||df7}r`| |df} |jjr`| |d f} q`| r@| |f} | sbt d d ||| | | fDSt ||| | | d S) Nr#zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fcsfdd}|S)Ncs|fSrer#r5)r7rrhr#r$r8@szNClapTextEncoder.forward..create_custom_forward..custom_forwardr#r9rwr:r$r;?sz6ClapTextEncoder.forward..create_custom_forwardrr)rr&css|]}|dk r|VqdSrer#r<r#r#r$radsz*ClapTextEncoder.forward..)rJr|rrKcross_attentions) rvrur'rnloggerZ warning_oncerr{r8r@rArbr )r`rrrrfrgr|rjrr2r4rCrDZall_cross_attentionsZnext_decoder_cacher rrr;r r#rwr$rrsv          zClapTextEncoder.forward) NNNNNNFFT)rLrMrNrgr8rrrPrrrr rrrsr#r#rir$rys. rycs0eZdZfddZejejdddZZS)ClapTextPoolercs*tt|j|j|_t|_dSre)rfrgrrrErZTanhrIr*rir#r$rgzs zClapTextPooler.__init__rcCs(|dddf}||}||}|Sr)rrI)r`rZfirst_token_tensor pooled_outputr#r#r$rrs  zClapTextPooler.forwardrr#r#rir$rys rc@s.eZdZdZeZdZdZddZd ddZ dS) ClapPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. clapFcCs*|jj}t|trD|jjjjd|dd|jjjjd|ddnt|t r|t j j|j |ddt j j|j |ddnt|t jr|jjjd|ddnt|t jr|jj|jjdn^t|t jt jfr&|jjdd|jjd|}t j j|j|d|jdk r&|jjdS) zInitialize the weightsrkg{Gz?)Zmeanstd)rg?gr&N)rvZinitializer_factorrrKrTweightdataZnormal_rU ClapModelrinit logit_scale_a logit_scale_trQrrZzero_Zfill_rzrrErz)r`r7factorZ in_proj_stdr#r#r$ _init_weightss"       z!ClapPreTrainedModel._init_weightscCst|tr||_dSre)rryr')r`r7rr#r#r$_set_gradient_checkpointings z/ClapPreTrainedModel._set_gradient_checkpointingN)F) rLrMrNrOr config_classZbase_model_prefixZsupports_gradient_checkpointingrrr#r#r#r$rs rc seZdZeZdZedfdd ZejdddZ e e e e edd eejeejeeeeeeeee fd d d ZZS)ClapAudioModelrBrucs"t|t||_|dSre)rfrgr audio_encoder post_initr*rir#r$rgs  zClapAudioModel.__init__rZcCs |jjjSre)rr$rr_r#r#r$get_input_embeddingssz#ClapAudioModel.get_input_embeddings output_typerNrBr1rr2r4r@cCsP|dk r |n|jj}|dk r |n|jj}|dk r4|n|jj}|j|||||dS)ar Returns: Examples: ```python >>> from datasets import load_dataset >>> from transformers import AutoProcessor, ClapAudioModel >>> dataset = load_dataset("ashraq/esc50") >>> audio_sample = dataset["train"]["audio"][0]["array"] >>> model = ClapAudioModel.from_pretrained("laion/clap-htsat-fused") >>> processor = AutoProcessor.from_pretrained("laion/clap-htsat-fused") >>> inputs = processor(audios=audio_sample, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state ```NrBr1rr2r4)rvuse_return_dictrr2r)r`rBr1rr2r4r#r#r$rrszClapAudioModel.forward)NNNNN)rLrMrNrrmain_input_namergrrrrCLAP_AUDIO_INPUTS_DOCSTRINGrr rr8rP BoolTensorrrrrrrsr#r#rir$rs&  rcseZdZdZeZd fdd ZddZddZde e j e e j e e j e e j e e j e e j e e j e e j e e e j e ee ee ee eeee j efd d d ZZS) ClapTextModela* The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in *Attention is all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 TcsDt|||_t||_t||_|r2t|nd|_| dSre) rfrgrvrKr^ryencoderrpoolerr)r`rvZadd_pooling_layerrir#r$rgs    zClapTextModel.__init__cCs|jjSrer^rRr_r#r#r$rsz"ClapTextModel.get_input_embeddingscCs ||j_dSrerr`rr#r#r$set_input_embeddings sz"ClapTextModel.set_input_embeddingsN)r:rrPrNrrYrfrgr|rjrr2r4r@c Cs^| dk r | n|jj} | dk r | n|jj} | dk r4| n|jj} |jjrZ| dk rP| n|jj} nd} |dk rx|dk rxtdn@|dk r||||}n"|dk r|dd}ntd|\}}|dk r|j n|j }| dk r| ddj dnd}|dkrt j |||f|d}|dkrft |jd rT|jjddd|f}|||}|}nt j|t j|d }|||}|jjr|dk r|\}}}||f}|dkrt j ||d}||}nd}|||jj}|j|||||d }|j|||||| | | | | d }|d}|jdk r$||nd}| sB||f|d dSt|||j|j|j|jdS)a  encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). NFzDYou cannot specify both input_ids and inputs_embeds at the same timer)z5You have to specify either input_ids or inputs_embedsrr&rArPrl)r:rNrPrYr<) rrrfrgr|rjrr2r4r)rJr>r|rrKr})rvrr2rrcrjrZ%warn_if_padding_and_no_attention_maskrrBrr8ZonesrXr^rPrVrr9Zget_extended_attention_maskZinvert_attention_maskZ get_head_maskrzrrr r|rrKr})r`r:rrPrNrrYrfrgr|rjrr2r4rZr!r[rBr<r\r]Zextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthrZencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputZencoder_outputsZsequence_outputrr#r#r$rr s$          zClapTextModel.forward)T) NNNNNNNNNNNNN)rLrMrNrOrrrgrrrr8rrrPrrrr rrrsr#r#rir$rsD  rcseZdZeZedfdd Zeedee j ee j ee j ee ee ee e j dddZ eedee j ee j ee j ee ee ee e j dd d Zeeeeed dee jee j ee jee j ee jee ee ee ee eeefd d dZZS)rrucst|t|jts.tdt|jdt|jtsPtdt|jd|j}|j}t t t |j|_t t t |j|_|j|_t||_t||_t||_t||_|dS)NzKconfig.text_config is expected to be of type ClapTextConfig but is of type .zMconfig.audio_config is expected to be of type ClapAudioConfig but is of type )rfrgr text_configrrtype audio_configrrrr8rirlogZlogit_scale_init_valuerrrGr text_modelrFtext_projectionr audio_modelaudio_projectionr)r`rvrrrir#r$rgs&       zClapModel.__init__Nr:rrNrr2r4r@c Cs|dk r |n|jj}|dk r |n|jj}|dk r4|n|jj}|j||||||d}|dk rb|dn|j}||} tj| dd} | S)a Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`ClapTextModel`]. Examples: ```python >>> from transformers import AutoTokenizer, ClapModel >>> model = ClapModel.from_pretrained("laion/clap-htsat-unfused") >>> tokenizer = AutoTokenizer.from_pretrained("laion/clap-htsat-unfused") >>> inputs = tokenizer(["the sound of a cat", "the sound of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) ```Nr:rrNrr2r4rr)r5) rvrr2rrr>rF normalize) r`r:rrNrr2r4 text_outputsrZ text_featuresr#r#r$get_text_featuress   zClapModel.get_text_features)rBr1rrr2r4r@c Csz|dk r |n|jj}|dk r |n|jj}|dk r4|n|jj}|j|||d}|sX|dn|j}||} tj| dd} | S)a Returns: audio_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The audio embeddings obtained by applying the projection layer to the pooled output of [`ClapAudioModel`]. Examples: ```python >>> from transformers import AutoFeatureExtractor, ClapModel >>> import torch >>> model = ClapModel.from_pretrained("laion/clap-htsat-unfused") >>> feature_extractor = AutoFeatureExtractor.from_pretrained("laion/clap-htsat-unfused") >>> random_audio = torch.rand((16_000)) >>> inputs = feature_extractor(random_audio, return_tensors="pt") >>> audio_features = model.get_audio_features(**inputs) ```N)rBr1r4rr)r5) rvrr2rrr>rrr) r`rBr1rrr2r4 audio_outputsrZaudio_featuresr#r#r$get_audio_featuress zClapModel.get_audio_featuresr) r:rBr1rrN return_lossrr2r4r@c  Csp|dk r |n|jj}|dk r |n|jj}| dk r4| n|jj} |j||||| d} |j|||||| d} | sr| dn| j} || } | s| dn| j} || } | | j dddd} | | j dddd} |j }|j }t | | |}t | | |}d}|r,t|}t|}||d }| sZ||| | | | f}|dk rV|f|S|St|||| | | | d S) ak Returns: Examples: ```python >>> from datasets import load_dataset >>> from transformers import AutoProcessor, ClapModel >>> dataset = load_dataset("ashraq/esc50") >>> audio_sample = dataset["train"]["audio"][0]["array"] >>> model = ClapModel.from_pretrained("laion/clap-htsat-unfused") >>> processor = AutoProcessor.from_pretrained("laion/clap-htsat-unfused") >>> input_text = ["Sound of a dog", "Sound of vaccum cleaner"] >>> inputs = processor(text=input_text, audios=audio_sample, return_tensors="pt", padding=True) >>> outputs = model(**inputs) >>> logits_per_audio = outputs.logits_per_audio # this is the audio-text similarity score >>> probs = logits_per_audio.softmax(dim=-1) # we can take the softmax to get the label probabilities ```Nrrrr&r)T)pr6Zkeepdimg@)rUrVrWrIrSrXrY)rvrr2rrrr>rrrrexprr8rtrGrT)r`r:rBr1rrNrrr2r4rrrSrIZlogit_scale_textZlogit_scale_audiorWrVrUZ caption_lossZ audio_lossrqr#r#r$rrs\&       zClapModel.forward)NNNNNN)NNNNNN) NNNNNNNNN)rLrMrNrrrgrCLAP_TEXT_INPUTS_DOCSTRINGrr8rrrPrrrCLAP_INPUTS_DOCSTRINGrrTZ LongTensorrrrrrrsr#r#rir$rsr /-  rzf CLAP Text Model with a projection layer on top (a linear layer on top of the pooled output). c seZdZeZedfdd ZejdddZddZ e e e e ed deejeejeejeeeeeeeee fd d d ZZS)ClapTextModelWithProjectionrucs,t|t||_t||_|dSre)rfrgrrrFrrr*rir#r$rg}s   z$ClapTextModelWithProjection.__init__rZcCs |jjjSrerr^rRr_r#r#r$rsz0ClapTextModelWithProjection.get_input_embeddingscCs||jj_dSrerrr#r#r$rsz0ClapTextModelWithProjection.set_input_embeddingsrNrc Cs|dk r |n|jj}|j||||||d}|s6|dn|j}||} |st| |df|dd} tdd| DSt| |j|j|j dS) a Returns: Examples: ```python >>> from transformers import AutoTokenizer, ClapTextModelWithProjection >>> model = ClapTextModelWithProjection.from_pretrained("laion/clap-htsat-unfused") >>> tokenizer = AutoTokenizer.from_pretrained("laion/clap-htsat-unfused") >>> inputs = tokenizer(["a sound of a cat", "a sound of a dog"], padding=True, return_tensors="pt") >>> outputs = model(**inputs) >>> text_embeds = outputs.text_embeds ```Nrrrr&css|]}|dk r|VqdSrer#r]rqr#r#r$rasz6ClapTextModelWithProjection.forward..)rIrJrrK) rvrrr>rrbrHrJrrK) r`r:rrNrr2r4rrrIrr#r#r$rrs(  z#ClapTextModelWithProjection.forward)NNNNNN)rLrMrNrrrgrrrrrrrrHrr8rrrrrrrsr#r#rir$rts*  rzg CLAP Audio Model with a projection layer on top (a linear layer on top of the pooled output). c seZdZeZdZedfdd ZejdddZ e e e e edd eejeejeeeeeeeee fd d d ZZS)ClapAudioModelWithProjectionrBrucs,t|t||_t||_|dSre)rfrgrrrFrrr*rir#r$rgs   z%ClapAudioModelWithProjection.__init__rZcCs |jjjjSre)rrr$rr_r#r#r$rsz1ClapAudioModelWithProjection.get_input_embeddingsrNrc Cs|dk r |n|jj}|dk r |n|jj}|dk r4|n|jj}|j|||||d}|s\|dn|j}||}|s||df|dd} tdd| DSt||j |j |j dS) a Returns: Examples: ```python >>> from datasets import load_dataset >>> from transformers import ClapAudioModelWithProjection, ClapProcessor >>> model = ClapAudioModelWithProjection.from_pretrained("laion/clap-htsat-fused") >>> processor = ClapProcessor.from_pretrained("laion/clap-htsat-fused") >>> dataset = load_dataset("ashraq/esc50") >>> audio_sample = dataset["train"]["audio"][0]["array"] >>> inputs = processor(audios=audio_sample, return_tensors="pt") >>> outputs = model(**inputs) >>> audio_embeds = outputs.audio_embeds ```Nrrrr&css|]}|dk r|VqdSrer#rr#r#r$ra sz7ClapAudioModelWithProjection.forward..)rSrJrKr) rvrrr2rr>rrbrRrJrKr) r`rBr1rr2r4rrrSrr#r#r$rrs, z$ClapAudioModelWithProjection.forward)NNNNN)rLrMrNrrrrgrrrrrrrRrr8rPrrrrrrrsr#r#rir$rs&  r)r)VrOrr dataclassesrtypingrrrrrr8Ztorch.nn.functionalrrErZ activationsr Zmodeling_outputsr r r Zmodeling_utilsrZ pytorch_utilsrrrrr@rrrrrZconfiguration_claprrrZ get_loggerrLr~Z_CHECKPOINT_FOR_DOCZ"CLAP_PRETRAINED_MODEL_ARCHIVE_LISTr%r2r4r>rrGrHrRrTrrdrtrrrrrrrr rrZCLAP_START_DOCSTRINGrrrrFrKr_rkrnrprrrsryrrrrrrrr#r#r#r$s      %(ce' |<6R &Z2Wc%:*dF