U ,È-eÛã@sldZddlZddlmZmZmZmZddlZddlZddlm Z ddl m Z m Z m Z ddlmZmZmZddlmZmZmZmZmZdd lmZmZmZdd lmZd d lmZe  e!¡Z"d Z#dZ$d dgZ%dd„Z&dd„Z'dd„Z(dd„Z)dd„Z*Gdd„dej j+ƒZ,Gdd„de j+ƒZ-Gdd„de j+ƒZ.Gd d!„d!e j+ƒZ/Gd"d#„d#e j+ƒZ0Gd$d%„d%e j+ƒZ1Gd&d'„d'e j+ƒZ2Gd(d)„d)e j+ƒZ3Gd*d+„d+e j+ƒZ4Gd,d-„d-e j+ƒZ5Gd.d/„d/e j+ƒZ6Gd0d1„d1eƒZ7d2Z8d3Z9ed4e8ƒGd5d6„d6e7ƒƒZ:ed7e8ƒGd8d9„d9e7ƒƒZ;Gd:d;„d;e j+ƒZ„d>e7ƒƒZ=ed?e8ƒGd@dA„dAe7ƒƒZ>GdBdC„dCe j+ƒZ?dFdDdE„Z@dS)Gz PyTorch ESM model.éN)ÚListÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forward)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModelÚ find_pruneable_heads_and_indicesÚprune_linear_layer)Úloggingé)Ú EsmConfigzfacebook/esm2_t6_8M_UR50Drzfacebook/esm2_t12_35M_UR50DcCs&|jddd\}}tj| |fddS)Nééÿÿÿÿ©Údim)ÚchunkÚtorchÚcat)ÚxÚx1Zx2©r"úe/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/esm/modeling_esm.pyÚ rotate_half3sr$cCs`|dd…dd…d|jd…dd…f}|dd…dd…d|jd…dd…f}||t|ƒ|S)Néþÿÿÿ)Úshaper$)r ÚcosÚsinr"r"r#Úapply_rotary_pos_emb8s&&r)cCs |ddt |t d¡¡S)zo This is the gelu implementation from the original ESM repo. Using F.gelu yields subtly wrong results. gà?çð?g@)rÚerfÚmathÚsqrt©r r"r"r#Úgelu?sr/cCs|| dd¡S)zJMake layer symmetric in final two dimensions, used for contact prediction.rr%)Ú transposer.r"r"r#Ú symmetrizeFsr1cCsH|jddd}|jddd}|jddd}||}| |¡||}|S)z=Perform average product correct, used for contact prediction.rT)Zkeepdimsr%)rr%)ÚsumZdiv_)r Za1Za2Za12ÚavgÚ normalizedr"r"r#Úaverage_product_correctKs r5csReZdZdZedœ‡fdd„ Zd dd„Zejeje ejejfdœd d „Z ‡Z S) ÚRotaryEmbeddingzå Rotary position embeddings based on those in [RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer). Query and keys are transformed by rotation matrices which depend on their relative positions. rcsNtƒ ¡ddt d|d¡ ¡|}|}| d|¡d|_d|_d|_dS)Nr*i'rrÚinv_freq) ÚsuperÚ__init__rÚarangeÚfloatÚregister_bufferÚ_seq_len_cachedÚ _cos_cachedÚ _sin_cached)Úselfrr7©Ú __class__r"r#r9^s  zRotaryEmbedding.__init__rcCs¶|j|}||jks"|jj|jkrª||_tj|j||jd |j¡}t ||j¡}tj ||fdd  |j¡}|  ¡dddd…dd…f|_|  ¡dddd…dd…f|_ |j|j fS)N©Údevicerr)r&r=r>rDrr:Útype_asr7ÚouterrÚtor'r(r?)r@r Ú seq_dimensionZseq_lenÚtZfreqsZembr"r"r#Ú_update_cos_sin_tablesis z&RotaryEmbedding._update_cos_sin_tables)ÚqÚkÚreturncCs6|j|dd\|_|_t||j|jƒt||j|jƒfS)Nr%)rH)rJr>r?r))r@rKrLr"r"r#ÚforwardysþzRotaryEmbedding.forward)r) Ú__name__Ú __module__Ú __qualname__Ú__doc__Úintr9rJrÚTensorrrNÚ __classcell__r"r"rAr#r6Ws r6cs2eZdZdZd eedœ‡fdd„ Zdd„Z‡ZS) ÚEsmContactPredictionHeadzWPerforms symmetrization, apc, and computes a logistic regression on the output featuresTr)Ú in_featuresÚeos_idxcs4tƒ ¡||_||_t |d|¡|_t ¡|_dS)Nr) r8r9rWrXrÚLinearÚ regressionZSigmoidÚ activation)r@rWÚbiasrXrAr"r#r9…s  z!EsmContactPredictionHead.__init__c CsÜ| |j¡ |¡}| d¡| d¡}||dd…dddd…dd…f}|ddd…dd…f}|ddd…dd…f}| ¡\}}}}}| |||||¡}| |jjj¡}t t |ƒƒ}|  dddd¡}|  | |¡  d¡¡S)Nrr.rrr )ÚnerXrGÚ unsqueezeÚsizeÚviewrZÚweightrDr5r1Úpermuter[Úsqueeze) r@ÚtokensÚ attentionsZeos_maskÚ batch_sizeZlayersÚheadsZseqlenÚ_r"r"r#rN‘s"ÿ z EsmContactPredictionHead.forward)Tr)rOrPrQrRrSr9rNrUr"r"rAr#rV‚süü rVcs2eZdZdZ‡fdd„Zd dd„Zdd „Z‡ZS) Ú EsmEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. cs²tƒ ¡tj|j|j|jd|_|jr>tj |j|j d|_ nd|_ t  |j ¡|_t|ddƒ|_|jdt |j¡ d¡dd|j|_tj|j|j|jd|_|j|_|j|_dS) N)Ú padding_idx©ZepsÚposition_embedding_typeÚabsoluteÚ position_ids)rrF)Ú persistent)r8r9rÚ EmbeddingÚ vocab_sizeÚ hidden_sizeZ pad_token_idÚword_embeddingsZemb_layer_norm_beforeÚ LayerNormÚlayer_norm_epsÚ layer_normÚDropoutÚhidden_dropout_probÚdropoutÚgetattrrlr<rr:Úmax_position_embeddingsÚexpandrjÚposition_embeddingsÚ token_dropoutÚ mask_token_id©r@ÚconfigrAr"r#r9ªs( ÿÿzEsmEmbeddings.__init__Nrc Cs|dkr*|dk r t||j|ƒ}n | |¡}|dkr<| |¡}|}|jr®| ||jk d¡d¡}d}| d¡}||jk d¡  ¡|} |d|d| dd…ddf  |j ¡}|j dkrÊ|  |¡} || }|jdk rÞ| |¡}|dk rü|| d¡  |j ¡}|S)Nrçg¸…ëQ¸¾?rrm)Ú"create_position_ids_from_input_idsrjÚ&create_position_ids_from_inputs_embedsrsr~Z masked_fillrr^r2r;rGÚdtyperlr}rv) r@Ú input_idsÚattention_maskrnÚ inputs_embedsÚpast_key_values_lengthÚ embeddingsZmask_ratio_trainZ src_lengthsZmask_ratio_observedr}r"r"r#rNÀs.    "ÿ    zEsmEmbeddings.forwardcCsN| ¡dd…}|d}tj|jd||jdtj|jd}| d¡ |¡S)z× We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nrr©r…rDr)r_rr:rjÚlongrDr^r|)r@rˆÚ input_shapeZsequence_lengthrnr"r"r#r„ís  ÿz4EsmEmbeddings.create_position_ids_from_inputs_embeds)NNNNr)rOrPrQrRr9rNr„rUr"r"rAr#ri¥s ÿ -ric s‚eZdZd ‡fdd„ Zejejdœdd„Zd ejeejeejeejeejee e ejee e ejdœd d „Z ‡Z S) ÚEsmSelfAttentionNcs tƒ ¡|j|jdkr>t|dƒs>td|j›d|j›dƒ‚|j|_t|j|jƒ|_|j|j|_t   |j|j¡|_ t   |j|j¡|_ t   |j|j¡|_ t  |j¡|_|p¸t|ddƒ|_d|_|jdksÖ|jd krú|j|_t  d |jd |j¡|_n|jd krt|jd |_|j|_dS)NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)rlrmÚ relative_keyÚrelative_key_queryrrÚrotaryr)r8r9rrÚnum_attention_headsÚhasattrÚ ValueErrorrSÚattention_head_sizeÚ all_head_sizerrYÚqueryÚkeyÚvaluerwZattention_probs_dropout_probryrzrlÚrotary_embeddingsr{rpÚdistance_embeddingr6Ú is_decoder)r@rrlrAr"r#r9s0 ÿÿ zEsmSelfAttention.__init__)r rMcCs6| ¡dd…|j|jf}| |¡}| dddd¡S)Nrrrrr )r_r“r–r`rb)r@r Z new_x_shaper"r"r#Útranspose_for_scoress z%EsmSelfAttention.transpose_for_scoresF)Ú hidden_statesr‡Ú head_maskÚencoder_hidden_statesÚencoder_attention_maskÚpast_key_valueÚoutput_attentionsrMcCs²| |¡}|dk } | r4|dk r4|d} |d} |}n | r^| | |¡¡} | | |¡¡} |}nv|dk r´| | |¡¡} | | |¡¡} tj|d| gdd} tj|d| gdd} n | | |¡¡} | | |¡¡} | |¡} | |jd} |jrú| | f}|jdkr|  | | ¡\} } t  | |   dd¡¡} |jd ksB|jd kr|  ¡d}tj |tj|jd  dd¡}tj |tj|jd  dd¡}||}| ||jd¡}|j| jd }|jd krÔt d | |¡}| |} n4|jd krt d | |¡}t d| |¡}| ||} |dk r| |} tjj| dd}| |¡}|dk rF||}t  || ¡}| dddd¡ ¡}|  ¡dd…|jf}| |¡}|r–||fn|f}|jr®||f}|S)Nrrrrgà¿r’rr%rr‘r‹)r…zbhld,lrd->bhlrzbhrd,lrd->bhlrr )r˜ržr™ršrrr–rrlr›Úmatmulr0r_r:rŒrDr`rœr{rGr…ZeinsumrZ functionalZsoftmaxryrbÚ contiguousr—)r@rŸr‡r r¡r¢r£r¤Zmixed_query_layerZis_cross_attentionZ key_layerZ value_layerZ query_layerZattention_scoresÚ seq_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZ context_layerZnew_context_layer_shapeÚoutputsr"r"r#rN"sh               zEsmSelfAttention.forward)N)NNNNNF) rOrPrQr9rrTržrÚ FloatTensorrÚboolrNrUr"r"rAr#rŽÿs$ø÷rŽcs$eZdZ‡fdd„Zdd„Z‡ZS)Ú EsmSelfOutputcs.tƒ ¡t |j|j¡|_t |j¡|_dS©N) r8r9rrYrrÚdenserwrxryr€rAr"r#r9Šs zEsmSelfOutput.__init__cCs | |¡}| |¡}||}|Sr¬©r­ry©r@rŸZ input_tensorr"r"r#rNs  zEsmSelfOutput.forward©rOrPrQr9rNrUr"r"rAr#r«‰s r«cs.eZdZ‡fdd„Zdd„Zd dd„Z‡ZS) Ú EsmAttentioncs>tƒ ¡t|ƒ|_t|ƒ|_tƒ|_tj |j |j d|_ dS)Nrk) r8r9rŽr@r«ÚoutputÚsetÚ pruned_headsrrtrrrur€rAr"r#r9—s    zEsmAttention.__init__cCs²t|ƒdkrdSt||jj|jj|jƒ\}}t|jj|ƒ|j_t|jj|ƒ|j_t|jj |ƒ|j_ t|j j |dd|j _ |jjt|ƒ|j_|jj|jj|j_ |j  |¡|_dS)Nrrr)Úlenrr@r“r–r´rr˜r™ršr²r­r—Úunion)r@rgÚindexr"r"r#Ú prune_headsžs ÿzEsmAttention.prune_headsNFc CsF| |¡}| |||||||¡} | | d|¡} | f| dd…} | S)Nrr)rtr@r²) r@rŸr‡r r¡r¢r£r¤Zhidden_states_lnZ self_outputsÚattention_outputr¨r"r"r#rN°s ù zEsmAttention.forward)NNNNNF)rOrPrQr9r¸rNrUr"r"rAr#r±–s ør±cs0eZdZ‡fdd„Zejejdœdd„Z‡ZS)ÚEsmIntermediatecs tƒ ¡t |j|j¡|_dSr¬)r8r9rrYrrÚintermediate_sizer­r€rAr"r#r9Ês zEsmIntermediate.__init__©rŸrMcCs| |¡}t|ƒ}|Sr¬)r­r/)r@rŸr"r"r#rNÎs zEsmIntermediate.forward©rOrPrQr9rrTrNrUr"r"rAr#rºÉs rºcs$eZdZ‡fdd„Zdd„Z‡ZS)Ú EsmOutputcs.tƒ ¡t |j|j¡|_t |j¡|_ dSr¬) r8r9rrYr»rrr­rwrxryr€rAr"r#r9Õs zEsmOutput.__init__cCs | |¡}| |¡}||}|Sr¬r®r¯r"r"r#rNÚs  zEsmOutput.forwardr°r"r"rAr#r¾Ôs r¾cs.eZdZ‡fdd„Zd dd„Zdd„Z‡ZS) ÚEsmLayercs‚tƒ ¡|j|_d|_t|ƒ|_|j|_|j|_|jrV|jsLt|›dƒ‚t|ƒ|_ t |ƒ|_ t |ƒ|_ tj|j|jd|_dS)Nrz> should be used as a decoder model if cross attention is addedrk)r8r9Zchunk_size_feed_forwardZ seq_len_dimr±Ú attentionrÚadd_cross_attentionÚ RuntimeErrorÚcrossattentionrºÚ intermediater¾r²rrtrrrur€rAr"r#r9âs     zEsmLayer.__init__NFc Cs|dk r|dd…nd}|j|||||d} | d} |jrP| dd…} | d} n | dd…} d} |jrÞ|dk rÞt|dƒsˆtd|›dƒ‚|dk rœ|d d…nd}| | ||||||¡}|d} | |dd…} |d} | | } | | ¡}|f| } |jr| | f} | S) Nr©r¤r£rrrrÃz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r%)rÀrr”ÚAttributeErrorrÃÚfeed_forward_chunk)r@rŸr‡r r¡r¢r£r¤Zself_attn_past_key_valueZself_attention_outputsr¹r¨Zpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputsÚ layer_outputr"r"r#rNñsL û     ÿù    zEsmLayer.forwardcCs$| |¡}| |¡}| ||¡}|Sr¬)rtrÄr²)r@r¹Zattention_output_lnZintermediate_outputrÈr"r"r#rÇ0s   zEsmLayer.feed_forward_chunk)NNNNNF)rOrPrQr9rNrÇrUr"r"rAr#r¿ás ø ?r¿c s&eZdZ‡fdd„Zddd„Z‡ZS) Ú EsmEncodercsNtƒ ¡ˆ|_t ‡fdd„tˆjƒDƒ¡|_tjˆj ˆj d|_ d|_ dS)Ncsg|] }tˆƒ‘qSr")r¿)Ú.0rh©rr"r#Ú ;sz'EsmEncoder.__init__..rkF) r8r9rrZ ModuleListÚrangeÚnum_hidden_layersÚlayerrtrrruÚemb_layer_norm_afterÚgradient_checkpointingr€rArËr#r98s   zEsmEncoder.__init__NFTc  s†|jr|jr|rt d¡d}| r&dnd} ˆr2dnd} ˆrF|jjrFdnd} |rRdnd}t|jƒD]Î\}}| rv| |f} |dk r†||nd}|dk rš||nd‰|jrÖ|jrÖ‡‡fdd„}tj j   ||ƒ|||||¡}n||||||ˆˆƒ}|d}|r||df}ˆr`| |df} |jjr`| |d f} q`|j rB|  |¡}| rR| |f} | stt d d „||| | | fDƒƒSt ||| | | d S) Nzh`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting `use_cache=False`...Fr"cs‡‡‡fdd„}|S)Ncsˆ|ˆˆfžŽSr¬r")Úinputs)Úmoduler¤r£r"r#Úcustom_forwardbszIEsmEncoder.forward..create_custom_forward..custom_forwardr")rÓrÔrÅ)rÓr#Úcreate_custom_forwardasz1EsmEncoder.forward..create_custom_forwardrrrrcss|]}|dk r|VqdSr¬r")rÊÚvr"r"r#Ú ‰søz%EsmEncoder.forward..)Úlast_hidden_stateÚpast_key_valuesrŸreÚcross_attentions)rÑZtrainingÚloggerZ warning_oncerrÁÚ enumeraterÏrÚutilsÚ checkpointrÐÚtupler)r@rŸr‡r r¡r¢rÙÚ use_cacher¤Úoutput_hidden_statesÚ return_dictZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheÚiZ layer_moduleZlayer_head_maskrÕZ layer_outputsr"rÅr#rN?sz ÿ     ú ù   ûþ ûzEsmEncoder.forward) NNNNNNFFTr°r"r"rAr#rÉ7s õrÉcs0eZdZ‡fdd„Zejejdœdd„Z‡ZS)Ú EsmPoolercs*tƒ ¡t |j|j¡|_t ¡|_dSr¬)r8r9rrYrrr­ZTanhr[r€rAr"r#r9Ÿs zEsmPooler.__init__r¼cCs(|dd…df}| |¡}| |¡}|S©Nr)r­r[)r@rŸZfirst_token_tensorÚ pooled_outputr"r"r#rN¤s  zEsmPooler.forwardr½r"r"rAr#räžs räc@s8eZdZdZeZdZdZdddgZdd„Z d d d „Z d S)ÚEsmPreTrainedModelz† An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. ÚesmTr¿Z#EsmFoldTriangularSelfAttentionBlockricCs¤t|tjƒr:|jjjd|jjd|jdk r |jj  ¡nft|tj ƒrz|jjjd|jjd|j dk r |jj|j   ¡n&t|tj ƒr |jj  ¡|jj  d¡dS)zInitialize the weightsr‚)ZmeanZstdNr*)Ú isinstancerrYraÚdataZnormal_rZinitializer_ranger\Zzero_rprjrtZfill_)r@rÓr"r"r#Ú _init_weights¹s      z EsmPreTrainedModel._init_weightsFcCst|tƒr||_dSr¬)rérÉrÑ)r@rÓršr"r"r#Ú_set_gradient_checkpointingÉs z.EsmPreTrainedModel._set_gradient_checkpointingN)F) rOrPrQrRrÚ config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesrërìr"r"r"r#rç­s rça= This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`EsmConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a¼ Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. z]The bare ESM Model transformer outputting raw hidden-states without any specific head on top.csØeZdZdZd‡fdd„ Zdd„Zdd„Zd d „Zee   d ¡ƒe e e ed deejeejeejeejeejeejeejeeejeeeeeeeeeeeje fdœ dd„ƒƒZdd„Z‡ZS)ÚEsmModela The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. TcsZtƒ |¡||_t|ƒ|_t|ƒ|_|r2t|ƒnd|_t |j |j dd|_ |  ¡dS)NT)rWr\)r8r9rrirŠrÉÚencoderräÚpoolerrVrÎr“Ú contact_headÚ post_init)r@rÚadd_pooling_layerrAr"r#r9s    ÿzEsmModel.__init__cCs|jjSr¬©rŠrs©r@r"r"r#Úget_input_embeddings)szEsmModel.get_input_embeddingscCs ||j_dSr¬rô)r@ršr"r"r#Úset_input_embeddings,szEsmModel.set_input_embeddingscCs*| ¡D]\}}|jj|j |¡qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)ÚitemsrïrÏrÀr¸)r@Zheads_to_prunerÏrgr"r"r#Ú _prune_heads/szEsmModel._prune_headsz(batch_size, sequence_length)©rÞÚ output_typeríN) r†r‡rnr rˆr¡r¢rÙràr¤rárârMc  Cs | dk r | n|jj} | dk r | n|jj} | dk r4| n|jj} |jjrZ| dk rP| n|jj} nd} |dk rx|dk rxtdƒ‚n@|dk r–| ||¡| ¡} n"|dk r°| ¡dd…} ntdƒ‚| \}}|dk rÎ|j n|j }|dk rî|ddj dnd}|dkrt j |||f|d}|  || ¡}|jjrl|dk rl| ¡\}}}||f}|dkr`t j ||d}| |¡}nd}| ||jj¡}|j|||||d }|j||||||| | | | d }|d}|jdk rÐ| |¡nd}| sî||f|d d…St|||j|j|j|jd S) a  encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). NFzDYou cannot specify both input_ids and inputs_embeds at the same timerz5You have to specify either input_ids or inputs_embedsrrrC)r†rnr‡rˆr‰) r‡r r¡r¢rÙràr¤rárâr)rØZ pooler_outputrÙrŸrerÚ)rr¤ráÚuse_return_dictrràr•Z%warn_if_padding_and_no_attention_maskr_rDr&rZonesZget_extended_attention_maskZinvert_attention_maskZ get_head_maskrÎrŠrïrðrrÙrŸrerÚ)r@r†r‡rnr rˆr¡r¢rÙràr¤rárârrfr§rDr‰Zextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthrhZencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputZencoder_outputsÚsequence_outputrær"r"r#rN7sx)ÿ       ûö úzEsmModel.forwardcCs`|||dddj}tj|dd}|| d¡ d¡ d¡9}|| d¡ d¡ d¡9}| ||¡S)NT)r‡râr¤rrrr é)rerÚstackr^rñ)r@rdr‡Zattnsr"r"r#Úpredict_contacts¶s zEsmModel.predict_contacts)T) NNNNNNNNNNNN)rOrPrQrRr9rör÷rùr ÚESM_INPUTS_DOCSTRINGÚformatr Ú_CHECKPOINT_FOR_DOCrÚ_CONFIG_FOR_DOCrrrTrr©rªrrrNrrUr"r"rAr#rîsN  ýó òyrîz1ESM Model with a `language modeling` head on top.csÂeZdZdgZ‡fdd„Zdd„Zdd„Zee  d¡ƒe e e e d d deejeejeejeejeejeejeejeejeeeeeeeee fd œ d d„ƒƒZdd„Z‡ZS)ÚEsmForMaskedLMzlm_head.decoder.weightcs@tƒ |¡|jrt d¡t|dd|_t|ƒ|_|  ¡dS)NzjIf you want to use `EsmForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.F©ró) r8r9rrÛÚwarningrîrèÚ EsmLMHeadÚlm_headÚ init_weightsr€rAr"r#r9Æs ÿ zEsmForMaskedLM.__init__cCs|jjSr¬©r Údecoderrõr"r"r#Úget_output_embeddingsÔsz$EsmForMaskedLM.get_output_embeddingscCs ||j_dSr¬r )r@Znew_embeddingsr"r"r#Úset_output_embeddings×sz$EsmForMaskedLM.set_output_embeddingsúbatch_size, sequence_lengthz)rÞrûríÚmaskN) r†r‡rnr rˆr¡r¢Úlabelsr¤rárârMc  Cs¾| dk r | n|jj} |j|||||||| | | d } | d} | | ¡}d}|dk r~tƒ}| |j¡}|| d|jj¡| d¡ƒ}| sª|f| dd…}|dk r¦|f|S|St ||| j | j dS)a& labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` kwargs (`Dict[str, any]`, optional, defaults to *{}*): Used to hide legacy arguments that have been deprecated. N) r‡rnr rˆr¡r¢r¤rárârrr©ÚlossÚlogitsrŸre) rrürèr rrGrDr`rqrrŸre)r@r†r‡rnr rˆr¡r¢rr¤rárâr¨rýZprediction_scoresZmasked_lm_lossÚloss_fctr²r"r"r#rNÚs:ö   üzEsmForMaskedLM.forwardcCs|jj||dS)N)r‡)rèr)r@rdr‡r"r"r#rszEsmForMaskedLM.predict_contacts) NNNNNNNNNNN)rOrPrQZ_tied_weights_keysr9r rr rrr rrrrrÚ LongTensorrTr©rªrrrNrrUr"r"rAr#rÂsJ  üô ó9rcs(eZdZdZ‡fdd„Zdd„Z‡ZS)rz&ESM Head for masked language modeling.cs^tƒ ¡t |j|j¡|_tj|j|jd|_tj|j|j dd|_ t  t   |j ¡¡|_dS)NrkF)r\)r8r9rrYrrr­rtrurvrqr Ú ParameterrZzerosr\r€rAr"r#r9!s  zEsmLMHead.__init__cKs0| |¡}t|ƒ}| |¡}| |¡|j}|Sr¬)r­r/rvr r\©r@ÚfeaturesÚkwargsr r"r"r#rN)s   zEsmLMHead.forward©rOrPrQrRr9rNrUr"r"rAr#rs rz› ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cs’eZdZ‡fdd„Zee d¡ƒeee e dd e e j e e je e j e e je e je e j e ee ee eeee fdœ dd„ƒƒZ‡ZS) ÚEsmForSequenceClassificationcs>tƒ |¡|j|_||_t|dd|_t|ƒ|_| ¡dS©NFr) r8r9Ú num_labelsrrîrèÚEsmClassificationHeadÚ classifierr r€rAr"r#r9;s   z%EsmForSequenceClassification.__init__rrúN© r†r‡rnr rˆrr¤rárârMc  Cs|| dk r | n|jj} |j|||||||| d} | d} | | ¡} d} |dk r8| | j¡}|jjdkr®|jdkrzd|j_n4|jdkr¦|jt j ksœ|jt j kr¦d|j_nd|j_|jjdkrêt ƒ}|jdkrÞ||   ¡|  ¡ƒ} n || |ƒ} nN|jjdkrtƒ}||  d|j¡| d¡ƒ} n|jjdkr8tƒ}|| |ƒ} | sh| f| d d…}| dk rd| f|S|St| | | j| jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). N©r‡rnr rˆr¤rárârrrZZsingle_label_classificationZmulti_label_classificationrrr)rrürèr rGrDZ problem_typerr…rrŒrSr rcrr`rrrŸre©r@r†r‡rnr rˆrr¤rárâr¨rýrrrr²r"r"r#rNEsTø       "     üz$EsmForSequenceClassification.forward) NNNNNNNNN)rOrPrQr9r rrr rrrrrrrTr©rªrrrNrUr"r"rAr#r3s8 ýö õrz¢ ESM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. cs’eZdZ‡fdd„Zee d¡ƒeee e dd e e j e e je e j e e je e je e j e ee ee eeee fdœ dd„ƒƒZ‡ZS) ÚEsmForTokenClassificationcsNtƒ |¡|j|_t|dd|_t |j¡|_t  |j |j¡|_ |  ¡dSr) r8r9rrîrèrrwrxryrYrrr r r€rAr"r#r9™s  z"EsmForTokenClassification.__init__rrúNr!c  CsÂ| dk r | n|jj} |j|||||||| d} | d} | | ¡} | | ¡} d} |dk r‚tƒ}| | j¡}||  d|j ¡| d¡ƒ} | s®| f| dd…}| dk rª| f|S|St | | | j | j dS)zÛ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Nr"rrrr) rrürèryr rrGrDr`rrrŸrer#r"r"r#rN£s8ø    üz!EsmForTokenClassification.forward) NNNNNNNNN)rOrPrQr9r rrr rrrrrrrTr©rªrrrNrUr"r"rAr#r$‘s8 ýö õr$cs(eZdZdZ‡fdd„Zdd„Z‡ZS)rz-Head for sentence-level classification tasks.cs@tƒ ¡t |j|j¡|_t |j¡|_t |j|j ¡|_ dSr¬) r8r9rrYrrr­rwrxryrÚout_projr€rAr"r#r9ás zEsmClassificationHead.__init__cKsL|dd…ddd…f}| |¡}| |¡}t |¡}| |¡}| |¡}|Srå)ryr­rÚtanhr%rr"r"r#rNçs     zEsmClassificationHead.forwardrr"r"rAr#rÞs rcCs6| |¡ ¡}tj|dd |¡||}| ¡|S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rr)r]rSrZcumsumrErŒ)r†rjr‰rZincremental_indicesr"r"r#rƒñs rƒ)r)ArRr,ÚtypingrrrrrZtorch.utils.checkpointrZtorch.nnrrr Z file_utilsr r r Zmodeling_outputsrrrrrZmodeling_utilsrrrrÝrZconfiguration_esmrZ get_loggerrOrÛrrZ!ESM_PRETRAINED_MODEL_ARCHIVE_LISTr$r)r/r1r5ÚModuler6rVrirŽr«r±rºr¾r¿rÉrärçZESM_START_DOCSTRINGrrîrrrr$rrƒr"r"r"r#Úst    þ +#Z  3  Vg!*þ7[ûWûF