U
    ,-e                     @   s>  d Z ddlZddlmZmZmZ ddlZddlmZ ddlm	Z	m
Z
mZ ddlmZmZ ddlmZmZmZmZmZmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZm Z  ddl!m"Z" e #e$Z%dZ&dZ'dgZ(G dd deZ)G dd dej*Z+G dd dej*Z,G dd dej*Z-G dd dej*Z.G dd dej*Z/G dd dej*Z0G dd dej*Z1G d d! d!ej*Z2d"Z3d#Z4ed$e3G d%d& d&e)Z5G d'd( d(e)Z6G d)d* d*ej*Z7ed+e3G d,d- d-e)Z8ed.e3G d/d0 d0e)Z9ed1e3G d2d3 d3e)Z:G d4d5 d5ej*Z;ed6e3G d7d8 d8e)Z<d9d: Z=dS );zPyTorch MPNet model.    N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FNgelu)BaseModelOutputBaseModelOutputWithPoolingMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging   )MPNetConfigzmicrosoft/mpnet-baser   c                   @   s    e Zd ZeZeZdZdd ZdS )MPNetPreTrainedModelmpnetc                 C   s   t |tjr:|jjjd| jjd |jdk	r|jj	  nft |tj
rz|jjjd| jjd |jdk	r|jj|j 	  n&t |tjr|jj	  |jjd dS )zInitialize the weightsg        )ZmeanZstdNg      ?)
isinstancer   LinearweightdataZnormal_configZinitializer_rangebiasZzero_	Embeddingpadding_idx	LayerNormZfill_)selfmodule r)   i/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/mpnet/modeling_mpnet.py_init_weights:   s    

z"MPNetPreTrainedModel._init_weightsN)	__name__
__module____qualname__r   config_class#MPNET_PRETRAINED_MODEL_ARCHIVE_LISTZpretrained_model_archive_mapZbase_model_prefixr+   r)   r)   r)   r*   r   5   s   r   c                       s.   e Zd Z fddZdddZdd Z  ZS )	MPNetEmbeddingsc                    s   t    d| _tj|j|j| jd| _tj|j|j| jd| _	tj
|j|jd| _
t|j| _| jdt|jddd d S )Nr   )r%   Zepsposition_ids)r   F)
persistent)super__init__r%   r   r$   
vocab_sizehidden_sizeword_embeddingsZmax_position_embeddingsposition_embeddingsr&   layer_norm_epsDropouthidden_dropout_probdropoutZregister_buffertorcharangeexpandr'   r"   	__class__r)   r*   r7   L   s    
    zMPNetEmbeddings.__init__Nc           	      K   s   |d kr(|d k	rt || j}n
| |}|d k	r:| }n| d d }|d }|d krp| jd d d |f }|d kr| |}| |}|| }| |}| |}|S )Nr4   r   )	"create_position_ids_from_input_idsr%   &create_position_ids_from_inputs_embedssizer3   r:   r;   r&   r?   )	r'   	input_idsr3   inputs_embedskwargsinput_shapeZ
seq_lengthr;   
embeddingsr)   r)   r*   forwardZ   s"    





zMPNetEmbeddings.forwardc                 C   sN   |  dd }|d }tj| jd || j d tj|jd}|d|S )z
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr4   r   )dtypedevicer   )rH   r@   rA   r%   longrP   	unsqueezerB   )r'   rJ   rL   Zsequence_lengthr3   r)   r)   r*   rG   t   s    	   z6MPNetEmbeddings.create_position_ids_from_inputs_embeds)NNN)r,   r-   r.   r7   rN   rG   __classcell__r)   r)   rD   r*   r1   K   s   
r1   c                       s.   e Zd Z fddZdd Zd	ddZ  ZS )
MPNetSelfAttentionc                    s   t    |j|j dkr>t|ds>td|j d|j d|j| _t|j|j | _| j| j | _t	
|j| j| _t	
|j| j| _t	
|j| j| _t	
|j|j| _t	|j| _d S )Nr   Zembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ())r6   r7   r9   num_attention_headshasattr
ValueErrorintattention_head_sizeall_head_sizer   r   qkvor=   Zattention_probs_dropout_probr?   rC   rD   r)   r*   r7      s    
zMPNetSelfAttention.__init__c                 C   s6   |  d d | j| jf }|j| }|ddddS )Nr4   r      r   r	   )rH   rV   rZ   viewpermute)r'   xZnew_x_shaper)   r)   r*   transpose_for_scores   s    
z'MPNetSelfAttention.transpose_for_scoresNFc                 K   s  |  |}| |}| |}	| |}| |}| |	}	t||dd}
|
t| j	 }
|d k	rp|
|7 }
|d k	r|
| }
t
jj|
dd}| |}|d k	r|| }t||	}|dddd }| d d | jf }|j| }| |}|r||fn|f}|S )Nr4   dimr   r`   r   r	   )r\   r]   r^   rd   r@   matmulZ	transposemathsqrtrZ   r   Z
functionalZsoftmaxr?   rb   
contiguousrH   r[   ra   r_   )r'   hidden_statesattention_mask	head_maskposition_biasoutput_attentionsrK   r\   r]   r^   Zattention_scoresZattention_probscZnew_c_shaper_   outputsr)   r)   r*   rN      s.    	








zMPNetSelfAttention.forward)NNNF)r,   r-   r.   r7   rd   rN   rS   r)   r)   rD   r*   rT      s       rT   c                       s.   e Zd Z fddZdd Zd	ddZ  ZS )
MPNetAttentionc                    sB   t    t|| _tj|j|jd| _t|j	| _
t | _d S Nr2   )r6   r7   rT   attnr   r&   r9   r<   r=   r>   r?   setpruned_headsrC   rD   r)   r*   r7      s
    

zMPNetAttention.__init__c                 C   s   t |dkrd S t|| jj| jj| j\}}t| jj|| j_t| jj|| j_t| jj	|| j_	t| jj
|dd| j_
| jjt | | j_| jj| jj | j_| j|| _d S )Nr   r   rf   )lenr   ru   rV   rZ   rw   r   r\   r]   r^   r_   r[   union)r'   headsindexr)   r)   r*   prune_heads   s       zMPNetAttention.prune_headsNFc           
      K   sB   | j |||||d}| | |d | }|f|dd   }	|	S )N)rp   r   r   )ru   r&   r?   )
r'   rl   rm   rn   ro   rp   rK   Zself_outputsattention_outputrr   r)   r)   r*   rN      s    	zMPNetAttention.forward)NNNF)r,   r-   r.   r7   r|   rN   rS   r)   r)   rD   r*   rs      s       rs   c                       s0   e Zd Z fddZejejdddZ  ZS )MPNetIntermediatec                    sB   t    t|j|j| _t|jt	r6t
|j | _n|j| _d S N)r6   r7   r   r   r9   intermediate_sizedenser   Z
hidden_actstrr
   intermediate_act_fnrC   rD   r)   r*   r7      s
    
zMPNetIntermediate.__init__rl   returnc                 C   s   |  |}| |}|S r   )r   r   )r'   rl   r)   r)   r*   rN     s    

zMPNetIntermediate.forwardr,   r-   r.   r7   r@   TensorrN   rS   r)   r)   rD   r*   r~      s   r~   c                       s4   e Zd Z fddZejejejdddZ  ZS )MPNetOutputc                    sB   t    t|j|j| _tj|j|jd| _t	|j
| _d S rt   )r6   r7   r   r   r   r9   r   r&   r<   r=   r>   r?   rC   rD   r)   r*   r7     s    
zMPNetOutput.__init__)rl   input_tensorr   c                 C   s&   |  |}| |}| || }|S r   )r   r?   r&   )r'   rl   r   r)   r)   r*   rN     s    

zMPNetOutput.forwardr   r)   r)   rD   r*   r     s   r   c                       s&   e Zd Z fddZdddZ  ZS )
MPNetLayerc                    s,   t    t|| _t|| _t|| _d S r   )r6   r7   rs   	attentionr~   intermediater   outputrC   rD   r)   r*   r7     s    


zMPNetLayer.__init__NFc                 K   sL   | j |||||d}|d }|dd  }	| |}
| |
|}|f|	 }	|	S )N)ro   rp   r   r   )r   r   r   )r'   rl   rm   rn   ro   rp   rK   Zself_attention_outputsr}   rr   Zintermediate_outputZlayer_outputr)   r)   r*   rN   #  s    	

zMPNetLayer.forward)NNNF)r,   r-   r.   r7   rN   rS   r)   r)   rD   r*   r     s   	    r   c                       s\   e Zd Z fddZdejeej eej eeedddZdd	d
Z	e
dddZ  ZS )MPNetEncoderc                    sN   t     | _ j| _t fddt jD | _	t
 j| j| _d S )Nc                    s   g | ]}t  qS r)   )r   ).0_r"   r)   r*   
<listcomp>A  s     z)MPNetEncoder.__init__.<locals>.<listcomp>)r6   r7   r"   rV   Zn_headsr   Z
ModuleListrangenum_hidden_layerslayerr$   Zrelative_attention_num_bucketsrelative_attention_biasrC   rD   r   r*   r7   =  s
    
 zMPNetEncoder.__init__NF)rl   rm   rn   rp   output_hidden_statesreturn_dictc                 K   s   |  |}|rdnd }	|rdnd }
t| jD ]N\}}|rB|	|f }	||||| |fd|i|}|d }|r,|
|d f }
q,|r|	|f }	|stdd ||	|
fD S t||	|
dS )Nr)   rp   r   r   c                 s   s   | ]}|d k	r|V  qd S r   r)   )r   r^   r)   r)   r*   	<genexpr>g  s      z'MPNetEncoder.forward.<locals>.<genexpr>)last_hidden_staterl   
attentions)compute_position_bias	enumerater   tupler   )r'   rl   rm   rn   rp   r   r   rK   ro   Zall_hidden_statesZall_attentionsiZlayer_moduleZlayer_outputsr)   r)   r*   rN   D  s8    



zMPNetEncoder.forward    c                 C   s   | d| d| d  }}}|d k	rX|d d d d d f }|d d d d d f }n8tj|tjdd d d f }tj|tjdd d d f }|| }	| j|	|d}
|
|j}
| |
}|dddg	d}|
|d||f }|S )Nr   r   )rO   )num_bucketsr`   r4   )rH   r@   rA   rQ   relative_position_buckettorP   r   rb   rR   rB   rk   )r'   rc   r3   r   ZbszqlenZklenZcontext_positionZmemory_positionrelative_positionZ	rp_bucketvaluesr)   r)   r*   r   n  s    "
z"MPNetEncoder.compute_position_bias   c                 C   s   d}|  }|d }||dk  tj| 7 }t|}|d }||k }|t| | t||  ||   tj }t|t||d }|t	|||7 }|S )Nr   r`   r   )
r   r@   rQ   abslogfloatri   minZ	full_likewhere)r   r   Zmax_distanceretnZ	max_exactZis_smallZval_if_larger)   r)   r*   r     s    
&z%MPNetEncoder.relative_position_bucket)NNFFF)Nr   )r   r   )r,   r-   r.   r7   r@   r   r   boolrN   r   staticmethodr   rS   r)   r)   rD   r*   r   <  s"   
     *
r   c                       s0   e Zd Z fddZejejdddZ  ZS )MPNetPoolerc                    s*   t    t|j|j| _t | _d S r   )r6   r7   r   r   r9   r   ZTanh
activationrC   rD   r)   r*   r7     s    
zMPNetPooler.__init__r   c                 C   s(   |d d df }|  |}| |}|S Nr   )r   r   )r'   rl   Zfirst_token_tensorpooled_outputr)   r)   r*   rN     s    

zMPNetPooler.forwardr   r)   r)   rD   r*   r     s   r   a?  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`MPNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z_The bare MPNet Model transformer outputting raw hidden-states without any specific head on top.c                       s   e Zd Zd fdd	Zdd Zdd Zdd	 Zee	d
e
eeeddeej eej eej eej eej ee ee ee eeej ef d	ddZ  ZS )
MPNetModelTc                    sD   t  | || _t|| _t|| _|r2t|nd | _| 	  d S r   )
r6   r7   r"   r1   rM   r   encoderr   pooler	post_init)r'   r"   add_pooling_layerrD   r)   r*   r7     s    

zMPNetModel.__init__c                 C   s   | j jS r   rM   r:   r'   r)   r)   r*   get_input_embeddings  s    zMPNetModel.get_input_embeddingsc                 C   s   || j _d S r   r   )r'   valuer)   r)   r*   set_input_embeddings  s    zMPNetModel.set_input_embeddingsc                 C   s*   |  D ]\}}| jj| j| qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr   r   r   r|   )r'   Zheads_to_pruner   rz   r)   r)   r*   _prune_heads  s    zMPNetModel._prune_headsbatch_size, sequence_length
checkpointoutput_typer/   N)	rI   rm   r3   rn   rJ   rp   r   r   r   c	                 K   sR  |d k	r|n| j j}|d k	r |n| j j}|d k	r4|n| j j}|d k	rV|d k	rVtdn@|d k	rt| || | }
n"|d k	r| d d }
ntd|d k	r|jn|j}|d krtj	|
|d}| 
||
}| || j j}| j|||d}| j||||||d}|d }| jd k	r | |nd }|s>||f|dd   S t|||j|jd	S )
NzDYou cannot specify both input_ids and inputs_embeds at the same timer4   z5You have to specify either input_ids or inputs_embeds)rP   )rI   r3   rJ   )rm   rn   rp   r   r   r   r   )r   Zpooler_outputrl   r   )r"   rp   r   use_return_dictrX   Z%warn_if_padding_and_no_attention_maskrH   rP   r@   ZonesZget_extended_attention_maskZget_head_maskr   rM   r   r   r   rl   r   )r'   rI   rm   r3   rn   rJ   rp   r   r   rK   rL   rP   Zextended_attention_maskZembedding_outputZencoder_outputssequence_outputr   r)   r)   r*   rN     sH    

zMPNetModel.forward)T)NNNNNNNN)r,   r-   r.   r7   r   r   r   r   MPNET_INPUTS_DOCSTRINGformatr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOCr   r@   
LongTensorFloatTensorr   r   r   r   rN   rS   r)   r)   rD   r*   r     s:           r   c                       s   e Zd ZdgZ fddZdd Zdd Zee	de
eeed	deej eej eej eej eej eej ee ee ee eeej ef d
ddZ  ZS )MPNetForMaskedLMzlm_head.decoderc                    s0   t  | t|dd| _t|| _|   d S NF)r   )r6   r7   r   r   MPNetLMHeadlm_headr   rC   rD   r)   r*   r7   @  s    
zMPNetForMaskedLM.__init__c                 C   s   | j jS r   r   decoderr   r)   r)   r*   get_output_embeddingsI  s    z&MPNetForMaskedLM.get_output_embeddingsc                 C   s   || j _d S r   r   )r'   Znew_embeddingsr)   r)   r*   set_output_embeddingsL  s    z&MPNetForMaskedLM.set_output_embeddingsr   r   N
rI   rm   r3   rn   rJ   labelsrp   r   r   r   c
              
   C   s   |	dk	r|	n| j j}	| j||||||||	d}
|
d }| |}d}|dk	rnt }||d| j j|d}|	s|f|
dd  }|dk	r|f| S |S t|||
j|
j	dS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        Nrm   r3   rn   rJ   rp   r   r   r   r4   r`   losslogitsrl   r   )
r"   r   r   r   r   ra   r8   r   rl   r   )r'   rI   rm   r3   rn   rJ   r   rp   r   r   rr   r   Zprediction_scoresZmasked_lm_lossloss_fctr   r)   r)   r*   rN   O  s4    
zMPNetForMaskedLM.forward)	NNNNNNNNN)r,   r-   r.   Z_tied_weights_keysr7   r   r   r   r   r   r   r   r   r   r   r@   r   r   r   r   r   r   rN   rS   r)   r)   rD   r*   r   =  s>   	         r   c                       s(   e Zd ZdZ fddZdd Z  ZS )r   z5MPNet Head for masked and permuted language modeling.c                    sh   t    t|j|j| _tj|j|jd| _tj|j|j	dd| _
tt|j	| _| j| j
_d S )Nr2   F)r#   )r6   r7   r   r   r9   r   r&   r<   
layer_normr8   r   	Parameterr@   Zzerosr#   rC   rD   r)   r*   r7     s    
zMPNetLMHead.__init__c                 K   s*   |  |}t|}| |}| |}|S r   )r   r   r   r   r'   featuresrK   rc   r)   r)   r*   rN     s
    


zMPNetLMHead.forwardr,   r-   r.   __doc__r7   rN   rS   r)   r)   rD   r*   r     s   r   z
    MPNet Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej ee ee ee eeej e	f d
ddZ  ZS )
MPNetForSequenceClassificationc                    s8   t  | |j| _t|dd| _t|| _|   d S r   )r6   r7   
num_labelsr   r   MPNetClassificationHead
classifierr   rC   rD   r)   r*   r7     s
    
z'MPNetForSequenceClassification.__init__r   r   Nr   c
              
   C   sp  |	dk	r|	n| j j}	| j||||||||	d}
|
d }| |}d}|dk	r,| j jdkr| jdkrnd| j _n4| jdkr|jtjks|jtj	krd| j _nd| j _| j jdkrt
 }| jdkr|| | }n
|||}nN| j jdkrt }||d| j|d}n| j jdkr,t }|||}|	s\|f|
d	d  }|dk	rX|f| S |S t|||
j|
jd
S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr   r   r   Z
regressionZsingle_label_classificationZmulti_label_classificationr4   r`   r   )r"   r   r   r   Zproblem_typer   rO   r@   rQ   rY   r   squeezer   ra   r   r   rl   r   r'   rI   rm   r3   rn   rJ   r   rp   r   r   rr   r   r   r   r   r   r)   r)   r*   rN     sR    




"


z&MPNetForSequenceClassification.forward)	NNNNNNNNN)r,   r-   r.   r7   r   r   r   r   r   r   r   r   r@   r   r   r   r   r   r   rN   rS   r)   r)   rD   r*   r     s8   
         r   z
    MPNet Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej ee ee ee eeej e	f d
ddZ  ZS )
MPNetForMultipleChoicec                    s@   t  | t|| _t|j| _t|j	d| _
|   d S )Nr   )r6   r7   r   r   r   r=   r>   r?   r   r9   r   r   rC   rD   r)   r*   r7     s
    
zMPNetForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthr   Nr   c
              
   C   sT  |	dk	r|	n| j j}	|dk	r&|jd n|jd }
|dk	rJ|d|dnd}|dk	rh|d|dnd}|dk	r|d|dnd}|dk	r|d|d|dnd}| j||||||||	d}|d }| |}| |}|d|
}d}|dk	rt }|||}|	s@|f|dd  }|dk	r<|f| S |S t	|||j
|jdS )aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r4   re   )r3   rm   rn   rJ   rp   r   r   r`   r   )r"   r   shapera   rH   r   r?   r   r   r   rl   r   )r'   rI   rm   r3   rn   rJ   r   rp   r   r   Znum_choicesZflat_input_idsZflat_position_idsZflat_attention_maskZflat_inputs_embedsrr   r   r   Zreshaped_logitsr   r   r   r)   r)   r*   rN     sH    




zMPNetForMultipleChoice.forward)	NNNNNNNNN)r,   r-   r.   r7   r   r   r   r   r   r   r   r   r@   r   r   r   r   r   r   rN   rS   r)   r)   rD   r*   r     s8   
         r   z
    MPNet Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej ee ee ee eeej e	f d
ddZ  ZS )
MPNetForTokenClassificationc                    sN   t  | |j| _t|dd| _t|j| _t	|j
|j| _|   d S r   )r6   r7   r   r   r   r   r=   r>   r?   r   r9   r   r   rC   rD   r)   r*   r7   ]  s    z$MPNetForTokenClassification.__init__r   r   Nr   c
              
   C   s   |	dk	r|	n| j j}	| j||||||||	d}
|
d }| |}| |}d}|dk	rvt }||d| j|d}|	s|f|
dd  }|dk	r|f| S |S t|||
j	|
j
dS )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr   r   r4   r`   r   )r"   r   r   r?   r   r   ra   r   r   rl   r   r   r)   r)   r*   rN   h  s6    

z#MPNetForTokenClassification.forward)	NNNNNNNNN)r,   r-   r.   r7   r   r   r   r   r   r   r   r   r@   r   r   r   r   r   r   rN   rS   r)   r)   rD   r*   r   U  s8            r   c                       s(   e Zd ZdZ fddZdd Z  ZS )r   z-Head for sentence-level classification tasks.c                    s@   t    t|j|j| _t|j| _t|j|j	| _
d S r   )r6   r7   r   r   r9   r   r=   r>   r?   r   out_projrC   rD   r)   r*   r7     s    
z MPNetClassificationHead.__init__c                 K   sL   |d d dd d f }|  |}| |}t|}|  |}| |}|S r   )r?   r   r@   tanhr   r   r)   r)   r*   rN     s    




zMPNetClassificationHead.forwardr   r)   r)   rD   r*   r     s   r   z
    MPNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej ee ee ee eeej e	f dddZ  ZS )
MPNetForQuestionAnsweringc                    s@   t  | |j| _t|dd| _t|j|j| _| 	  d S r   )
r6   r7   r   r   r   r   r   r9   
qa_outputsr   rC   rD   r)   r*   r7     s
    z"MPNetForQuestionAnswering.__init__r   r   N)rI   rm   r3   rn   rJ   start_positionsend_positionsrp   r   r   r   c              
   C   sN  |
dk	r|
n| j j}
| j|||||||	|
d}|d }| |}|jddd\}}|d }|d }d}|dk	r|dk	rt| dkr|d}t| dkr|d}|d}|	d|}|	d|}t
|d}|||}|||}|| d }|
s8||f|dd  }|dk	r4|f| S |S t||||j|jd	S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        Nr   r   r   r4   rf   )Zignore_indexr`   )r   start_logits
end_logitsrl   r   )r"   r   r   r   splitr   rk   rx   rH   clampr   r   rl   r   )r'   rI   rm   r3   rn   rJ   r   r   rp   r   r   rr   r   r   r   r   Z
total_lossZignored_indexr   Z
start_lossZend_lossr   r)   r)   r*   rN     sN    






z!MPNetForQuestionAnswering.forward)
NNNNNNNNNN)r,   r-   r.   r7   r   r   r   r   r   r   r   r   r@   r   r   r   r   r   r   rN   rS   r)   r)   rD   r*   r     s<   
          r   c                 C   s2   |  | }tj|dd|| }| | S )z
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`. :param torch.Tensor x: :return torch.Tensor:
    r   rf   )nerY   r@   ZcumsumZtype_asrQ   )rI   r%   maskZincremental_indicesr)   r)   r*   rF     s    rF   )>r   ri   typingr   r   r   r@   r   Ztorch.nnr   r   r   Zactivationsr
   r   Zmodeling_outputsr   r   r   r   r   r   r   Zmodeling_utilsr   Zpytorch_utilsr   r   utilsr   r   r   r   Zconfiguration_mpnetr   Z
get_loggerr,   loggerr   r   r0   r   Moduler1   rT   rs   r~   r   r   r   r   ZMPNET_START_DOCSTRINGr   r   r   r   r   r   r   r   r   rF   r)   r)   r)   r*   <module>   sp   $	
;I/ Z*ZKUQF[