U
    ,-ej                    @   sx  d Z ddlZddlmZ ddlmZmZmZmZ ddl	Z	ddl	m
Z
 ddlmZmZmZ ddlmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZmZmZmZmZmZ ddl m!Z! e"e#Z$dZ%dZ&ddgZ'dAddZ(dd Z)G dd de
j*Z+G dd de
j*Z,G dd de
j*Z-G dd deZ.eG dd deZ/eG dd  d eZ0eG d!d" d"eZ1eG d#d$ d$eZ2eG d%d& d&eZ3eG d'd( d(eZ4eG d)d* d*eZ5d+Z6d,Z7ed-e6G d.d/ d/e.Z8ed0e6G d1d2 d2e.Z9ed3e6G d4d5 d5e.Z:ed6e6G d7d8 d8e.Z;ed9e6G d:d; d;e.Z<ed<e6G d=d> d>e.Z=ed<e6G d?d@ d@e.Z>dS )Bz
 PyTorch XLNet model.
    N)	dataclass)ListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)PoolerAnswerClassPoolerEndLogitsPoolerStartLogitsPreTrainedModelSequenceSummary)apply_chunking_to_forward)ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )XLNetConfigzxlnet-base-casedr   zxlnet-large-casedc                 C   s.  i }t | drt | dr$| jj|d< t | drRd|krR| jjj|d< | jjj|d< t | dr|jdk	rd	|j d
|kr| jj|d	|j d
< | jj|d	|j d< | j} |	| j
j| jd t| jD ]\}}d| d}|	|d |jjj|d |jjj|d |jj|d |jj|d |jj|d |jj|d |jj|d |jjj|d |jjj|d |jjj|d |jjj|d |jjj|d |jjji q|jrg }g }g }	g }
| jD ]>}||jj ||jj |	|jj |
|jj qn | jg}| jg}| jg}	| jg}
|	|||	|
d |S )z
    A map of modules from TF to PyTorch. I use a map to keep the PyTorch model as identical to the original PyTorch
    model as possible.
    transformerlm_losszmodel/lm_loss/biassequence_summaryz%model/sequnece_summary/summary/kernelz#model/sequnece_summary/summary/biaslogits_projNzmodel/regression_z/logit/kernelz/logit/bias)z-model/transformer/word_embedding/lookup_tablez#model/transformer/mask_emb/mask_embzmodel/transformer/layer_/zrel_attn/LayerNorm/gammazrel_attn/LayerNorm/betazrel_attn/o/kernelzrel_attn/q/kernelzrel_attn/k/kernelzrel_attn/r/kernelzrel_attn/v/kernelzff/LayerNorm/gammazff/LayerNorm/betazff/layer_1/kernelzff/layer_1/biaszff/layer_2/kernelzff/layer_2/bias)zmodel/transformer/r_r_biaszmodel/transformer/r_w_biaszmodel/transformer/r_s_biaszmodel/transformer/seg_embed)hasattrr   biasr   summaryweightZfinetuning_taskr   r   updateword_embeddingmask_emb	enumeratelayerrel_attn
layer_normoqkrvfflayer_1layer_2Zuntie_rappendr_r_biasr_w_biasr_s_bias	seg_embed)modelconfig
tf_weightstf_to_pt_mapibZ	layer_strZr_r_listZr_w_listZr_s_listZseg_embed_list r>   i/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/xlnet/modeling_xlnet.pybuild_tf_xlnet_to_pytorch_map5   s    

             
r@   c                 C   s  zddl }ddl}W n  tk
r4   td  Y nX |j|}i }|D ]4\}}td| d|  |j||}	|	||< qJt	| ||}
|

 D ]\}}td|  ||krt| d q|| }	d|krd	|ksd
|ksd|krtd ||	}	t|trt||	jd ksHtdt| d|	jd  dt|D ]\}}|	|df }z,|j|jkstd|j d|j dW n< tk
r } z| j|j|jf7  _ W 5 d}~X Y nX td| d|  t||_qPnz,|j|	jks td|j d|	j dW n< tk
r^ } z| j|j|	jf7  _ W 5 d}~X Y nX td|  t|	|_||d ||d d ||d d qtdd|   | S )z&Load tf checkpoints in a pytorch modelr   NzLoading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.zLoading TF weight z with shape z
Importing z( not in tf pre-trained weights, skippingZkernelr0   r"   ZlogitZTransposingzPointer length z and array length z mismatched.zPointer shape z and array shape zInitialize PyTorch weight z for layer z/Adamz/Adam_1z%Weights not copied to PyTorch model: z, )numpyZ
tensorflowImportErrorloggererrortrainZlist_variablesinfoZload_variabler@   items	transpose
isinstancelistlenshapeAssertionErrorr'   argstorchZ
from_numpydatapopjoinkeys)r8   r9   Ztf_pathnptfZ	init_varsr:   namerL   arrayr;   Zpointerr<   Zp_iZarr_ier>   r>   r?   load_tf_weights_in_xlnet   sp    
$



rY   c                       s^   e Zd Z fddZdd ZedddZeddd	ZdddZdddZ	dddZ
  ZS )XLNetRelativeAttentionc                    sn  t    |j|j dkr2td|j d|j |j| _|j| _|j| _d|jd  | _tt	
|j| j| j| _tt	
|j| j| j| _tt	
|j| j| j| _tt	
|j| j| j| _tt	
|j| j| j| _tt	
| j| j| _tt	
| j| j| _tt	
| j| j| _tt	
d| j| j| _tj|j|jd| _t|j| _d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (r         ?   Zeps)super__init__d_modelZn_head
ValueErrorZd_headscaler   	ParameterrO   FloatTensorr,   r-   r/   r+   r.   r4   r6   r5   r7   	LayerNormlayer_norm_epsr*   Dropoutdropoutselfr9   	__class__r>   r?   r_      s(    
zXLNetRelativeAttention.__init__c                 C   s   t d S NNotImplementedError)rj   Zheadsr>   r>   r?   prune_heads   s    z"XLNetRelativeAttention.prune_headsc              	   C   s|   | j }| |d |d |d |d } | dddf } | |d |d d |d |d } t| dtj|| jtjd} | S )z<perform relative shift to form the relative attention score.r   r   r\   r   N.devicedtyperL   ZreshaperO   index_selectarangers   longxklenZx_sizer>   r>   r?   	rel_shift   s     $z XLNetRelativeAttention.rel_shiftc              	   C   s   | j }| |d |d |d |d } | d d d d dd d d f } | |d |d |d |d d } t| dtj|| jtjd} | S )Nr   r   r   r\   rr   ru   ry   r>   r>   r?   rel_shift_bnij   s      $z%XLNetRelativeAttention.rel_shift_bnijNFc	                 C   s  t d|| j |}	t d|| j |}
| j|
|	jd d}
|dkrJd}n$t d|| j | j}t d||}|	|
 | | j }|dk	r|j	t j
kr|dt d	|  }n|d
t d	|  }tjj|dd}| |}|dk	r|t d	| }t d||}|r|t d|fS |S )z.Core relative positional attention operations.zibnd,jbnd->bnijr   )r{   Nr   zibnd,snd->ibnszijbs,ibns->bniji  z
ijbn->bnijgꌠ9Y>)Fdimzbnij,jbnd->ibndz
bnij->ijbn)rO   einsumr5   r4   r}   rL   r6   r7   rb   rt   Zfloat16r   
functionalsoftmaxrh   )rj   Zq_headk_head_hv_head_hk_head_rseg_mat	attn_mask	head_maskoutput_attentionsacZbdZefZ
attn_score	attn_probattn_vecr>   r>   r?   rel_attn_core  s(    
z$XLNetRelativeAttention.rel_attn_coreTc                 C   s4   t d|| j}| |}|r&|| }| |}|S )zPost-attention processing.zibnd,hnd->ibh)rO   r   r+   rh   r*   )rj   hr   ZresidualZattn_outoutputr>   r>   r?   post_attention;  s    

z%XLNetRelativeAttention.post_attentionc              
   C   s  |d k	rJ|d k	r2|  dkr2tj||gdd}n|}td|| j}td|| j}td|| j}td|| j}| j|||||||	|
d}|
r|\}}| 	||}td|| j}|d k	rtd||}| j|||||||	|
d}|
r|\}}td||}n(| j|||||||	|
d}|
r.|\}}| 	||}|
r||f}n|d k	rv|  dkrvtj||gdd}n|}td|| j}td|| j}td|| j}td|
| jj| j}| j|||||||	|
d}|
r|\}}| 	||}d }||f}|
r||f }|S )Nr   r   r~   zibh,hnd->ibnd)r   r   r   r   zmbnd,mlb->lbndzlbnd,mlb->mbnd)r   rO   catr   r-   r/   r.   r,   r   r   typert   )rj   r   gattn_mask_hattn_mask_gr.   r   memstarget_mappingr   r   r   r   r   r   Zq_head_hZ
attn_vec_hZattn_prob_houtput_hZq_head_gZ
attn_vec_gZattn_prob_goutput_gr   r   outputsr>   r>   r?   forwardG  s    



zXLNetRelativeAttention.forward)rq   )rq   )NNNF)T)NNNF)__name__
__module____qualname__r_   rp   staticmethodr|   r}   r   r   r   __classcell__r>   r>   rk   r?   rZ      s"       
4
    rZ   c                       s$   e Zd Z fddZdd Z  ZS )XLNetFeedForwardc                    sv   t    tj|j|jd| _t|j|j| _	t|j|j| _
t|j| _t|jtrjt|j | _n|j| _d S )Nr]   )r^   r_   r   re   r`   rf   r*   LinearZd_innerr1   r2   rg   rh   rI   Zff_activationstrr   activation_functionri   rk   r>   r?   r_     s    
zXLNetFeedForward.__init__c                 C   sH   |}|  |}| |}| |}| |}| |}| || }|S rm   )r1   r   rh   r2   r*   )rj   Zinpr   r>   r>   r?   r     s    




zXLNetFeedForward.forward)r   r   r   r_   r   r   r>   r>   rk   r?   r     s   r   c                       s.   e Zd Z fddZd	ddZdd Z  ZS )

XLNetLayerc                    s>   t    t|| _t|| _t|j| _|j	| _	d| _
d S Nr   )r^   r_   rZ   r)   r   r0   r   rg   rh   chunk_size_feed_forwardseq_len_dimri   rk   r>   r?   r_     s    


zXLNetLayer.__init__NFc                 C   sv   | j |||||||||	|
d
}|d d \}}|d k	rJt| j| j| j|}t| j| j| j|}||f|dd   }|S )N)r   r   r   r   r\   )r)   r   ff_chunkr   r   )rj   r   r   r   r   r.   r   r   r   r   r   r   r>   r>   r?   r     s.       zXLNetLayer.forwardc                 C   s   |  |}|S rm   )r0   )rj   Zoutput_xr>   r>   r?   r     s    
zXLNetLayer.ff_chunk)NNNF)r   r   r   r_   r   r   r   r>   r>   rk   r?   r     s       
$r   c                   @   s$   e Zd ZdZeZeZdZdd Z	dS )XLNetPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    r   c              	   C   s  t |tjr:|jjjd| jjd |jdk	r8|jj	  nt |tj
rz|jjjd| jjd |jdk	rx|jj|j 	  nt |tjr|jj	  |jjd npt |tr|j|j|j|j|j|j|j|j|jf	D ]}|jjd| jjd qn"t |tr|jjjd| jjd dS )zInitialize the weights.g        )ZmeanZstdN      ?)rI   r   r   r#   rP   Znormal_r9   Zinitializer_ranger!   Zzero_	EmbeddingZpadding_idxre   Zfill_rZ   r,   r-   r/   r+   r.   r4   r6   r5   r7   
XLNetModelr&   )rj   moduleparamr>   r>   r?   _init_weights"  s2    


z"XLNetPreTrainedModel._init_weightsN)
r   r   r   __doc__r   config_classrY   Zload_tf_weightsZbase_model_prefixr   r>   r>   r>   r?   r     s
   r   c                   @   s^   e Zd ZU dZejed< dZee	ej  ed< dZ
eeej  ed< dZeeej  ed< dS )XLNetModelOutputa@  
    Output type of [`XLNetModel`].

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_predict, hidden_size)`):
            Sequence of hidden-states at the last layer of the model.

            `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
            corresponds to `sequence_length`.
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    last_hidden_stateNr   hidden_states
attentions)r   r   r   r   rO   rd   __annotations__r   r   r   r   r   r   r>   r>   r>   r?   r   B  s
   

r   c                   @   st   e Zd ZU dZdZeej ed< dZ	ejed< dZ
eeej  ed< dZeeej  ed< dZeeej  ed< dS )XLNetLMHeadModelOutputa  
    Output type of [`XLNetLMHeadModel`].

    Args:
        loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided)
            Language modeling loss (for next-token prediction).
        logits (`torch.FloatTensor` of shape `(batch_size, num_predict, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).

            `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
            corresponds to `sequence_length`.
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nlosslogitsr   r   r   r   r   r   r   r   r   rO   rd   r   r   r   r   r   r   r   r>   r>   r>   r?   r   d  s   
r   c                   @   st   e Zd ZU dZdZeej ed< dZ	ejed< dZ
eeej  ed< dZeeej  ed< dZeeej  ed< dS )$XLNetForSequenceClassificationOutputa_  
    Output type of [`XLNetForSequenceClassification`].

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `label` is provided):
            Classification (or regression if config.num_labels==1) loss.
        logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
            Classification (or regression if config.num_labels==1) scores (before SoftMax).
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr   r   r   r   r   r   r>   r>   r>   r?   r     s   
r   c                   @   st   e Zd ZU dZdZeej ed< dZ	ejed< dZ
eeej  ed< dZeeej  ed< dZeeej  ed< dS )!XLNetForTokenClassificationOutputa%  
    Output type of [`XLNetForTokenClassificationOutput`].

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided) :
            Classification loss.
        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
            Classification scores (before SoftMax).
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr   r   r   r   r   r   r>   r>   r>   r?   r     s   
r   c                   @   st   e Zd ZU dZdZeej ed< dZ	ejed< dZ
eeej  ed< dZeeej  ed< dZeeej  ed< dS )XLNetForMultipleChoiceOutputad  
    Output type of [`XLNetForMultipleChoice`].

    Args:
        loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided):
            Classification loss.
        logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
            *num_choices* is the second dimension of the input tensors. (see *input_ids* above).

            Classification scores (before SoftMax).
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr   r   r   r   r   r   r>   r>   r>   r?   r     s   
r   c                   @   s   e Zd ZU dZdZeej ed< dZ	ejed< dZ
ejed< dZeeej  ed< dZeeej  ed< dZeeej  ed< dS )	%XLNetForQuestionAnsweringSimpleOutputa  
    Output type of [`XLNetForQuestionAnsweringSimple`].

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
        start_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
            Span-start scores (before SoftMax).
        end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
            Span-end scores (before SoftMax).
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr   start_logits
end_logitsr   r   r   )r   r   r   r   r   r   rO   rd   r   r   r   r   r   r   r   r   r>   r>   r>   r?   r     s   
r   c                   @   s   e Zd ZU dZdZeej ed< dZ	eej ed< dZ
eej ed< dZeej ed< dZeej ed< dZeej ed< dZeeej  ed	< dZeeej  ed
< dZeeej  ed< dS )XLNetForQuestionAnsweringOutputam  
    Output type of [`XLNetForQuestionAnswering`].

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
            Classification loss as the sum of start token, end token (and is_impossible if provided) classification
            losses.
        start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top config.start_n_top start token possibilities (beam-search).
        start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top config.start_n_top start token possibilities (beam-search).
        end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
            (beam-search).
        end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
        cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the `is_impossible` label of the answers.
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
            token ids which have their past given to this model should not be passed as `input_ids` as they have
            already been computed.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nr   start_top_log_probsstart_top_indexend_top_log_probsend_top_index
cls_logitsr   r   r   )r   r   r   r   r   r   rO   rd   r   r   r   Z
LongTensorr   r   r   r   r   r   r   r   r>   r>   r>   r?   r     s   
#r   a?  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`XLNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aP  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        mems (`List[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        input_mask (`torch.FloatTensor` of shape `{0}`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z_The bare XLNet Model transformer outputting raw hidden-states without any specific head on top.c                       s   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Ze	dddZ
dddZeedeeeeddeej eej eej eej eej eej eej eej eej ee ee ee ee eeef dddZ  ZS )r   c                    s   t     j| _ j| _ j| _ j| _ j| _ j| _ j| _ j	| _	t
 j j| _t
tdd j| _t
 fddt j	D | _t
 j| _|   d S )Nr   c                    s   g | ]}t  qS r>   )r   ).0_r9   r>   r?   
<listcomp>  s     z'XLNetModel.__init__.<locals>.<listcomp>)r^   r_   mem_len	reuse_lenr`   same_length	attn_typebi_data	clamp_lenn_layerr   r   
vocab_sizer%   rc   rO   rd   r&   Z
ModuleListranger(   rg   rh   	post_initri   rk   r   r?   r_     s     zXLNetModel.__init__c                 C   s   | j S rm   r%   rj   r>   r>   r?   get_input_embeddings  s    zXLNetModel.get_input_embeddingsc                 C   s
   || _ d S rm   r   rj   Znew_embeddingsr>   r>   r?   set_input_embeddings  s    zXLNetModel.set_input_embeddingsc                 C   s   t d S rm   rn   )rj   Zheads_to_pruner>   r>   r?   _prune_heads  s    zXLNetModel._prune_headsc                 C   sr   t ||| | j}| jr`|ddd|f d}||d  |ddd|f  |7  < n||d  |S )aD  
        Creates causal attention mask. Float mask where 1.0 indicates masked, 0.0 indicates not-masked.

        Args:
            qlen: Sequence length
            mlen: Mask length

        ::

                  same_length=False: same_length=True: <mlen > < qlen > <mlen > < qlen >
               ^ [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 1 1 1 1]
                 [0 0 0 0 0 0 1 1 1] [1 0 0 0 0 0 1 1 1]
            qlen [0 0 0 0 0 0 0 1 1] [1 1 0 0 0 0 0 1 1]
                 [0 0 0 0 0 0 0 0 1] [1 1 1 0 0 0 0 0 1]
               v [0 0 0 0 0 0 0 0 0] [1 1 1 1 0 0 0 0 0]

        Nrq   r   )rO   Zonesrs   r   ZtrilZtriu_)rj   qlenmlenmaskZmask_lor>   r>   r?   create_mask  s    zXLNetModel.create_maskc                 C   s|   | j d k	r"| j dkr"|d | j  }| jd ks6| jdkr<d}n| j }|d krZ||d  }ntj||gdd|d  }| S )Nr   r~   )r   r   rO   r   detach)rj   Zcurr_outZprev_memcutoffZnew_memr>   r>   r?   	cache_mem  s    zXLNetModel.cache_memNc                 C   s\   t d| |}t jt |t |gdd}|d d d d d f }|d k	rX|d|d}|S )Nzi,d->idrq   r~   )rO   r   r   sincosexpand)Zpos_seqinv_freqbszZsinusoid_inppos_embr>   r>   r?   positional_embedding  s    zXLNetModel.positional_embeddingc                 C   sh  t jd| jdt jd}dt d|| j  }| jdkrD||  }}n(| jdkrZ|d }}ntd	| j d
| jr*t j||dt jd}t j| | dt jd}	| jdkr|	| j | j}|		| j | j}	|d k	 r| 
|||d }
| 
|	||d }n| 
||}
| 
|	|}t j|
|gdd}n:t ||d}| jdkrV|	| j | j}| 
|||}|S )Nr   g       @rt   r   i'  biunirq   zUnknown `attn_type` .g      r   r\   r~   )rO   rw   r`   floatpowr   ra   r   r   clampr   r   )rj   r   r{   r   Zfreq_seqr   begendZfwd_pos_seqZbwd_pos_seqZfwd_pos_embZbwd_pos_embr   r>   r>   r?   relative_positional_encoding  s0    



z'XLNetModel.relative_positional_encodingbatch_size, sequence_length
checkpointoutput_typer   )	input_idsattention_maskr   	perm_maskr   token_type_ids
input_maskr   inputs_embedsuse_memsr   output_hidden_statesreturn_dictreturnc           (      K   s:  |d k	r|n| j j}|d k	r |n| j j}|d k	r4|n| j j}d|krXtdt |d }
| jrt|
d k	rj|
n| j j}
n|
d k	r|
n| j j	}
|d k	r|	d k	rt
dnj|d k	r|dd }|jd |jd  }}n:|	d k	r|	dd }	|	jd |	jd  }}nt
d|d k	r&|dd nd }|d k	rD|dd nd }|d k	rb|dd nd }|d k	r|ddd nd }|d k	r|ddd nd }|d k	r|d d k	r|d jd nd}|| }| j}| j}| jdkr| ||}|d d d d d d f }n"| jd	kr(d }nt
d
| j |d ksT|d ksTtd|d krp|d k	rpd| }|d k	r|d k	r|d  | }n<|d k	r|d kr|d  }n|d kr|d k	r|}nd }|d k	rX|dkrt|jd ||g|}tj||gdd}|d kr8|d d d d d d d f }n ||d d d d d d d f 7 }|d k	rp|dk|}|d k	rt|| }|dkrtjt||g||gdd}||d d d d d d f  dk|}nd }|	d k	r|	}n
| |}| |}|d k	r4| j|jd |d}| |}nd }|d k	r|dkrvtj||gtj|d}tj||gdd}n|}|d d d f |d d d f k }tjj |dd|}nd }| j!|||d} | |j} | | } |d k	rf|" dkr*|#d#d#d#d}|| j$dddd}n$|" dkrN|#d#d#d}|jt%| & jd}nd g| j$ }d}!|d krd gt'| j( }|rg nd }"|rg nd }#t)| j(D ]\}$}%|
r|!| *|||$ f }!|r|#+|d k	r||fn| |%||||| |||$ |||$ |d
}&|&d d \}}|r|"+|&d  q|rj|#+|d k	rd||fn| | |d k	r||n|}'|'ddd }'|
sd }!|r|d k	rt,dd |#D }#nt,dd |#D }#|r
|d k	rt,dd |"D }"nt,dd |"D }"|s*t,dd |'|!|#|"fD S t-|'|!|#|"dS )NZ	use_cachezgThe `use_cache` argument is deprecated and will be removed in a future version, use `use_mems` instead.zDYou cannot specify both input_ids and inputs_embeds at the same timer   r   z5You have to specify either input_ids or inputs_embedsr\   r   r   zUnsupported attention type: z8You can only use one of input_mask (uses 1 for padding) r   r~   rq   rt   rs   )Znum_classes)r   r   r>   )r   r   r.   r   r   r   r   r   c                 s   s*   | ]"}|D ]}| d dd V  q
qdS r   r   r\   Npermute
contiguous)r   hsr   r>   r>   r?   	<genexpr>  s       z%XLNetModel.forward.<locals>.<genexpr>c                 s   s    | ]}| d dd V  qdS r  r  )r   r  r>   r>   r?   r    s     c                 s   s    | ]}t d d |D V  qdS )c                 s   s"   | ]}| d ddd V  qdS r\   r   r   r   Nr  )r   Z
att_streamr>   r>   r?   r    s     z/XLNetModel.forward.<locals>.<genexpr>.<genexpr>N)tupler   tr>   r>   r?   r    s    c                 s   s"   | ]}| d ddd V  qdS r	  r  r  r>   r>   r?   r    s     c                 s   s   | ]}|d k	r|V  qd S rm   r>   )r   r/   r>   r>   r?   r    s      )r   r   r   r   ).r9   r   r   use_return_dictwarningswarnFutureWarningZtrainingZuse_mems_trainZuse_mems_evalra   rH   r  rL   r  rt   rs   r   r   rM   rO   zerostor   eyer%   rh   r&   r   rx   r   r   Zone_hotr   r   	unsqueezer   next
parametersrK   r(   r'   r   r3   r
  r   )(rj   r   r   r   r   r   r   r   r   r   r   r   r   r   kwargsr   r   r   r{   Zdtype_floatrs   r   Z	data_maskZ	mems_maskZnon_tgt_maskZ
word_emb_kr   Z
word_emb_qr   Zmem_padZcat_idsr   r   Znew_memsr   r   r<   Zlayer_moduler   r   r>   r>   r?   r   #  s   

  *



 


"(





$





   zXLNetModel.forward)N)N)NNNNNNNNNNNNN)r   r   r   r_   r   r   r   r   r   r   r   r   r   XLNET_INPUTS_DOCSTRINGformatr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOCr   rO   Tensorboolr   r   r   r   r>   r>   rk   r?   r     sX   

&             
r   zt
    XLNet Model with a language modeling head on top (linear layer with weights tied to the input embeddings).
    c                       s   e Zd ZdgZ fddZdd Zdd Zdd	d
Zee	
deeeddeej eej eej eej eej eej eej eej eej eej ee ee ee ee eeef dddZeeej ejeej dddZ  ZS )XLNetLMHeadModelzlm_loss.weightc                    sH   t  | |j| _|j| _t|| _tj|j|j	dd| _
|   d S )NT)r!   )r^   r_   r   r   r   r   r   r   r`   r   r   r   ri   rk   r>   r?   r_     s    
zXLNetLMHeadModel.__init__c                 C   s   | j S rm   r   r   r>   r>   r?   get_output_embeddings  s    z&XLNetLMHeadModel.get_output_embeddingsc                 C   s
   || _ d S rm   r  r   r>   r>   r?   set_output_embeddings  s    z&XLNetLMHeadModel.set_output_embeddingsNc                    s   |j d }tj|dftj|jd}d |rPtj|d d   d f |gdd}ntj||gdd}|j d }tj|||ftj|jd}d|d d d d df< tj|d|ftj|jd}	d|	d d ddf< |||	|d}
|rt fd	d
|D |
d< |
S )Nr   r   r  r\   r~   r   rq   )r   r   r   r   c                 3   s*   | ]"}|d   d d d d f V  qd S rm   r>   r   Z
layer_pastoffsetr>   r?   r  H  s     zAXLNetLMHeadModel.prepare_inputs_for_generation.<locals>.<genexpr>r   )rL   rO   r  rx   rs   r   r   r
  )rj   r   Zpast_key_valuesr   r  Zeffective_batch_sizeZdummy_tokenZsequence_lengthr   r   inputsr>   r#  r?   prepare_inputs_for_generation"  s6    
&
    z.XLNetLMHeadModel.prepare_inputs_for_generationr   r   r   r   r   r   r   r   r   r   r   r   labelsr   r   r   r   r  c                 K   s   |dk	r|n| j j}| j|f||||||||	||||d|}| |d }d}|
dk	r~t }||d|d|
d}|s|f|dd  }|dk	r|f| S |S t|||j|j	|j
dS )a=  
        labels (`torch.LongTensor` of shape `(batch_size, num_predict)`, *optional*):
            Labels for masked language modeling. `num_predict` corresponds to `target_mapping.shape[1]`. If
            `target_mapping` is `None`, then `num_predict` corresponds to `sequence_length`.

            The labels should correspond to the masked input words that should be predicted and depends on
            `target_mapping`. Note in order to perform standard auto-regressive language modeling a *<mask>* token has
            to be added to the `input_ids` (see the `prepare_inputs_for_generation` function and examples below)

            Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored, the loss
            is only computed for labels in `[0, ..., config.vocab_size]`

        Return:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, XLNetLMHeadModel
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-large-cased")
        >>> model = XLNetLMHeadModel.from_pretrained("xlnet-large-cased")

        >>> # We show how to setup inputs to predict a next token using a bi-directional context.
        >>> input_ids = torch.tensor(
        ...     tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)
        ... ).unsqueeze(
        ...     0
        ... )  # We will predict the masked token
        >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
        >>> perm_mask[:, :, -1] = 1.0  # Previous tokens don't see last token
        >>> target_mapping = torch.zeros(
        ...     (1, 1, input_ids.shape[1]), dtype=torch.float
        ... )  # Shape [1, 1, seq_length] => let's predict one token
        >>> target_mapping[
        ...     0, 0, -1
        ... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

        >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping)
        >>> next_token_logits = outputs[
        ...     0
        ... ]  # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]

        >>> # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling.
        >>> input_ids = torch.tensor(
        ...     tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)
        ... ).unsqueeze(
        ...     0
        ... )  # We will predict the masked token
        >>> labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0)
        >>> assert labels.shape[0] == 1, "only one word will be predicted"
        >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
        >>> perm_mask[
        ...     :, :, -1
        ... ] = 1.0  # Previous tokens don't see last token as is done in standard auto-regressive lm training
        >>> target_mapping = torch.zeros(
        ...     (1, 1, input_ids.shape[1]), dtype=torch.float
        ... )  # Shape [1, 1, seq_length] => let's predict one token
        >>> target_mapping[
        ...     0, 0, -1
        ... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

        >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels)
        >>> loss = outputs.loss
        >>> next_token_logits = (
        ...     outputs.logits
        ... )  # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]
        ```Nr   r   r   r   r   r   r   r   r   r   r   r   r   rq   r   r   r   r   r   r   )r9   r  r   r   r	   viewsizer   r   r   r   )rj   r   r   r   r   r   r   r   r   r   r)  r   r   r   r   r  transformer_outputsr   r   loss_fctr   r>   r>   r?   r   L  sD    XzXLNetLMHeadModel.forward)r   beam_idxr  c                    s    fdd| D S )z
        This function is used to re-order the `mems` cache if [`~PreTrainedModel.beam_search`] or
        [`~PreTrainedModel.beam_sample`] is called. This is required to match `mems` with the correct beam_idx at every
        generation step.
        c                    s    g | ]}| d  |jqS )r   )rv   r  rs   r"  r0  r>   r?   r     s     z3XLNetLMHeadModel._reorder_cache.<locals>.<listcomp>r>   )r   r0  r>   r1  r?   _reorder_cache  s    zXLNetLMHeadModel._reorder_cache)NN)NNNNNNNNNNNNNN)r   r   r   Z_tied_weights_keysr_   r   r!  r&  r   r  r  r   r   r  r   rO   r  r  r   r   r   r   r   r2  r   r>   r>   rk   r?   r    sP   
*
              
}r  z
    XLNet Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g.
    for GLUE tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej eej eej eej ee ee ee ee eee	f dddZ  ZS )
XLNetForSequenceClassificationc                    sL   t  | |j| _|| _t|| _t|| _t	|j
|j| _|   d S rm   )r^   r_   
num_labelsr9   r   r   r   r   r   r   r`   r   r   ri   rk   r>   r?   r_     s    

z'XLNetForSequenceClassification.__init__r   r   Nr(  c                 K   s  |dk	r|n| j j}| j|f||||||||	||||d|}|d }| |}| |}d}|
dk	rH| j jdkr| jdkrd| j _n4| jdkr|
jtj	ks|
jtj
krd| j _nd| j _| j jdkrt }| jdkr|| |
 }n
|||
}nN| j jdkr*t }||d| j|
d}n| j jdkrHt }|||
}|sx|f|dd  }|dk	rt|f| S |S t|||j|j|jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr*  r   r   Z
regressionZsingle_label_classificationZmulti_label_classificationrq   r+  )r9   r  r   r   r   Zproblem_typer4  rt   rO   rx   intr
   squeezer	   r,  r   r   r   r   r   )rj   r   r   r   r   r   r   r   r   r   r)  r   r   r   r   r  r.  r   r   r   r/  r>   r>   r?   r     sf    




"


z&XLNetForSequenceClassification.forward)NNNNNNNNNNNNNN)r   r   r   r_   r   r  r  r   r  r   r  r   rO   r  r  r   r   r   r   r>   r>   rk   r?   r3    sL                 
r3  z
    XLNet Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej eej eej eej ee ee ee ee eee	f dddZ  ZS )
XLNetForTokenClassificationc                    s<   t  | |j| _t|| _t|j|j| _| 	  d S rm   )
r^   r_   r4  r   r   r   r   hidden_size
classifierr   ri   rk   r>   r?   r_   J  s
    
z$XLNetForTokenClassification.__init__r   r   Nr(  c                 K   s   |dk	r|n| j j}| j|||||||||	||||d}|d }| |}d}|
dk	rvt }||d| j|
d}|s|f|dd  }|dk	r|f| S |S t|||j|j	|j
dS )a<  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
            where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)
        Nr*  r   rq   r   r+  )r9   r  r   r9  r	   r,  r4  r   r   r   r   )rj   r   r   r   r   r   r   r   r   r   r)  r   r   r   r   r  r   sequence_outputr   r   r/  r   r>   r>   r?   r   T  s@    
z#XLNetForTokenClassification.forward)NNNNNNNNNNNNNN)r   r   r   r_   r   r  r  r   r  r   r  r   rO   r  r  r   r   r   r   r>   r>   rk   r?   r7  B  sL   
              
r7  z
    XLNet Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RACE/SWAG tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej eej eej eej ee ee ee ee eee	f dddZ  ZS )
XLNetForMultipleChoicec                    s<   t  | t|| _t|| _t|jd| _	| 
  d S r   )r^   r_   r   r   r   r   r   r   r`   r   r   ri   rk   r>   r?   r_     s
    

zXLNetForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthr   N)r   r   r   r   r   r   r   r   r   r)  r   r   r   r   r  c                 K   s  |dk	r|n| j j}|dk	r&|jd n|	jd }|dk	rJ|d|dnd}|dk	rh|d|dnd}|dk	r|d|dnd}|dk	r|d|dnd}|	dk	r|	d|	d|	dnd}| j|f||||||||||||d|}|d }| |}| |}|d|}d}|
dk	rFt }|||
d}|sv|f|dd  }|dk	rr|f| S |S t	|||j
|j|jdS )aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rq   )r   r   r   r   r   r   r   r   r   r   r   r   r   r+  )r9   r  rL   r,  r-  r   r   r   r	   r   r   r   r   )rj   r   r   r   r   r   r   r   r   r   r)  r   r   r   r   r  Znum_choicesZflat_input_idsZflat_token_type_idsZflat_attention_maskZflat_input_maskZflat_inputs_embedsr.  r   r   Zreshaped_logitsr   r/  r>   r>   r?   r     s\    


zXLNetForMultipleChoice.forward)NNNNNNNNNNNNNN)r   r   r   r_   r   r  r  r   r  r   r  r   rO   r  r  r   r   r   r   r>   r>   rk   r?   r;    sL   
              
r;  z
    XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej eej eej eej eej ee ee ee ee eee	f dddZ  ZS )
XLNetForQuestionAnsweringSimplec                    s<   t  | |j| _t|| _t|j|j| _| 	  d S rm   )
r^   r_   r4  r   r   r   r   r8  
qa_outputsr   ri   rk   r>   r?   r_     s
    
z(XLNetForQuestionAnsweringSimple.__init__r   r   N)r   r   r   r   r   r   r   r   r   start_positionsend_positionsr   r   r   r   r  c                 K   sd  |dk	r|n| j j}| j|f||||||||	||||d|}|d }| |}|jddd\}}|d }|d }d}|
dk	r|dk	rt|
 dkr|
d}
t| dkr|d}|d}|
	d|}
|	d|}t
|d}|||
}|||}|| d }|sJ||f|dd  }|dk	rF|f| S |S t||||j|j|jd	S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        Nr*  r   r   rq   r~   )Zignore_indexr\   )r   r   r   r   r   r   )r9   r  r   r>  splitr6  r  rK   r-  r   r	   r   r   r   r   )rj   r   r   r   r   r   r   r   r   r   r?  r@  r   r   r   r   r  r   r:  r   r   r   
total_lossZignored_indexr/  
start_lossend_lossr   r>   r>   r?   r     s`    #






z'XLNetForQuestionAnsweringSimple.forward)NNNNNNNNNNNNNNN)r   r   r   r_   r   r  r  r   r  r   r  r   rO   r  r  r   r   r   r   r>   r>   rk   r?   r=     sP   
               
r=  c                       s   e Zd Z fddZeedeee	dd	e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
e e
e e
e e
e eeef dddZ  ZS )
XLNetForQuestionAnsweringc                    sP   t  | |j| _|j| _t|| _t|| _t|| _	t
|| _|   d S rm   )r^   r_   start_n_top	end_n_topr   r   r   r   r   r   r   answer_classr   ri   rk   r>   r?   r_   v  s    



z"XLNetForQuestionAnswering.__init__r   r'  N)r   r   r   r   r   r   r   r   r   r?  r@  is_impossible	cls_indexp_maskr   r   r   r   r  c           -      K   s  |dk	r|n| j j}| j|f||||||||	||||d|}|d }| j||d}|dd }|
dk	rP|dk	rP|
|||fD ]"}|dk	r| dkr|d q| j||
|d}t }|||
}|||}|| d }|dk	r|dk	r| j||
|d	}t	
 }|||} || d
 7 }|s6|f|dd  S t||j|j|jdS nR| \}!}"}#t	jj|dd}$tj|$| jdd\}%}&|&ddd|#}'t|d|'}(|(dd|"dd}(|d|(})|dk	r|dnd}| j|)|(|d}t	jj|dd}*tj|*| jdd\}+},|+d| j| j }+|,d| j| j },td||$}(| j||(|d}|s|%|&|+|,|f}||dd  S t|%|&|+|,||j|j|jdS dS )a>  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels whether a question has an answer or no answer (SQuAD 2.0)
        cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the classification token to use as input for computing plausibility of the
            answer.
        p_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be
            masked. 0.0 mean token is not masked.

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, XLNetForQuestionAnswering
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")
        >>> model = XLNetForQuestionAnswering.from_pretrained("xlnet-base-cased")

        >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(
        ...     0
        ... )  # Batch size 1
        >>> start_positions = torch.tensor([1])
        >>> end_positions = torch.tensor([3])
        >>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions)

        >>> loss = outputs.loss
        ```Nr*  r   )rK  r   rq   )r?  rK  r\   )r?  rJ  r[   )r   r   r   r   r~   r<  )start_statesrK  z
blh,bl->bh)rL  rJ  )r   r   r   r   r   r   r   r   )r9   r  r   r   r   Zsqueeze_r   r	   rH  r   r   r   r   r   r   r-  r   r   rO   ZtopkrF  r  r   gatherZ	expand_asrG  r,  r   )-rj   r   r   r   r   r   r   r   r   r   r?  r@  rI  rJ  rK  r   r   r   r   r  r.  r   r   r   rz   r   r/  rC  rD  rB  r   Zloss_fct_clsZcls_lossr   slenZhszZstart_log_probsr   r   Zstart_top_index_exprL  Zhidden_states_expandedZend_log_probsr   r   r>   r>   r?   r     s    >



	  

  
    z!XLNetForQuestionAnswering.forward)NNNNNNNNNNNNNNNNNN)r   r   r   r_   r   r  r  r   r   r  r   rO   r  r  r   r   r   r   r>   r>   rk   r?   rE  n  sT   
                  
rE  )N)?r   r  dataclassesr   typingr   r   r   r   rO   r   Ztorch.nnr   r	   r
   Zactivationsr   Zmodeling_utilsr   r   r   r   r   Zpytorch_utilsr   utilsr   r   r   r   r   r   Zconfiguration_xlnetr   Z
get_loggerr   rC   r  r  Z#XLNET_PRETRAINED_MODEL_ARCHIVE_LISTr@   rY   ModulerZ   r   r   r   r   r   r   r   r   r   r   ZXLNET_START_DOCSTRINGr  r   r  r3  r7  r;  r=  rE  r>   r>   r>   r?   <module>   s    

SC  2*!$!!#$/I  g HfP`g