U
    ,-e                  
   @   s@  d Z ddlZddlZddlmZ ddlmZmZmZm	Z	 ddl
ZddlZddlmZ ddlmZmZmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZ ddlm Z m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ e(,e-Z.dZ/dZ0ddddddddddg
Z1dd Z2d?ddZ3G dd dej4Z5G d d! d!ej4Z6G d"d# d#eZ7eG d$d% d%e$Z8d&Z9d'Z:e&d(e9G d)d* d*e7Z;G d+d, d,ej4Z<e&d-e9G d.d/ d/e7Z=e&d0e9G d1d2 d2e7Z>e&d3e9G d4d5 d5e7Z?e&d6e9G d7d8 d8e7Z@e&d9e9G d:d; d;e7ZAe&d<e9G d=d> d>e7ZBdS )@z
 PyTorch XLM model.
    N)	dataclass)DictOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )gelu)BaseModelOutputMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModelSequenceSummary	SQuADHead)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )	XLMConfigzxlm-mlm-en-2048r    zxlm-mlm-ende-1024zxlm-mlm-enfr-1024zxlm-mlm-enro-1024zxlm-mlm-tlm-xnli15-1024zxlm-mlm-xnli15-1024zxlm-clm-enfr-1024zxlm-clm-ende-1024zxlm-mlm-17-1280zxlm-mlm-100-1280c              	      s   t  fddt| D }tt |d d dd df |d d dd df< tt |d d dd df |d d dd df< |  d|_d S )Nc                    s$   g | ]  fd dt D qS )c              	      s(   g | ] }t d d|d     qS )i'     )nppower).0j)dimpos e/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/xlm/modeling_xlm.py
<listcomp>H   s     z;create_sinusoidal_embeddings.<locals>.<listcomp>.<listcomp>)range)r$   r&   )r'   r)   r*   H   s     z0create_sinusoidal_embeddings.<locals>.<listcomp>r   r!   r   F)	r"   arrayr+   torchFloatTensorsincosZdetach_Zrequires_grad)Zn_posr&   outZposition_encr(   r,   r)   create_sinusoidal_embeddingsG   s
    44r3   c                 C   s   t j| t j|jd}|dk	r"|}n(|  | ks6t||dddf k }|d}|r|ddddf || d|ddddf k}n|}| || fkst|dks| || | fkst||fS )zH
    Generate hidden states mask, and optionally an attention mask.
    dtypedeviceNr   r   F)	r.   arangelongr6   maxitemAssertionErrorsizerepeat)slenlengthscausalpadding_maskalenmaskbs	attn_maskr(   r(   r)   	get_masksO   s    
0rF   c                       s6   e Zd Ze Z fddZdd Zd	ddZ  Z	S )
MultiHeadAttentionc                    s   t    ttj| _|| _|| _|j| _	| j| j dks>t
t||| _t||| _t||| _t||| _t | _d S )Nr   )super__init__nextrG   NEW_IDlayer_idr&   n_headsattention_dropoutdropoutr;   r   Linearq_link_linv_linout_linsetpruned_heads)selfrM   r&   config	__class__r(   r)   rI   k   s    
zMultiHeadAttention.__init__c                 C   s   | j | j }t|dkrd S t|| j|| j\}}t| j|| _t| j|| _t| j|| _t| j	|dd| _	| jt| | _|| j | _ | j
|| _d S )Nr   r   r,   )r&   rM   lenr   rV   r   rQ   rR   rS   rT   union)rW   headsZattention_head_sizeindexr(   r(   r)   prune_headsy   s    zMultiHeadAttention.prune_headsNFc                    s*  |  \ }}|dkr0|dkr"|n
|d | }	n
| d}	j}
j|
 | dkrb d||	fn
 dd|	f} fdd} fdd}||}|dkr||}||}n6|dksԈj|kr| }}||}||}|dk	rhj|krZ|dkrL|j \}}tj||gd	d
}tj||gd	d
}n|j \}}||f|j< |t	
 }t||d	d}|dk||}||t|jj tjj| dd
|}tjj|jjd}|dk	r|| }t||}||}|f}|r&||f }|S )zd
        Self-attention (if kv is None) or attention over source sentence (provided by kv).
        Nr>   r   r   c                    s   |   djddS )Z
projectionr   r!   )viewrM   	transposexrD   Zdim_per_headrW   r(   r)   shape   s    z)MultiHeadAttention.forward.<locals>.shapec                    s    |  dd  dj S )zcompute contextr   r!   r`   )rb   
contiguousra   rM   rc   re   r(   r)   unshape   s    z+MultiHeadAttention.forward.<locals>.unshaper!   r,   r   r`   ptraining)r<   rM   r&   rQ   rR   rS   rL   r.   catmathsqrtmatmulrb   ra   	expand_asZmasked_fill_Zfinfor5   minr   
functionalZsoftmaxfloatZtype_asrO   rk   rT   )rW   inputrC   kvcache	head_maskoutput_attentionsqlenr&   ZklenrM   Zmask_reshaperf   rh   qkvZk_Zv_scoresweightscontextoutputsr(   re   r)   forward   sN    

$



zMultiHeadAttention.forward)NNNF)
__name__
__module____qualname__	itertoolscountrK   rI   r_   r   __classcell__r(   r(   rY   r)   rG   h   s   rG   c                       s,   e Zd Z fddZdd Zdd Z  ZS )TransformerFFNc                    sT   t    |j| _t||| _t||| _|jr8tntj	j
| _|j| _d| _d S Nr   )rH   rI   rO   r   rP   lin1lin2Zgelu_activationr   rr   Zreluactchunk_size_feed_forwardseq_len_dim)rW   Zin_dimZ
dim_hiddenZout_dimrX   rY   r(   r)   rI      s    
zTransformerFFN.__init__c                 C   s   t | j| j| j|S N)r   ff_chunkr   r   )rW   rt   r(   r(   r)   r      s    zTransformerFFN.forwardc                 C   s8   |  |}| |}| |}tjj|| j| jd}|S )Nri   )r   r   r   r   rr   rO   rk   )rW   rt   rd   r(   r(   r)   r      s
    


zTransformerFFN.ff_chunk)r   r   r   rI   r   r   r   r(   r(   rY   r)   r      s   	r   c                       s@   e Zd ZdZeZdZdZ fddZe	dd Z
dd	 Z  ZS )
XLMPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    Ntransformerc                    s   t  j|| d S r   )rH   rI   )rW   inputskwargsrY   r(   r)   rI      s    zXLMPreTrainedModel.__init__c              	   C   s   t dddddgdddddgdddddgg}t dddddgdddddgdddddgg}| jjr| jjdkrt dddddgdddddgdddddgg}nd }|||d	S )
N      r   r   r!   r         )	input_idsattention_masklangs)r.   tensorrX   use_lang_embn_langs)rW   Zinputs_listZ
attns_listZ
langs_listr(   r(   r)   dummy_inputs   s    ..0zXLMPreTrainedModel.dummy_inputsc                 C   s   t |tjrV| jdk	r:| jjdk	r:tjj|jd| jjd |jdk	rV|jj	|j 
  t |tjr| jdk	r| jjdk	rtjj|jd| jjd |jdk	rtj|jd t |tjr|jj	
  |jj	d dS )zInitialize the weights.Nr   )meanZstdg        g      ?)
isinstancer   	EmbeddingrX   Zembed_init_stdinitZnormal_weightpadding_idxdataZzero_rP   Zinit_stdbiasZ	constant_	LayerNormZfill_)rW   moduler(   r(   r)   _init_weights   s    

z XLMPreTrainedModel._init_weights)r   r   r   __doc__r    config_classZload_tf_weightsZbase_model_prefixrI   propertyr   r   r   r(   r(   rY   r)   r      s   
	r   c                   @   s   e Zd ZU dZdZeej ed< dZ	eej ed< dZ
eej ed< dZeej ed< dZeej ed< dZeej ed< dZeeej  ed	< dZeeej  ed
< dS )XLMForQuestionAnsweringOutputa9
  
    Base class for outputs of question answering models using a `SquadHead`.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
            Classification loss as the sum of start token, end token (and is_impossible if provided) classification
            losses.
        start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top config.start_n_top start token possibilities (beam-search).
        start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top config.start_n_top start token possibilities (beam-search).
        end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
            (beam-search).
        end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
        cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the `is_impossible` label of the answers.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nlossstart_top_log_probsstart_top_indexend_top_log_probsend_top_index
cls_logitshidden_states
attentions)r   r   r   r   r   r   r.   r/   __annotations__r   r   Z
LongTensorr   r   r   r   r   r   r(   r(   r(   r)   r     s   
r   a=  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`XLMConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a8  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        langs (`torch.LongTensor` of shape `({0})`, *optional*):
            A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are
            languages ids which can be obtained from the language names by using two conversion mappings provided in
            the configuration of the model (only provided for multilingual models). More precisely, the *language name
            to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the
            *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string).

            See usage examples detailed in the [multilingual documentation](../multilingual).
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Length of each sentence that can be used to avoid performing attention on padding token indices. You can
            also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in
            `[0, ..., input_ids.size(-1)]`.
        cache (`Dict[str, torch.FloatTensor]`, *optional*):
            Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the
            attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential
            decoding.

            The dictionary object will be modified in-place during the forward pass to add newly computed
            hidden-states.
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z]The bare XLM Model transformer outputting raw hidden-states without any specific head on top.c                       s   e Zd Z fddZdd Zdd Zdd Zee	d	e
eeed
deej eej eej eej eej eej eeeejf  eej eej ee ee ee eeef dddZ  ZS )XLMModelc              	      sX  t  | |j| _|j | _| jr,td|j| _|j| _|j| _|j| _|j	| _	|j
| _
|j| _| jd | _|j| _|j| _|j| _|j| _| j| j dkstdt|j| j| _|jrt|j| j| jjd |jdkr|jrt| j| j| _tj| j| j| j
d| _tj| j|jd| _t | _ t | _!t | _"t | _#t$| jD ]r}| j %t&| j| j|d	 | j!%tj| j|jd | j"%t'| j| j| j|d	 | j#%tj| j|jd qVt(|d
r.|j)* + }i |_)|D ]>\}}| j t,| j|jkr| -t,|t.t/t,|i q| 0  | j1dt23|j4ddd d S )Nz,Currently XLM can only be used as an encoderr   r   z-transformer dim must be a multiple of n_heads)r2   r   )r   )Zeps)rX   rV   position_ids)r   r`   F)
persistent)5rH   rI   Z
is_encoderZ
is_decoderNotImplementedErrorr@   r   r   n_wordsZ	eos_index	pad_indexemb_dimr&   Z
hidden_dimrM   n_layersrO   rN   r;   r   r   Zmax_position_embeddingsposition_embeddingsZsinusoidal_embeddingsr3   r   lang_embeddings
embeddingsr   Zlayer_norm_epslayer_norm_embZ
ModuleListr   layer_norm1ffnslayer_norm2r+   appendrG   r   hasattrrV   copyitemsintr_   listmap	post_initZregister_bufferr.   r7   expand)rW   rX   _rV   layerr]   rY   r(   r)   rI     s\    




   zXLMModel.__init__c                 C   s   | j S r   r   rW   r(   r(   r)   get_input_embeddings  s    zXLMModel.get_input_embeddingsc                 C   s
   || _ d S r   r   rW   Znew_embeddingsr(   r(   r)   set_input_embeddings  s    zXLMModel.set_input_embeddingsc                 C   s&   |  D ]\}}| j| | qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)r   r   r_   )rW   Zheads_to_pruner   r]   r(   r(   r)   _prune_heads  s    zXLMModel._prune_headsbatch_size, sequence_length
checkpointoutput_typer   N)r   r   r   token_type_idsr   r?   rv   rw   inputs_embedsrx   output_hidden_statesreturn_dictreturnc                 C   s  |
d k	r|
n| j j}
|d k	r |n| j j}|d k	r4|n| j j}|d k	rR| \}}n|	 d d \}}|d k	rt|jn|	j}|d kr|d k	r|| jkjdd }nt	j
|g| |d}|d|kst|  |kstt||| j|d\}}|d kr| jd d d |f }n| ||fks*t|d k	rJ| ||fksJt| || j j}|d k	r|d k	r||d  }|d d | d f }|d d | d f }|d k	r|d d | d f }|d d | d f }|d d | d f }|	d kr| |}	|	| ||	 }|d k	rF| jrF| jdkrF|| | }|d k	r^|| | }| |}tjj|| j| jd}||d|j 9 }|rd	nd }|
rd	nd }t!| jD ]}|r||f }| j"| ||||| |
d
}|d }|
r||d f }tjj|| j| jd}|| }| j#| |}|| j$| | }| j%| |}||d|j 9 }q|r|||f }|d k	r|d  |d7  < |st&dd |||fD S t'|||dS )Nr`   r   r,   )r6   r   )rA   r>   ri   r(   )rv   rw   rx   c                 s   s   | ]}|d k	r|V  qd S r   r(   )r$   r|   r(   r(   r)   	<genexpr>k  s      z#XLMModel.forward.<locals>.<genexpr>)Zlast_hidden_stater   r   )(rX   rx   r   use_return_dictr<   r6   r   sumr8   r.   r   r;   r9   r:   rF   r@   r   Zget_head_maskr   r   r   rp   r   r   r   r   r   rr   rO   rk   Z	unsqueezetor5   r+   r   r   r   r   tupler   )rW   r   r   r   r   r   r?   rv   rw   r   rx   r   r   rD   r>   r6   rC   rE   Z_slenr   r   r   iZattn_outputsZattnr(   r(   r)   r     s    










zXLMModel.forward)NNNNNNNNNNNN)r   r   r   rI   r   r   r   r   XLM_INPUTS_DOCSTRINGformatr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOCr   r.   Tensorr   strboolr   r   r   r   r(   r(   rY   r)   r     sJ   H            
r   c                       s*   e Zd ZdZ fddZdddZ  ZS )XLMPredLayerz?
    Prediction layer (cross_entropy or adaptive_softmax).
    c                    sh   t    |j| _|j| _|j| _|j}|jdkrHtj||jdd| _ntj	||j|j
|jdd| _d S )NFT)r   )Zin_featuresZ	n_classesZcutoffsZ	div_valueZ	head_bias)rH   rI   asmr   r   r   r   rP   projZAdaptiveLogSoftmaxWithLossZasm_cutoffsZasm_div_value)rW   rX   r&   rY   r(   r)   rI   t  s    

zXLMPredLayer.__init__Nc                 C   s   d}| j dkrX| |}|f| }|dk	rtjj|d| j|ddd}|f| }n8| j|}|f| }|dk	r| ||\}}|f| }|S )z,Compute the loss, and optionally the scores.r(   FNr`   r   )Z	reduction)r   r   r   rr   Zcross_entropyra   r   Zlog_prob)rW   rd   yr   r}   r   r   r(   r(   r)   r     s    


"

zXLMPredLayer.forward)N)r   r   r   r   rI   r   r   r(   r(   rY   r)   r   o  s   r   z
    The XLM Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                       s   e Zd ZdgZ fddZdd Zdd Zdd	 Zee	
d
eeeedddeej eej eej eej eej eej eeeejf  eej eej eej ee ee ee eeef dddZ  ZS )XLMWithLMHeadModelzpred_layer.proj.weightc                    s,   t  | t|| _t|| _|   d S r   )rH   rI   r   r   r   
pred_layerr   rW   rX   rY   r(   r)   rI     s    

zXLMWithLMHeadModel.__init__c                 C   s   | j jS r   r   r   r   r(   r(   r)   get_output_embeddings  s    z(XLMWithLMHeadModel.get_output_embeddingsc                 C   s   || j _d S r   r   r   r(   r(   r)   set_output_embeddings  s    z(XLMWithLMHeadModel.set_output_embeddingsc                 K   sj   | j j}| j j}|jd }tj|df|tj|jd}tj||gdd}|d k	r\t	||}nd }||dS )Nr   r   r4   r,   )r   r   )
rX   mask_token_idlang_idrf   r.   fullr8   r6   rl   Z	full_like)rW   r   r   r   r   Zeffective_batch_sizeZ
mask_tokenr   r(   r(   r)   prepare_inputs_for_generation  s    
z0XLMWithLMHeadModel.prepare_inputs_for_generationr   z
<special1>)r   r   r   rC   Nr   r   r   r   r   r?   rv   rw   r   labelsrx   r   r   r   c                 C   s   |dk	r|n| j j}| j|||||||||	|||d}|d }| ||
}|s^||dd  S t|
dk	rp|d nd|
dkr|d n|d |j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        Nr   r   r   r   r?   rv   rw   r   rx   r   r   r   r   r   logitsr   r   )rX   r   r   r   r   r   r   )rW   r   r   r   r   r   r?   rv   rw   r   r   rx   r   r   transformer_outputsoutputr   r(   r(   r)   r     s2    zXLMWithLMHeadModel.forward)NNNNNNNNNNNNN)r   r   r   Z_tied_weights_keysrI   r   r   r   r   r   r   r   r   r   r   r   r.   r   r   r   r   r   r   r   r   r(   r(   rY   r)   r     sR                
r   z
    XLM Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g.
    for GLUE tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eeeejf  eej eej eej ee ee ee eee	f dddZ  ZS )
XLMForSequenceClassificationc                    s:   t  | |j| _|| _t|| _t|| _|   d S r   )	rH   rI   
num_labelsrX   r   r   r   sequence_summaryr   r   rY   r(   r)   rI     s    

z%XLMForSequenceClassification.__init__r   r   Nr   c                 C   sx  |dk	r|n| j j}| j|||||||||	|||d}|d }| |}d}|
dk	r4| j jdkr| jdkrvd| j _n4| jdkr|
jtjks|
jtj	krd| j _nd| j _| j jdkrt
 }| jdkr|| |
 }n
|||
}nN| j jdkrt }||d| j|
d}n| j jdkr4t }|||
}|sd|f|dd  }|dk	r`|f| S |S t|||j|jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr   r   r   Z
regressionZsingle_label_classificationZmulti_label_classificationr`   r   )rX   r   r   r  Zproblem_typer  r5   r.   r8   r   r
   squeezer	   ra   r   r   r   r   )rW   r   r   r   r   r   r?   rv   rw   r   r   rx   r   r   r  r  r   r   loss_fctr(   r(   r)   r     sZ    



"


z$XLMForSequenceClassification.forward)NNNNNNNNNNNNN)r   r   r   rI   r   r   r   r   r   r   r   r   r.   r   r   r   r   r   r   r   r   r(   r(   rY   r)   r    sH                
r  z
    XLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eeeejf  eej eej eej eej ee ee ee eee	f dddZ  ZS )
XLMForQuestionAnsweringSimplec                    s4   t  | t|| _t|j|j| _| 	  d S r   )
rH   rI   r   r   r   rP   hidden_sizer  
qa_outputsr   r   rY   r(   r)   rI   h  s    
z&XLMForQuestionAnsweringSimple.__init__r   r   N)r   r   r   r   r   r?   rv   rw   r   start_positionsend_positionsrx   r   r   r   c                 C   sV  |dk	r|n| j j}| j|||||||||	|||d}|d }| |}|jddd\}}|d }|d }d}|
dk	r|dk	rt|
 dkr|
d}
t| dkr|d}|d}|
	d|}
|	d|}t
|d}|||
}|||}|| d }|s@||f|dd  }|dk	r<|f| S |S t||||j|jd	S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        Nr   r   r   r`   r,   )Zignore_indexr!   )r   start_logits
end_logitsr   r   )rX   r   r   r
  splitr  rg   r[   r<   clampr	   r   r   r   )rW   r   r   r   r   r   r?   rv   rw   r   r  r  rx   r   r   r  sequence_outputr   r  r  Z
total_lossZignored_indexr  Z
start_lossZend_lossr  r(   r(   r)   r   q  sV    !






z%XLMForQuestionAnsweringSimple.forward)NNNNNNNNNNNNNN)r   r   r   rI   r   r   r   r   r   r   r   r   r.   r   r   r   r   r   r   r   r   r(   r(   rY   r)   r  `  sL   	              
r  z
    XLM Model with a beam-search span classification head on top for extractive question-answering tasks like SQuAD (a
    linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                       s   e Zd Z fddZeedeee	dd	e
ej e
ej e
ej e
ej e
ej e
ej e
eeejf  e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
e e
e e
e eeef dddZ  ZS )
XLMForQuestionAnsweringc                    s,   t  | t|| _t|| _|   d S r   )rH   rI   r   r   r   r
  r   r   rY   r(   r)   rI     s    

z XLMForQuestionAnswering.__init__r   )r   r   N)r   r   r   r   r   r?   rv   rw   r   r  r  is_impossible	cls_indexp_maskrx   r   r   r   c                 C   s   |dk	r|n| j j}| j|||||||||	|||d}|d }| j||
|||||d}|sj||dd  S t|j|j|j|j|j	|j
|j|jdS )a8  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels whether a question has an answer or no answer (SQuAD 2.0)
        cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the classification token to use as input for computing plausibility of the
            answer.
        p_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be
            masked. 0.0 mean token is not masked.

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, XLMForQuestionAnswering
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("xlm-mlm-en-2048")
        >>> model = XLMForQuestionAnswering.from_pretrained("xlm-mlm-en-2048")

        >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(
        ...     0
        ... )  # Batch size 1
        >>> start_positions = torch.tensor([1])
        >>> end_positions = torch.tensor([3])

        >>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions)
        >>> loss = outputs.loss
        ```Nr   r   )r  r  r  r  r  r   r   )r   r   r   r   r   r   r   r   )rX   r   r   r
  r   r   r   r   r   r   r   r   r   )rW   r   r   r   r   r   r?   rv   rw   r   r  r  r  r  r  rx   r   r   r  r  r   r(   r(   r)   r     sJ    <
zXLMForQuestionAnswering.forward)NNNNNNNNNNNNNNNNN)r   r   r   rI   r   r   r   r   r   r   r   r.   r   r   r   r   r   r   r   r   r(   r(   rY   r)   r    sP   	
                 
r  z
    XLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eeeejf  eej eej eej ee ee ee eee	f dddZ  ZS )
XLMForTokenClassificationc                    sJ   t  | |j| _t|| _t|j| _t|j	|j| _
|   d S r   )rH   rI   r  r   r   r   ZDropoutrO   rP   r	  
classifierr   r   rY   r(   r)   rI   I  s    
z"XLMForTokenClassification.__init__r   r   Nr   c                 C   s   |dk	r|n| j j}| j|||||||||	|||d}|d }| |}| |}d}|
dk	r~t }||d| j|
d}|s|f|dd  }|dk	r|f| S |S t|||j	|j
dS )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr   r   r`   r   r   )rX   r   r   rO   r  r	   ra   r  r   r   r   )rW   r   r   r   r   r   r?   rv   rw   r   r   rx   r   r   r   r  r   r   r  r  r(   r(   r)   r   T  s>    

z!XLMForTokenClassification.forward)NNNNNNNNNNNNN)r   r   r   rI   r   r   r   r   r   r   r   r   r.   r   r   r   r   r   r   r   r   r(   r(   rY   r)   r  A  sH                
r  z
    XLM Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eeeejf  eej eej eej ee ee ee eee	f dddZ  ZS )
XLMForMultipleChoicec                    sD   t  j|f|| t|| _t|| _t|jd| _	| 
  d S r   )rH   rI   r   r   r   r  r   rP   r  logits_projr   )rW   rX   r   r   rY   r(   r)   rI     s
    

zXLMForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthr   Nr   c                 C   s  |dk	r|n| j j}|dk	r&|jd n|	jd }|dk	rJ|d|dnd}|dk	rh|d|dnd}|dk	r|d|dnd}|dk	r|d|dnd}|dk	r|d|dnd}|	dk	r|	d|	d|	dnd}	|dk	rtd d}| j|||||||||	|||d}|d }| |}| 	|}|d|}d}|
dk	rlt
 }|||
}|s|f|dd  }|dk	r|f| S |S t|||j|jdS )	aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r`   zrThe `lengths` parameter cannot be used with the XLM multiple choice models. Please use the attention mask instead.)r   r   r   r   r   r?   rv   rw   r   rx   r   r   r   r   )rX   r   rf   ra   r<   loggerwarningr   r  r  r	   r   r   r   )rW   r   r   r   r   r   r?   rv   rw   r   r   rx   r   r   Znum_choicesr  r  r   Zreshaped_logitsr   r  r(   r(   r)   r     s^    




zXLMForMultipleChoice.forward)NNNNNNNNNNNNN)r   r   r   rI   r   r   r   r   r   r   r   r   r.   r   r   r   r   r   r   r   r   r(   r(   rY   r)   r    sH   
             
r  )N)Cr   r   rm   dataclassesr   typingr   r   r   r   numpyr"   r.   r   Ztorch.nnr   r	   r
   Zactivationsr   Zmodeling_outputsr   r   r   r   r   r   Zmodeling_utilsr   r   r   Zpytorch_utilsr   r   r   utilsr   r   r   r   r   r   Zconfiguration_xlmr    Z
get_loggerr   r  r   r   Z!XLM_PRETRAINED_MODEL_ARCHIVE_LISTr3   rF   ModulerG   r   r   r   ZXLM_START_DOCSTRINGr   r   r   r   r  r  r  r  r  r(   r(   r(   r)   <module>   s     

`(*E g*Z_arM