U
    9%e{-                    @   s:  d Z ddlZddlZddlZddlmZmZmZmZm	Z	 ddl
Z
ddlZ
ddl
mZ ddlmZmZmZ ddlmZ ddlmZmZmZmZmZmZmZmZmZ dd	lmZ dd
lm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ e',e-Z.e& r<zddl/mZ0 ddl1m2Z2 W n  e3k
r:   e.4d Y nX dZ5dZ6dgZ7dd Z8G dd dej9Z:G dd dej9Z;G dd dej9Z<G dd dej9Z=G dd dej9Z>G dd dej9Z?G d d! d!ej9Z@G d"d# d#ej9ZAG d$d% d%ej9ZBG d&d' d'ej9ZCG d(d) d)ej9ZDG d*d+ d+ej9ZEG d,d- d-ej9ZFG d.d/ d/ej9ZGG d0d1 d1eZHd2ZId3ZJe$d4eIG d5d6 d6eHZKe$d7eIG d8d9 d9eHZLe$d:eIG d;d< d<eHZMe$d=eIG d>d? d?eHZNe$d@eIG dAdB dBeHZOe$dCeIG dDdE dEeHZPe$dFeIG dGdH dHeHZQe$dIeIG dJdK dKeHZRdS )Lz PyTorch QDQBERT model.    N)DictListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)	)BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputNextSentencePredictorOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forward!is_pytorch_quantization_availableloggingreplace_return_docstringsrequires_backends   )QDQBertConfig)TensorQuantizerzQDQBERT model are not usable since `pytorch_quantization` can't be loaded. Please try to reinstall it following the instructions here: https://github.com/NVIDIA/TensorRT/tree/master/tools/pytorch-quantization.zbert-base-uncasedr!   c                 C   s  zddl }ddl}ddl}W n  tk
r<   td  Y nX tj|}t	d|  |j
|}g }g }|D ]@\}	}
t	d|	 d|
  |j
||	}||	 || qrt||D ]\}	}|	d}	tdd	 |	D rt	d
d|	  q| }|	D ]}|d|r&|d|}n|g}|d dksH|d dkrTt|d}n|d dksp|d dkr|t|d}nz|d dkrt|d}n`|d dkrt|d}nFzt||d }W n2 tk
r   t	d
d|	  Y qY nX t|dkrt|d }|| }q|dd dkr:t|d}n|dkrN||}z,|j|jkrxtd|j d|j dW n< tk
r } z| j|j|jf7  _ W 5 d}~X Y nX t	d|	  t||_q| S )z'Load tf checkpoints in a pytorch model.r   NzLoading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape /c                 s   s   | ]}|d kV  qdS ))Zadam_vZadam_mZAdamWeightDecayOptimizerZAdamWeightDecayOptimizer_1Zglobal_stepN ).0nr$   r$   k/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/models/qdqbert/modeling_qdqbert.py	<genexpr>k   s   z-load_tf_weights_in_qdqbert.<locals>.<genexpr>z	Skipping z[A-Za-z]+_\d+z_(\d+)ZkernelgammaweightZoutput_biasbetabiasZoutput_weightsZsquad
classifier   r    iZ_embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )renumpyZ
tensorflowImportErrorloggererrorospathabspathinfotrainZlist_variablesZload_variableappendzipsplitanyjoin	fullmatchgetattrAttributeErrorlenint	transposeshape
ValueErrorAssertionErrorargstorchZ
from_numpydata)modelZtf_checkpoint_pathr/   nptfZtf_pathZ	init_varsnamesZarraysnamerD   arrayZpointerZm_nameZscope_namesnumer$   r$   r'   load_tf_weights_in_qdqbertN   sx    




rR   c                       sT   e Zd ZdZ fddZd	eej eej eej eej e	ej
dddZ  ZS )
QDQBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    s   t    tj|j|j|jd| _t|j|j| _	t|j
|j| _tj|j|jd| _t|j| _t|dd| _| jdt|jddd | jd	tj| j tjd
dd d S )N)padding_idxZepsposition_embedding_typeabsoluteposition_ids)r    F)
persistenttoken_type_idsdtype)super__init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsZtype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutr?   rV   Zregister_bufferrH   arangeexpandzerosrX   sizelongselfconfig	__class__r$   r'   r_      s"    
    zQDQBertEmbeddings.__init__Nr   )	input_idsr[   rX   inputs_embedspast_key_values_lengthreturnc                 C   s   |d k	r|  }n|  d d }|d }|d krL| jd d ||| f }|d krt| dr| jd d d |f }||d |}	|	}ntj|tj| jjd}|d kr| 	|}| 
|}
||
 }| jdkr| |}||7 }| |}| |}|S )NrY   r    r[   r   r]   devicerW   )rp   rX   hasattrr[   rn   rH   ro   rq   r|   rd   rg   rV   rf   rh   rl   )rs   rw   r[   rX   rx   ry   input_shape
seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedrg   
embeddingsrf   r$   r$   r'   forward   s,    







zQDQBertEmbeddings.forward)NNNNr   )__name__
__module____qualname____doc__r_   r   rH   
LongTensorFloatTensorrB   Tensorr   __classcell__r$   r$   ru   r'   rS      s        rS   c                       s.   e Zd Z fddZdd Zd	ddZ  ZS )
QDQBertSelfAttentionc                    s2  t    |j|j dkr>t|ds>td|j d|j d|j| _t|j|j | _| j| j | _t	
|j| j| _t	
|j| j| _t	
|j| j| _t|j| _t|dd| _| jdks| jd	kr|j| _td
|j d | j| _|j| _tt	j
j| _tt	j
j| _tt	j
j| _tt	j
j| _d S )Nr   Zembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()rV   rW   relative_keyrelative_key_queryr.   r    )r^   r_   rb   num_attention_headsr}   rE   rB   attention_head_sizeall_head_sizequant_nnQuantLinearquerykeyvaluer   rj   Zattention_probs_dropout_probrl   r?   rV   re   r`   distance_embedding
is_decoderr"   default_quant_desc_inputmatmul_q_input_quantizermatmul_k_input_quantizermatmul_v_input_quantizermatmul_a_input_quantizerrr   ru   r$   r'   r_      s*    
zQDQBertSelfAttention.__init__c                 C   s6   |  d d | j| jf }|j| }|ddddS )NrY   r   r.   r    r   )rp   r   r   viewpermute)rs   xZnew_x_shaper$   r$   r'   transpose_for_scores   s    
z)QDQBertSelfAttention.transpose_for_scoresNFc              	   C   s  |  |}|d k	}	|	r4|d k	r4|d }
|d }|}n|	r^| | |}
| | |}|}nv|d k	r| | |}
| | |}tj|d |
gdd}
tj|d |gdd}n | | |}
| | |}| |}| jr|
|f}t| || 	|

dd}| jdks$| jdkr| d }tj|tj|jd	dd}tj|tj|jd	dd}|| }| || j d }|j|jd
}| jdkrtd||}|| }n4| jdkrtd||}td|
|}|| | }|t| j }|d k	r|| }tjdd|}| |}|d k	r8|| }t| || |}|dddd }| d d | j f }|j| }|r||fn|f}| jr||f }|S )Nr   r    r.   dimrY   r   r   r{   r\   zbhld,lrd->bhlrzbhrd,lrd->bhlrr   )!r   r   r   r   rH   catr   matmulr   r   rC   rV   rp   rm   rq   r|   r   r   re   tor]   Zeinsummathsqrtr   r   ZSoftmaxrl   r   r   r   
contiguousr   )rs   hidden_statesattention_mask	head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsZmixed_query_layerZis_cross_attentionZ	key_layerZvalue_layerZquery_layerZattention_scoresr   Zposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZcontext_layerZnew_context_layer_shapeoutputsr$   r$   r'   r      sp    


 



 

zQDQBertSelfAttention.forward)NNNNNF)r   r   r   r_   r   r   r   r$   r$   ru   r'   r      s         r   c                       s$   e Zd Z fddZdd Z  ZS )QDQBertSelfOutputc                    s^   t    t|j|j| _tj|j|jd| _t	|j
| _ttjj| _ttjj| _d S NrU   )r^   r_   r   r   rb   denser   rh   ri   rj   rk   rl   r"   r   add_local_input_quantizeradd_residual_input_quantizerrr   ru   r$   r'   r_   _  s    
zQDQBertSelfOutput.__init__c                 C   s:   |  |}| |}| |}| |}| || }|S Nr   rl   r   r   rh   rs   r   Zinput_tensorZ	add_localZadd_residualr$   r$   r'   r   k  s    



zQDQBertSelfOutput.forwardr   r   r   r_   r   r   r$   r$   ru   r'   r   ^  s   r   c                       s.   e Zd Z fddZdd Zd	ddZ  ZS )
QDQBertAttentionc                    s*   t    t|| _t|| _t | _d S r   )r^   r_   r   rs   r   outputsetpruned_headsrr   ru   r$   r'   r_   w  s    


zQDQBertAttention.__init__c                 C   s   t |dkrd S t|| jj| jj| j\}}t| jj|| j_t| jj|| j_t| jj	|| j_	t| j
j|dd| j
_| jjt | | j_| jj| jj | j_| j|| _d S )Nr   r    r   )rA   r   rs   r   r   r   r   r   r   r   r   r   r   union)rs   headsindexr$   r$   r'   prune_heads}  s       zQDQBertAttention.prune_headsNFc              	   C   s<   |  |||||||}| |d |}	|	f|dd   }
|
S )Nr   r    )rs   r   )rs   r   r   r   r   r   r   r   Zself_outputsattention_outputr   r$   r$   r'   r     s    
	zQDQBertAttention.forward)NNNNNF)r   r   r   r_   r   r   r   r$   r$   ru   r'   r   v  s         r   c                       s$   e Zd Z fddZdd Z  ZS )QDQBertIntermediatec                    sB   t    t|j|j| _t|jt	r6t
|j | _n|j| _d S r   )r^   r_   r   r   rb   intermediate_sizer   
isinstance
hidden_actstrr   intermediate_act_fnrr   ru   r$   r'   r_     s
    
zQDQBertIntermediate.__init__c                 C   s   |  |}| |}|S r   )r   r   rs   r   r$   r$   r'   r     s    

zQDQBertIntermediate.forwardr   r$   r$   ru   r'   r     s   	r   c                       s$   e Zd Z fddZdd Z  ZS )QDQBertOutputc                    s^   t    t|j|j| _tj|j|j	d| _t
|j| _ttjj| _ttjj| _d S r   )r^   r_   r   r   r   rb   r   r   rh   ri   rj   rk   rl   r"   r   r   r   rr   ru   r$   r'   r_     s    
zQDQBertOutput.__init__c                 C   s:   |  |}| |}| |}| |}| || }|S r   r   r   r$   r$   r'   r     s    



zQDQBertOutput.forwardr   r$   r$   ru   r'   r     s   r   c                       s.   e Zd Z fddZd	ddZdd Z  ZS )
QDQBertLayerc                    sf   t    d| _t|| _|j| _|j| _| jrN| jsDt|  dt|| _t	|| _
t|| _d S )Nr    z> should be used as a decoder model if cross attention is added)r^   r_   Zseq_len_dimr   	attentionr   add_cross_attentionrE   crossattentionr   intermediater   r   rr   ru   r$   r'   r_     s    



zQDQBertLayer.__init__NFc              	   C   s  |d k	r|d d nd }| j |||||d}	|	d }
| jrP|	dd }|	d }n|	dd  }d }| jr|d k	rt| dstd|  d|d k	r|d	d  nd }| |
||||||}|d }
||dd  }|d }|| }| |
}|f| }| jr||f }|S )
Nr.   r   r   r   r    rY   r   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r   )r   r   r}   rE   r   feed_forward_chunk)rs   r   r   r   r   r   r   r   Zself_attn_past_key_valueZself_attention_outputsr   r   Zpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputslayer_outputr$   r$   r'   r     sL    


	


zQDQBertLayer.forwardc                 C   s   |  |}| ||}|S r   )r   r   )rs   r   Zintermediate_outputr   r$   r$   r'   r     s    
zQDQBertLayer.feed_forward_chunk)NNNNNF)r   r   r   r_   r   r   r   r$   r$   ru   r'   r     s         
?r   c                	       s&   e Zd Z fddZdddZ  ZS )	QDQBertEncoderc                    s:   t     | _t fddt jD | _d| _d S )Nc                    s   g | ]}t  qS r$   )r   )r%   _rt   r$   r'   
<listcomp>&  s     z+QDQBertEncoder.__init__.<locals>.<listcomp>F)	r^   r_   rt   r   Z
ModuleListrangenum_hidden_layerslayergradient_checkpointingrr   ru   r   r'   r_   #  s    
 zQDQBertEncoder.__init__NFTc              	      sf  |	rdnd } rdnd } r(| j jr(dnd }|r4dnd }t| jD ]\}}|	rX||f }|d k	rh|| nd }|d k	r||| nd | jr| jr|rtd d} fdd}tj	j

|||||||}n|||||| }|d }|r||d f7 } rB||d f }| j jrB||d	 f }qB|	r2||f }|
sTtd
d |||||fD S t|||||dS )Nr$   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fc                    s    fdd}|S )Nc                     s    | f S r   r$   )inputs)moduler   r   r$   r'   custom_forwardJ  s    zMQDQBertEncoder.forward.<locals>.create_custom_forward.<locals>.custom_forwardr$   )r   r   r   )r   r'   create_custom_forwardI  s    z5QDQBertEncoder.forward.<locals>.create_custom_forwardr   rY   r    r.   c                 s   s   | ]}|d k	r|V  qd S r   r$   )r%   vr$   r$   r'   r(   n  s   z)QDQBertEncoder.forward.<locals>.<genexpr>)last_hidden_statepast_key_valuesr   
attentionscross_attentions)rt   r   	enumerater   r   Ztrainingr2   Zwarning_oncerH   utils
checkpointtupler   )rs   r   r   r   r   r   r   	use_cacher   output_hidden_statesreturn_dictZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheiZlayer_moduleZlayer_head_maskr   Zlayer_outputsr$   r   r'   r   )  st    
	

zQDQBertEncoder.forward)	NNNNNNFFTr   r$   r$   ru   r'   r   "  s   	         r   c                       s0   e Zd Z fddZejejdddZ  ZS )QDQBertPoolerc                    s*   t    t|j|j| _t | _d S r   )r^   r_   r   Linearrb   r   ZTanh
activationrr   ru   r$   r'   r_     s    
zQDQBertPooler.__init__r   rz   c                 C   s(   |d d df }|  |}| |}|S )Nr   )r   r   )rs   r   Zfirst_token_tensorpooled_outputr$   r$   r'   r     s    

zQDQBertPooler.forwardr   r   r   r_   rH   r   r   r   r$   r$   ru   r'   r     s   r   c                       s0   e Zd Z fddZejejdddZ  ZS )QDQBertPredictionHeadTransformc                    sV   t    t|j|j| _t|jtr6t	|j | _
n|j| _
tj|j|jd| _d S r   )r^   r_   r   r   rb   r   r   r   r   r   transform_act_fnrh   ri   rr   ru   r$   r'   r_     s    
z'QDQBertPredictionHeadTransform.__init__r   c                 C   s"   |  |}| |}| |}|S r   )r   r   rh   r   r$   r$   r'   r     s    


z&QDQBertPredictionHeadTransform.forwardr   r$   r$   ru   r'   r     s   	r   c                       s$   e Zd Z fddZdd Z  ZS )QDQBertLMPredictionHeadc                    sL   t    t|| _tj|j|jdd| _t	t
|j| _| j| j_d S )NF)r,   )r^   r_   r   	transformr   r   rb   ra   decoder	ParameterrH   ro   r,   rr   ru   r$   r'   r_     s
    

z QDQBertLMPredictionHead.__init__c                 C   s   |  |}| |}|S r   )r   r   r   r$   r$   r'   r     s    

zQDQBertLMPredictionHead.forwardr   r$   r$   ru   r'   r     s   r   c                       s$   e Zd Z fddZdd Z  ZS )QDQBertOnlyMLMHeadc                    s   t    t|| _d S r   )r^   r_   r   predictionsrr   ru   r$   r'   r_     s    
zQDQBertOnlyMLMHead.__init__c                 C   s   |  |}|S r   )r   )rs   sequence_outputprediction_scoresr$   r$   r'   r     s    
zQDQBertOnlyMLMHead.forwardr   r$   r$   ru   r'   r     s   r   c                       s$   e Zd Z fddZdd Z  ZS )QDQBertOnlyNSPHeadc                    s   t    t|jd| _d S Nr.   )r^   r_   r   r   rb   seq_relationshiprr   ru   r$   r'   r_     s    
zQDQBertOnlyNSPHead.__init__c                 C   s   |  |}|S r   )r  )rs   r   seq_relationship_scorer$   r$   r'   r     s    
zQDQBertOnlyNSPHead.forwardr   r$   r$   ru   r'   r     s   r   c                       s$   e Zd Z fddZdd Z  ZS )QDQBertPreTrainingHeadsc                    s(   t    t|| _t|jd| _d S r   )r^   r_   r   r   r   r   rb   r  rr   ru   r$   r'   r_     s    

z QDQBertPreTrainingHeads.__init__c                 C   s   |  |}| |}||fS r   )r   r  )rs   r   r   r   r  r$   r$   r'   r     s    

zQDQBertPreTrainingHeads.forwardr   r$   r$   ru   r'   r    s   r  c                   @   s2   e Zd ZdZeZeZdZdZ	dd Z
d
ddZd	S )QDQBertPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    bertTc                 C   s   t |tjr:|jjjd| jjd |jdk	r|jj	  nft |tj
rz|jjjd| jjd |jdk	r|jj|j 	  n&t |tjr|jj	  |jjd dS )zInitialize the weightsg        )ZmeanZstdNg      ?)r   r   r   r*   rI   Znormal_rt   Zinitializer_ranger,   Zzero_r`   rT   rh   Zfill_)rs   r   r$   r$   r'   _init_weights  s    

z$QDQBertPreTrainedModel._init_weightsFc                 C   s   t |tr||_d S r   )r   r   r   )rs   r   r   r$   r$   r'   _set_gradient_checkpointing  s    
z2QDQBertPreTrainedModel._set_gradient_checkpointingN)F)r   r   r   r   r!   config_classrR   Zload_tf_weightsZbase_model_prefixZsupports_gradient_checkpointingr  r  r$   r$   r$   r'   r    s   r  aA  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`QDQBertConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a5
  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zaThe bare QDQBERT Model transformer outputting raw hidden-states without any specific head on top.c                       s   e Zd ZdZded fddZdd Zdd	 Zee	e
e	 f d
ddZeedeeeeddeej eej eej eej eej eej eej eej eeeej   ee ee ee ee eeef dddZ  ZS )QDQBertModela  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    Tadd_pooling_layerc                    sN   t | d t | || _t|| _t|| _|r<t|nd | _	| 
  d S )Npytorch_quantization)r   r^   r_   rt   rS   r   r   encoderr   pooler	post_init)rs   rt   r  ru   r$   r'   r_   P  s    


zQDQBertModel.__init__c                 C   s   | j jS r   r   rd   rs   r$   r$   r'   get_input_embeddings]  s    z!QDQBertModel.get_input_embeddingsc                 C   s   || j _d S r   r  )rs   r   r$   r$   r'   set_input_embeddings`  s    z!QDQBertModel.set_input_embeddings)heads_to_prunec                 C   s*   |  D ]\}}| jj| j| qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr  r   r   r   )rs   r  r   r   r$   r$   r'   _prune_headsc  s    zQDQBertModel._prune_headsbatch_size, sequence_lengthr   output_typer  N)rw   r   r[   rX   r   rx   r   r   r   r   r   r   r   rz   c                 C   sf  |dk	r|n| j j}|dk	r |n| j j}|dk	r4|n| j j}| j jrZ|
dk	rP|
n| j j}
nd}
|dk	rx|dk	rxtdnP|dk	r| || | }|\}}n*|dk	r| dd }|\}}ntd|dk	r|j	n|j	}|	dk	r|	d d j
d nd}|dkrtj||| f|d}|dkrnt| jd	r\| jjddd|f }|||}|}ntj|tj|d
}| ||}| j jr|dk	r| \}}}||f}|dkrtj||d}| |}nd}| || j j}| j|||||d}| j||||||	|
|||d
}|d }| jdk	r,| |nd}|sJ||f|dd  S t|||j|j|j|jdS )a  
        encoder_hidden_states  (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        NFzDYou cannot specify both input_ids and inputs_embeds at the same timerY   z5You have to specify either input_ids or inputs_embedsr   r.   )r|   r[   r{   )rw   rX   r[   rx   ry   )	r   r   r   r   r   r   r   r   r   r    )r   Zpooler_outputr   r   r   r   )rt   r   r   use_return_dictr   r   rE   Z%warn_if_padding_and_no_attention_maskrp   r|   rD   rH   Zonesr}   r   r[   rn   ro   rq   Zget_extended_attention_maskZinvert_attention_maskZget_head_maskr   r  r  r   r   r   r   r   )rs   rw   r   r[   rX   r   rx   r   r   r   r   r   r   r   r~   Z
batch_sizer   r|   ry   r   r   Zextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthr   Zencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputZencoder_outputsr   r   r$   r$   r'   r   k  s    *





zQDQBertModel.forward)T)NNNNNNNNNNNNN)r   r   r   r   boolr_   r  r  r   rB   r   r  r   QDQBERT_INPUTS_DOCSTRINGformatr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOCr   rH   r   r   r   r   r   r   r$   r$   ru   r'   r	  ?  sP                
r	  zIQDQBERT Model with a `language modeling` head on top for CLM fine-tuning.c                       s   e Zd ZddgZ fddZdd Zdd Zee	d	e
eed
deej eej eej eej eej eej eej eej eej eeeej   ee ee ee ee eeef dddZdeej eej dddZdd Z  ZS )QDQBertLMHeadModelpredictions.decoder.weightpredictions.decoder.biasc                    s@   t  | |jstd t|dd| _t|| _| 	  d S )NzOIf you want to use `QDQBertLMHeadModel` as a standalone, add `is_decoder=True.`Fr
  
r^   r_   r   r2   warningr	  r  r   clsr  rr   ru   r$   r'   r_     s    

zQDQBertLMHeadModel.__init__c                 C   s
   | j jjS r   r%  r   r   r  r$   r$   r'   get_output_embeddings  s    z(QDQBertLMHeadModel.get_output_embeddingsc                 C   s   || j j_d S r   r&  rs   Znew_embeddingsr$   r$   r'   set_output_embeddings
  s    z(QDQBertLMHeadModel.set_output_embeddingsr  r  r  N)rw   r   r[   rX   r   rx   r   r   labelsr   r   r   r   r   rz   c                 C   s  |dk	r|n| j j}|	dk	r d}| j|||||||||
||||d}|d }| |}d}|	dk	r|ddddddf  }|	ddddf  }	t }||d| j j|	d}|s|f|dd  }|dk	r|f| S |S t|||j	|j
|j|jdS )	a
  
        encoder_hidden_states  (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`
        past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, QDQBertLMHeadModel, QDQBertConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
        >>> config = QDQBertConfig.from_pretrained("bert-base-cased")
        >>> config.is_decoder = True
        >>> model = QDQBertLMHeadModel.from_pretrained("bert-base-cased", config=config)

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```NF)r   r[   rX   r   rx   r   r   r   r   r   r   r   r   rY   r    r.   )losslogitsr   r   r   r   )rt   r  r  r%  r   r	   r   ra   r   r   r   r   r   )rs   rw   r   r[   rX   r   rx   r   r   r+  r   r   r   r   r   r   r   r   Zlm_lossZshifted_prediction_scoresloss_fctr   r$   r$   r'   r     sJ    =
zQDQBertLMHeadModel.forwardrw   r   c                 K   s@   |j }|d kr||}|d k	r4|d d dd f }|||dS )NrY   )rw   r   r   )rD   Znew_ones)rs   rw   r   r   model_kwargsr~   r$   r$   r'   prepare_inputs_for_generationv  s    
z0QDQBertLMHeadModel.prepare_inputs_for_generationc                    s.   d}|D ] }|t  fdd|D f7 }q|S )Nr$   c                 3   s"   | ]}| d  |jV  qdS )r   N)Zindex_selectr   r|   )r%   Z
past_statebeam_idxr$   r'   r(     s     z4QDQBertLMHeadModel._reorder_cache.<locals>.<genexpr>)r   )rs   r   r3  Zreordered_pastZ
layer_pastr$   r2  r'   _reorder_cache  s    z!QDQBertLMHeadModel._reorder_cache)NNNNNNNNNNNNNN)NN)r   r   r   _tied_weights_keysr_   r'  r)  r   r  r  r   r   r  r   rH   r   r   r   r   r  r   r   r1  r4  r   r$   r$   ru   r'   r     sX   
              
j  r   z5QDQBERT Model with a `language modeling` head on top.c                       s   e Zd ZddgZ fddZdd Zdd Zee	d	e
eeed
deej eej eej eej eej eej eej eej eej ee ee ee eeef dddZdejeej dddZ  ZS )QDQBertForMaskedLMr!  r"  c                    s@   t  | |jrtd t|dd| _t|| _| 	  d S )NznIf you want to use `QDQBertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fr
  r#  rr   ru   r$   r'   r_     s    
zQDQBertForMaskedLM.__init__c                 C   s
   | j jjS r   r&  r  r$   r$   r'   r'    s    z(QDQBertForMaskedLM.get_output_embeddingsc                 C   s   || j j_d S r   r&  r(  r$   r$   r'   r)    s    z(QDQBertForMaskedLM.set_output_embeddingsr  r  N)rw   r   r[   rX   r   rx   r   r   r+  r   r   r   rz   c                 C   s   |dk	r|n| j j}| j|||||||||
||d}|d }| |}d}|	dk	rtt }||d| j j|	d}|s|f|dd  }|dk	r|f| S |S t|||j|j	dS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        N)
r   r[   rX   r   rx   r   r   r   r   r   r   rY   r.   r,  r-  r   r   )
rt   r  r  r%  r	   r   ra   r   r   r   )rs   rw   r   r[   rX   r   rx   r   r   r+  r   r   r   r   r   r   Zmasked_lm_lossr.  r   r$   r$   r'   r     s:    
zQDQBertForMaskedLM.forwardr/  c                 K   s~   |j }|d }| jjd kr"tdtj|||j d dfgdd}tj|df| jjtj|j	d}tj||gdd}||dS )Nr   z.The PAD token should be defined for generationr    rY   r   r{   r/  )
rD   rt   rc   rE   rH   r   Z	new_zerosfullrq   r|   )rs   rw   r   r0  r~   Zeffective_batch_sizeZdummy_tokenr$   r$   r'   r1    s    "   z0QDQBertForMaskedLM.prepare_inputs_for_generation)NNNNNNNNNNNN)N)r   r   r   r5  r_   r'  r)  r   r  r  r   r  r   r  r   rH   r   r   r  r   r   r   r1  r   r$   r$   ru   r'   r6    sT               
:  r6  zJBert Model with a `next sentence prediction (classification)` head on top.c                       s   e Zd Z fddZeedeee	dd	e
ej e
ej e
ej e
ej e
ej e
ej e
ej e
e e
e e
e eeef dddZ  ZS )
 QDQBertForNextSentencePredictionc                    s,   t  | t|| _t|| _|   d S r   )r^   r_   r	  r  r   r%  r  rr   ru   r$   r'   r_     s    

z)QDQBertForNextSentencePrediction.__init__r  r*  Nrw   r   r[   rX   r   rx   r+  r   r   r   rz   c                 K   s   d|krt dt |d}|
dk	r*|
n| jj}
| j||||||||	|
d	}|d }| |}d}|dk	rt }||	dd|	d}|
s|f|dd  }|dk	r|f| S |S t
|||j|jdS )	a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
            (see `input_ids` docstring). Indices should be in `[0, 1]`:

            - 0 indicates sequence B is a continuation of sequence A,
            - 1 indicates sequence B is a random sequence.

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, QDQBertForNextSentencePrediction
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
        >>> model = QDQBertForNextSentencePrediction.from_pretrained("bert-base-uncased")

        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
        >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt")

        >>> outputs = model(**encoding, labels=torch.LongTensor([1]))
        >>> logits = outputs.logits
        >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
        ```Znext_sentence_labelzoThe `next_sentence_label` argument is deprecated and will be removed in a future version, use `labels` instead.Nr   r[   rX   r   rx   r   r   r   r    rY   r.   r7  )warningswarnFutureWarningpoprt   r  r  r%  r	   r   r   r   r   )rs   rw   r   r[   rX   r   rx   r+  r   r   r   kwargsr   r   Zseq_relationship_scoresZnext_sentence_lossr.  r   r$   r$   r'   r   
  sB    ,

z(QDQBertForNextSentencePrediction.forward)
NNNNNNNNNN)r   r   r   r_   r   r  r  r   r   r  r   rH   r   r   r  r   r   r   r   r$   r$   ru   r'   r9    s4   	
          
r9  z
    Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej ee ee ee eee	f dddZ  ZS )
 QDQBertForSequenceClassificationc                    sP   t  | |j| _|| _t|| _t|j| _	t
|j|j| _|   d S r   )r^   r_   
num_labelsrt   r	  r  r   rj   rk   rl   r   rb   r-   r  rr   ru   r$   r'   r_   i  s    
z)QDQBertForSequenceClassification.__init__r  r  Nr:  c                 C   s|  |
dk	r|
n| j j}
| j||||||||	|
d	}|d }| |}| |}d}|dk	r8| j jdkr| jdkrzd| j _n4| jdkr|jtj	ks|jtj
krd| j _nd| j _| j jdkrt }| jdkr|| | }n
|||}nN| j jdkrt }||d| j|d}n| j jdkr8t }|||}|
sh|f|dd  }|dk	rd|f| S |S t|||j|jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr;  r    Z
regressionZsingle_label_classificationZmulti_label_classificationrY   r.   r7  )rt   r  r  rl   r-   Zproblem_typerB  r]   rH   rq   rB   r
   squeezer	   r   r   r   r   r   )rs   rw   r   r[   rX   r   rx   r+  r   r   r   r   r   r-  r,  r.  r   r$   r$   r'   r   t  sV    




"


z(QDQBertForSequenceClassification.forward)
NNNNNNNNNN)r   r   r   r_   r   r  r  r   r  r   r  r   rH   r   r   r  r   r   r   r   r$   r$   ru   r'   rA  a  s<             
rA  z
    Bert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej ee ee ee eee	f dddZ  ZS )
QDQBertForMultipleChoicec                    s@   t  | t|| _t|j| _t|j	d| _
|   d S )Nr    )r^   r_   r	  r  r   rj   rk   rl   r   rb   r-   r  rr   ru   r$   r'   r_     s
    
z!QDQBertForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthr  Nr:  c                 C   st  |
dk	r|
n| j j}
|dk	r&|jd n|jd }|dk	rJ|d|dnd}|dk	rh|d|dnd}|dk	r|d|dnd}|dk	r|d|dnd}|dk	r|d|d|dnd}| j||||||||	|
d	}|d }| |}| |}|d|}d}|dk	r0t }|||}|
s`|f|dd  }|dk	r\|f| S |S t	|||j
|jdS )aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr    rY   r   r;  r.   r7  )rt   r  rD   r   rp   r  rl   r-   r	   r   r   r   )rs   rw   r   r[   rX   r   rx   r+  r   r   r   Znum_choicesr   r   r-  Zreshaped_logitsr,  r.  r   r$   r$   r'   r     sL    



z QDQBertForMultipleChoice.forward)
NNNNNNNNNN)r   r   r   r_   r   r  r  r   r  r   r  r   rH   r   r   r  r   r   r   r   r$   r$   ru   r'   rD    s<   
          
rD  z
    QDQBERT Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej ee ee ee eee	f dddZ  ZS )
QDQBertForTokenClassificationc                    sN   t  | |j| _t|dd| _t|j| _t	|j
|j| _|   d S NFr
  )r^   r_   rB  r	  r  r   rj   rk   rl   r   rb   r-   r  rr   ru   r$   r'   r_   %  s    z&QDQBertForTokenClassification.__init__r  r  Nr:  c                 C   s   |
dk	r|
n| j j}
| j||||||||	|
d	}|d }| |}| |}d}|dk	rxt }||d| j|d}|
s|f|dd  }|dk	r|f| S |S t|||j	|j
dS )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr;  r   rY   r.   r7  )rt   r  r  rl   r-   r	   r   rB  r   r   r   )rs   rw   r   r[   rX   r   rx   r+  r   r   r   r   r   r-  r,  r.  r   r$   r$   r'   r   0  s8    

z%QDQBertForTokenClassification.forward)
NNNNNNNNNN)r   r   r   r_   r   r  r  r   r  r   r  r   rH   r   r   r  r   r   r   r   r$   r$   ru   r'   rE    s<             
rE  z
    QDQBERT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                       s   e Zd Z fddZeedeee	e
dd	eej eej eej eej eej eej eej eej ee ee ee eee	f dddZ  ZS )
QDQBertForQuestionAnsweringc                    s@   t  | |j| _t|dd| _t|j|j| _| 	  d S rF  )
r^   r_   rB  r	  r  r   r   rb   
qa_outputsr  rr   ru   r$   r'   r_   s  s
    z$QDQBertForQuestionAnswering.__init__r  r  N)rw   r   r[   rX   r   rx   start_positionsend_positionsr   r   r   rz   c                 C   sP  |dk	r|n| j j}| j|||||||	|
|d	}|d }| |}|jddd\}}|d }|d }d}|dk	r|dk	rt| dkr|d}t| dkr|d}|d}|	d|}|	d|}t
|d}|||}|||}|| d }|s:||f|dd  }|dk	r6|f| S |S t||||j|jd	S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        Nr;  r   r    rY   r   )Zignore_indexr.   )r,  start_logits
end_logitsr   r   )rt   r  r  rH  r;   rC  r   rA   rp   clampr	   r   r   r   )rs   rw   r   r[   rX   r   rx   rI  rJ  r   r   r   r   r   r-  rK  rL  Z
total_lossZignored_indexr.  Z
start_lossZend_lossr   r$   r$   r'   r   }  sP    






z#QDQBertForQuestionAnswering.forward)NNNNNNNNNNN)r   r   r   r_   r   r  r  r   r  r   r  r   rH   r   r   r  r   r   r   r   r$   r$   ru   r'   rG  k  s@   
           
rG  )Sr   r   r4   r<  typingr   r   r   r   r   rH   Ztorch.utils.checkpointr   Ztorch.nnr   r	   r
   Zactivationsr   Zmodeling_outputsr   r   r   r   r   r   r   r   r   Zmodeling_utilsr   Zpytorch_utilsr   r   r   r   r   r   r   r   r   r   Zconfiguration_qdqbertr!   Z
get_loggerr   r2   r  r   Z0pytorch_quantization.nn.modules.tensor_quantizerr"   OSErrorr3   r  r  Z%QDQBERT_PRETRAINED_MODEL_ARCHIVE_LISTrR   ModulerS   r   r   r   r   r   r   r   r   r   r   r   r   r  r  ZQDQBERT_START_DOCSTRINGr  r	  r   r6  r9  rA  rD  rE  rG  r$   r$   r$   r'   <module>   s   ,$	

J@ 1Ta 2 3  jaZTG