U
    ,-e(                     @   sV  d Z ddlZddlmZmZmZ ddlZddlZddlmZ ddl	m
Z
mZmZ ddlmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZmZmZ ddlmZmZmZm Z  ddl!m"Z" e#e$Z%dZ&dZ'ddgZ(ej)Z*G dd dej+Z,G dd dej+Z-G dd dej+Z.G dd dej+Z/G dd dej+Z0G dd dej+Z1G dd dej+Z2G d d! d!ej+Z3G d"d# d#ej+Z4G d$d% d%ej+Z5G d&d' d'ej+Z6G d(d) d)ej+Z7G d*d+ d+eZ8d,Z9d-Z:ed.e9G d/d0 d0e8Z;ed1e9G d2d3 d3e8Z<ed4e9G d5d6 d6e8Z=ed7e9G d8d9 d9e8Z>ed:e9G d;d< d<e8Z?dS )=z PyTorch LayoutLM model.    N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )LayoutLMConfigr   zmicrosoft/layoutlm-base-uncasedzlayoutlm-base-uncasedzlayoutlm-large-uncasedc                       s*   e Zd ZdZ fddZdddZ  ZS )LayoutLMEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    s   t t|   tj|j|j|jd| _t|j	|j| _
t|j|j| _t|j|j| _t|j|j| _t|j|j| _t|j|j| _t|j|jd| _t|j| _| jdt|j	ddd d S )N)padding_idxZepsposition_ids)r   F)
persistent)superr   __init__r   	Embedding
vocab_sizehidden_sizeZpad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsZmax_2d_position_embeddingsx_position_embeddingsy_position_embeddingsh_position_embeddingsw_position_embeddingsZtype_vocab_sizetoken_type_embeddingsLayoutLMLayerNormlayer_norm_eps	LayerNormDropouthidden_dropout_probdropoutZregister_buffertorcharangeexpandselfconfig	__class__ o/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/layoutlm/modeling_layoutlm.pyr"   :   s      zLayoutLMEmbeddings.__init__Nc              
   C   s  |d k	r|  }n|  d d }|d }|d k	r8|jn|j}|d kr\| jd d d |f }|d krvtj|tj|d}|d kr| |}|}	| |}
zt| |d d d d df }| 	|d d d d df }| |d d d d df }| 	|d d d d df }W n. t
k
r8 } zt
d|W 5 d }~X Y nX | |d d d d df |d d d d df  }| |d d d d df |d d d d df  }| |}|	|
 | | | | | | | }| |}| |}|S )Nr   r   dtypedevicer      r	   z:The `bbox`coordinate values should be within 0-1000 range.)sizer@   r   r4   zeroslongr&   r(   r)   r*   
IndexErrorr+   r,   r-   r0   r3   )r8   	input_idsbboxtoken_type_idsr   inputs_embedsinput_shapeZ
seq_lengthr@   Zwords_embeddingsr(   Zleft_position_embeddingsZupper_position_embeddingsZright_position_embeddingsZlower_position_embeddingser+   r,   r-   
embeddingsr<   r<   r=   forwardK   sX    


 22


zLayoutLMEmbeddings.forward)NNNNN)__name__
__module____qualname____doc__r"   rM   __classcell__r<   r<   r:   r=   r   7   s        r   c                
       s   e Zd Zd fdd	ZejejdddZdejeej eej eej eej ee	e	ej   ee
 e	ej dd	d
Z  ZS )LayoutLMSelfAttentionNc                    s   t    |j|j dkr>t|ds>td|j d|j d|j| _t|j|j | _| j| j | _t	
|j| j| _t	
|j| j| _t	
|j| j| _t	|j| _|pt|dd| _| jdks| jd	kr|j| _t	d
|j d | j| _|j| _d S )Nr   Zembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()position_embedding_typeabsoluterelative_keyrelative_key_queryrA   r   )r!   r"   r%   num_attention_headshasattr
ValueErrorintattention_head_sizeall_head_sizer   Linearquerykeyvaluer1   Zattention_probs_dropout_probr3   getattrrU   r'   r#   distance_embedding
is_decoderr8   r9   rU   r:   r<   r=   r"      s*    
  zLayoutLMSelfAttention.__init__)xreturnc                 C   s6   |  d d | j| jf }||}|ddddS )Nr   r   rA   r   r	   )rB   rY   r]   viewpermute)r8   rg   Znew_x_shaper<   r<   r=   transpose_for_scores   s    
z*LayoutLMSelfAttention.transpose_for_scoresFhidden_statesattention_mask	head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsrh   c                 C   s  |  |}|d k	}	|	r4|d k	r4|d }
|d }|}n|	r^| | |}
| | |}|}nv|d k	r| | |}
| | |}tj|d |
gdd}
tj|d |gdd}n | | |}
| | |}| |}|d k	}| jr|
|f}t||
dd}| j	dks | j	dkr|j
d |
j
d  }}|r^tj|d tj|jd	dd}ntj|tj|jd	dd}tj|tj|jd	dd}|| }| || j d }|j|jd
}| j	dkrtd||}|| }n4| j	dkrtd||}td|
|}|| | }|t| j }|d k	r:|| }tjj|dd}| |}|d k	rf|| }t||}|dddd }| d d | jf }||}|r||fn|f}| jr||f }|S )Nr   r   rA   dimr   rW   rX   r>   r?   zbhld,lrd->bhlrzbhrd,lrd->bhlrr	   ) r`   rk   ra   rb   r4   catre   matmulZ	transposerU   shapeZtensorrD   r@   ri   r5   rd   r'   tor?   Zeinsummathsqrtr]   r   Z
functionalZsoftmaxr3   rj   
contiguousrB   r^   )r8   rm   rn   ro   rp   rq   rr   rs   Zmixed_query_layerZis_cross_attentionZ	key_layerZvalue_layerZquery_layer	use_cacheZattention_scoresZquery_lengthZ
key_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZcontext_layerZnew_context_layer_shapeoutputsr<   r<   r=   rM      sp    


 





zLayoutLMSelfAttention.forward)N)NNNNNF)rN   rO   rP   r"   r4   Tensorrk   r   FloatTensorr   boolrM   rR   r<   r<   r:   r=   rS      s$         rS   c                       s4   e Zd Z fddZejejejdddZ  ZS )LayoutLMSelfOutputc                    sB   t    t|j|j| _tj|j|jd| _t|j	| _
d S Nr   )r!   r"   r   r_   r%   denser0   r/   r1   r2   r3   r7   r:   r<   r=   r"     s    
zLayoutLMSelfOutput.__init__rm   input_tensorrh   c                 C   s&   |  |}| |}| || }|S Nr   r3   r0   r8   rm   r   r<   r<   r=   rM     s    

zLayoutLMSelfOutput.forwardrN   rO   rP   r"   r4   r   rM   rR   r<   r<   r:   r=   r     s   r   c                
       sv   e Zd Zd
 fdd	Zdd Zdejeej eej eej eej ee	e	ej   ee
 e	ej ddd	Z  ZS )LayoutLMAttentionNc                    s.   t    t||d| _t|| _t | _d S )NrU   )r!   r"   rS   r8   r   outputsetpruned_headsrf   r:   r<   r=   r"     s    

zLayoutLMAttention.__init__c                 C   s   t |dkrd S t|| jj| jj| j\}}t| jj|| j_t| jj|| j_t| jj	|| j_	t| j
j|dd| j
_| jjt | | j_| jj| jj | j_| j|| _d S )Nr   r   rt   )lenr   r8   rY   r]   r   r   r`   ra   rb   r   r   r^   union)r8   headsindexr<   r<   r=   prune_heads!  s       zLayoutLMAttention.prune_headsFrl   c              	   C   s<   |  |||||||}| |d |}	|	f|dd   }
|
S )Nr   r   )r8   r   )r8   rm   rn   ro   rp   rq   rr   rs   Zself_outputsattention_outputr   r<   r<   r=   rM   3  s    
	zLayoutLMAttention.forward)N)NNNNNF)rN   rO   rP   r"   r   r4   r   r   r   r   r   rM   rR   r<   r<   r:   r=   r     s$         r   c                       s0   e Zd Z fddZejejdddZ  ZS )LayoutLMIntermediatec                    sB   t    t|j|j| _t|jt	r6t
|j | _n|j| _d S r   )r!   r"   r   r_   r%   intermediate_sizer   
isinstance
hidden_actstrr
   intermediate_act_fnr7   r:   r<   r=   r"   M  s
    
zLayoutLMIntermediate.__init__rm   rh   c                 C   s   |  |}| |}|S r   )r   r   r8   rm   r<   r<   r=   rM   U  s    

zLayoutLMIntermediate.forwardr   r<   r<   r:   r=   r   L  s   r   c                       s4   e Zd Z fddZejejejdddZ  ZS )LayoutLMOutputc                    sB   t    t|j|j| _tj|j|jd| _t	|j
| _d S r   )r!   r"   r   r_   r   r%   r   r0   r/   r1   r2   r3   r7   r:   r<   r=   r"   ]  s    
zLayoutLMOutput.__init__r   c                 C   s&   |  |}| |}| || }|S r   r   r   r<   r<   r=   rM   c  s    

zLayoutLMOutput.forwardr   r<   r<   r:   r=   r   \  s   r   c                
       st   e Zd Z fddZd
ejeej eej eej eej eeeej   ee	 eej dddZ
dd	 Z  ZS )LayoutLMLayerc                    sr   t    |j| _d| _t|| _|j| _|j| _| jrZ| jsLt|  dt|dd| _	t
|| _t|| _d S )Nr   z> should be used as a decoder model if cross attention is addedrV   r   )r!   r"   chunk_size_feed_forwardseq_len_dimr   	attentionre   add_cross_attentionr[   crossattentionr   intermediater   r   r7   r:   r<   r=   r"   l  s    


zLayoutLMLayer.__init__NFrl   c              	   C   s  |d k	r|d d nd }| j |||||d}	|	d }
| jrP|	dd }|	d }n|	dd  }d }| jr|d k	rt| dstd|  d|d k	r|d	d  nd }| |
||||||}|d }
||dd  }|d }|| }t| j| j| j|
}|f| }| jr||f }|S )
NrA   rs   rr   r   r   r   r   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`rv   )	r   re   rZ   r[   r   r   feed_forward_chunkr   r   )r8   rm   rn   ro   rp   rq   rr   rs   Zself_attn_past_key_valueZself_attention_outputsr   r   Zpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputslayer_outputr<   r<   r=   rM   z  sV    


	   

zLayoutLMLayer.forwardc                 C   s   |  |}| ||}|S r   )r   r   )r8   r   Zintermediate_outputr   r<   r<   r=   r     s    
z LayoutLMLayer.feed_forward_chunk)NNNNNF)rN   rO   rP   r"   r4   r   r   r   r   r   rM   r   rR   r<   r<   r:   r=   r   k  s$         Ar   c                       s   e Zd Z fddZd	ejeej eej eej eej eeeej   ee	 ee	 ee	 ee	 e
eej ef dddZ  ZS )
LayoutLMEncoderc                    s:   t     | _t fddt jD | _d| _d S )Nc                    s   g | ]}t  qS r<   )r   ).0_r9   r<   r=   
<listcomp>  s     z,LayoutLMEncoder.__init__.<locals>.<listcomp>F)	r!   r"   r9   r   Z
ModuleListrangenum_hidden_layerslayergradient_checkpointingr7   r:   r   r=   r"     s    
 zLayoutLMEncoder.__init__NFT)rm   rn   ro   rp   rq   past_key_valuesr   rs   output_hidden_statesreturn_dictrh   c              	      st  |	rdnd } rdnd } r(| j jr(dnd }| jrJ| jrJ|rJtd d}|rRdnd }t| jD ]\}}|	rv||f }|d k	r|| nd }|d k	r|| nd | jr| jrև fdd}tj	j

|||||||}n|||||| }|d }|r||d f7 } r`||d f }| j jr`||d	 f }q`|	r@||f }|
sbtd
d |||||fD S t|||||dS )Nr<   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fc                    s    fdd}|S )Nc                     s    | f S r   r<   )inputs)modulers   rr   r<   r=   custom_forward  s    zNLayoutLMEncoder.forward.<locals>.create_custom_forward.<locals>.custom_forwardr<   )r   r   r   )r   r=   create_custom_forward  s    z6LayoutLMEncoder.forward.<locals>.create_custom_forwardr   r   r   rA   c                 s   s   | ]}|d k	r|V  qd S r   r<   )r   vr<   r<   r=   	<genexpr>  s   z*LayoutLMEncoder.forward.<locals>.<genexpr>)last_hidden_stater   rm   
attentionscross_attentions)r9   r   r   ZtrainingloggerZwarning_once	enumerater   r4   utils
checkpointtupler   )r8   rm   rn   ro   rp   rq   r   r   rs   r   r   Zall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheiZlayer_moduleZlayer_head_maskr   Zlayer_outputsr<   r   r=   rM     sv    
	

zLayoutLMEncoder.forward)	NNNNNNFFT)rN   rO   rP   r"   r4   r   r   r   r   r   r   r   rM   rR   r<   r<   r:   r=   r     s.   	         r   c                       s0   e Zd Z fddZejejdddZ  ZS )LayoutLMPoolerc                    s*   t    t|j|j| _t | _d S r   )r!   r"   r   r_   r%   r   ZTanh
activationr7   r:   r<   r=   r"   &  s    
zLayoutLMPooler.__init__r   c                 C   s(   |d d df }|  |}| |}|S )Nr   )r   r   )r8   rm   Zfirst_token_tensorpooled_outputr<   r<   r=   rM   +  s    

zLayoutLMPooler.forwardr   r<   r<   r:   r=   r   %  s   r   c                       s0   e Zd Z fddZejejdddZ  ZS )LayoutLMPredictionHeadTransformc                    sV   t    t|j|j| _t|jtr6t	|j | _
n|j| _
tj|j|jd| _d S r   )r!   r"   r   r_   r%   r   r   r   r   r
   transform_act_fnr0   r/   r7   r:   r<   r=   r"   6  s    
z(LayoutLMPredictionHeadTransform.__init__r   c                 C   s"   |  |}| |}| |}|S r   )r   r   r0   r   r<   r<   r=   rM   ?  s    


z'LayoutLMPredictionHeadTransform.forwardr   r<   r<   r:   r=   r   5  s   	r   c                       s$   e Zd Z fddZdd Z  ZS )LayoutLMLMPredictionHeadc                    sL   t    t|| _tj|j|jdd| _t	t
|j| _| j| j_d S )NF)bias)r!   r"   r   	transformr   r_   r%   r$   decoder	Parameterr4   rC   r   r7   r:   r<   r=   r"   H  s
    

z!LayoutLMLMPredictionHead.__init__c                 C   s   |  |}| |}|S r   )r   r   r   r<   r<   r=   rM   U  s    

z LayoutLMLMPredictionHead.forward)rN   rO   rP   r"   rM   rR   r<   r<   r:   r=   r   G  s   r   c                       s0   e Zd Z fddZejejdddZ  ZS )LayoutLMOnlyMLMHeadc                    s   t    t|| _d S r   )r!   r"   r   predictionsr7   r:   r<   r=   r"   ]  s    
zLayoutLMOnlyMLMHead.__init__)sequence_outputrh   c                 C   s   |  |}|S r   )r   )r8   r   prediction_scoresr<   r<   r=   rM   a  s    
zLayoutLMOnlyMLMHead.forwardr   r<   r<   r:   r=   r   \  s   r   c                   @   s2   e Zd ZdZeZeZdZdZ	dd Z
d
ddZd	S )LayoutLMPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    layoutlmTc                 C   s   t |tjr:|jjjd| jjd |jdk	r|jj	  ndt |tj
rz|jjjd| jjd |jdk	r|jj|j 	  n$t |tr|jj	  |jjd dS )zInitialize the weightsg        )ZmeanZstdN      ?)r   r   r_   weightdataZnormal_r9   Zinitializer_ranger   Zzero_r#   r   r.   Zfill_)r8   r   r<   r<   r=   _init_weightsq  s    


z%LayoutLMPreTrainedModel._init_weightsFc                 C   s   t |tr||_d S r   )r   r   r   )r8   r   rb   r<   r<   r=   _set_gradient_checkpointing  s    
z3LayoutLMPreTrainedModel._set_gradient_checkpointingN)F)rN   rO   rP   rQ   r   config_class&LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LISTZpretrained_model_archive_mapZbase_model_prefixZsupports_gradient_checkpointingr   r   r<   r<   r<   r=   r   f  s   r   a4  
    The LayoutLM model was proposed in [LayoutLM: Pre-training of Text and Layout for Document Image
    Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei and
    Ming Zhou.

    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`LayoutLMConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization.
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: `1` for
            tokens that are NOT MASKED, `0` for MASKED tokens.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`: `0` corresponds to a *sentence A* token, `1` corresponds to a *sentence B* token

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: `1`
            indicates the head is **not masked**, `0` indicates the head is **masked**.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            If set to `True`, the attentions tensors of all attention layers are returned. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            If set to `True`, the hidden states of all layers are returned. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            If set to `True`, the model will return a [`~utils.ModelOutput`] instead of a plain tuple.
zbThe bare LayoutLM Model transformer outputting raw hidden-states without any specific head on top.c                       s   e Zd Z fddZdd Zdd Zdd Zee	d	e
eed
deej eej eej eej eej eej eej eej eej ee ee ee eeef dddZ  ZS )LayoutLMModelc                    s@   t t| | || _t|| _t|| _t|| _	| 
  d S r   )r!   r   r"   r9   r   rL   r   encoderr   pooler	post_initr7   r:   r<   r=   r"     s    


zLayoutLMModel.__init__c                 C   s   | j jS r   rL   r&   r8   r<   r<   r=   get_input_embeddings  s    z"LayoutLMModel.get_input_embeddingsc                 C   s   || j _d S r   r   )r8   rb   r<   r<   r=   set_input_embeddings  s    z"LayoutLMModel.set_input_embeddingsc                 C   s*   |  D ]\}}| jj| j| qdS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr   r   r   r   )r8   Zheads_to_pruner   r   r<   r<   r=   _prune_heads  s    zLayoutLMModel._prune_headsbatch_size, sequence_lengthoutput_typer   N)rF   rG   rn   rH   r   ro   rI   rp   rq   rs   r   r   rh   c                 C   s0  |
dk	r|
n| j j}
|dk	r |n| j j}|dk	r4|n| j j}|dk	rV|dk	rVtdn@|dk	rt| || | }n"|dk	r| dd }ntd|dk	r|jn|j}|dkrtj	||d}|dkrtj
|tj|d}|dkrtj
|d tj|d}|dd	}|j| jd
}d| t| jj }|dk	r| dkrx|dddd}|| j jdddd}n$| d	kr|ddd}|jt|  jd
}ndg| j j }| j|||||d}| j||||
||d}|d }| |}|s||f|dd  S t|||j|j|jdS )a	  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMModel
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
        >>> model = LayoutLMModel.from_pretrained("microsoft/layoutlm-base-uncased")

        >>> words = ["Hello", "world"]
        >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782]

        >>> token_boxes = []
        >>> for word, box in zip(words, normalized_word_boxes):
        ...     word_tokens = tokenizer.tokenize(word)
        ...     token_boxes.extend([box] * len(word_tokens))
        >>> # add bounding boxes of cls + sep tokens
        >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]]

        >>> encoding = tokenizer(" ".join(words), return_tensors="pt")
        >>> input_ids = encoding["input_ids"]
        >>> attention_mask = encoding["attention_mask"]
        >>> token_type_ids = encoding["token_type_ids"]
        >>> bbox = torch.tensor([token_boxes])

        >>> outputs = model(
        ...     input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids
        ... )

        >>> last_hidden_states = outputs.last_hidden_state
        ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer   z5You have to specify either input_ids or inputs_embeds)r@   r>   )   r   rA   rw   r   r   )rF   rG   r   rH   rI   )ro   rs   r   r   )r   Zpooler_outputrm   r   r   )r9   rs   r   use_return_dictr[   Z%warn_if_padding_and_no_attention_maskrB   r@   r4   ZonesrC   rD   Z	unsqueezer{   r?   Zfinfominru   r6   r   next
parametersrL   r   r   r   rm   r   r   )r8   rF   rG   rn   rH   r   ro   rI   rp   rq   rs   r   r   rJ   r@   Zextended_attention_maskZembedding_outputZencoder_outputsr   r   r<   r<   r=   rM     sp    3



zLayoutLMModel.forward)NNNNNNNNNNNN)rN   rO   rP   r"   r   r   r   r   LAYOUTLM_INPUTS_DOCSTRINGformatr   r   _CONFIG_FOR_DOCr   r4   
LongTensorr   r   r   r   rM   rR   r<   r<   r:   r=   r     sB   
            
r   z6LayoutLM Model with a `language modeling` head on top.c                       s   e Zd ZddgZ fddZdd Zdd Zd	d
 Zee	
deeeddeej eej eej eej eej eej eej eej eej eej ee ee ee eeef dddZ  ZS )LayoutLMForMaskedLMzcls.predictions.decoder.biaszcls.predictions.decoder.weightc                    s,   t  | t|| _t|| _|   d S r   )r!   r"   r   r   r   clsr   r7   r:   r<   r=   r"   `  s    

zLayoutLMForMaskedLM.__init__c                 C   s
   | j jjS r   r   rL   r&   r   r<   r<   r=   r   i  s    z(LayoutLMForMaskedLM.get_input_embeddingsc                 C   s
   | j jjS r   r   r   r   r   r<   r<   r=   get_output_embeddingsl  s    z)LayoutLMForMaskedLM.get_output_embeddingsc                 C   s   || j j_d S r   r   )r8   Znew_embeddingsr<   r<   r=   set_output_embeddingso  s    z)LayoutLMForMaskedLM.set_output_embeddingsr   r   N)rF   rG   rn   rH   r   ro   rI   labelsrp   rq   rs   r   r   rh   c                 C   s   |dk	r|n| j j}| j||||||||	|
|||d}|d }| |}d}|dk	rvt }||d| j j|d}|s|f|dd  }|dk	r|f| S |S t|||j|j	dS )a6  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMForMaskedLM
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
        >>> model = LayoutLMForMaskedLM.from_pretrained("microsoft/layoutlm-base-uncased")

        >>> words = ["Hello", "[MASK]"]
        >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782]

        >>> token_boxes = []
        >>> for word, box in zip(words, normalized_word_boxes):
        ...     word_tokens = tokenizer.tokenize(word)
        ...     token_boxes.extend([box] * len(word_tokens))
        >>> # add bounding boxes of cls + sep tokens
        >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]]

        >>> encoding = tokenizer(" ".join(words), return_tensors="pt")
        >>> input_ids = encoding["input_ids"]
        >>> attention_mask = encoding["attention_mask"]
        >>> token_type_ids = encoding["token_type_ids"]
        >>> bbox = torch.tensor([token_boxes])

        >>> labels = tokenizer("Hello world", return_tensors="pt")["input_ids"]

        >>> outputs = model(
        ...     input_ids=input_ids,
        ...     bbox=bbox,
        ...     attention_mask=attention_mask,
        ...     token_type_ids=token_type_ids,
        ...     labels=labels,
        ... )

        >>> loss = outputs.loss
        ```N)
rn   rH   r   ro   rI   rp   rq   rs   r   r   r   r   rA   losslogitsrm   r   )
r9   r   r   r   r   ri   r$   r   rm   r   )r8   rF   rG   rn   rH   r   ro   rI   r   rp   rq   rs   r   r   r   r   r   Zmasked_lm_lossloss_fctr   r<   r<   r=   rM   r  sB    ?
zLayoutLMForMaskedLM.forward)NNNNNNNNNNNNN)rN   rO   rP   Z_tied_weights_keysr"   r   r   r   r   r   r   r   r   r   r   r4   r   r   r   r   r   rM   rR   r<   r<   r:   r=   r   \  sH   	
             
r   z
    LayoutLM Model with a sequence classification head on top (a linear layer on top of the pooled output) e.g. for
    document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset.
    c                       s   e Zd Z fddZdd Zeedee	e
ddeej eej eej eej eej eej eej eej ee ee ee eee	f dd	d
Z  ZS )!LayoutLMForSequenceClassificationc                    sJ   t  | |j| _t|| _t|j| _t	|j
|j| _|   d S r   r!   r"   
num_labelsr   r   r   r1   r2   r3   r_   r%   
classifierr   r7   r:   r<   r=   r"     s    
z*LayoutLMForSequenceClassification.__init__c                 C   s
   | j jjS r   r   r   r<   r<   r=   r     s    z6LayoutLMForSequenceClassification.get_input_embeddingsr   r   NrF   rG   rn   rH   r   ro   rI   r   rs   r   r   rh   c                 C   s~  |dk	r|n| j j}| j||||||||	|
|d
}|d }| |}| |}d}|dk	r:| j jdkr| jdkr|d| j _n4| jdkr|jtj	ks|jtj
krd| j _nd| j _| j jdkrt }| jdkr|| | }n
|||}nN| j jdkrt }||d| j|d}n| j jdkr:t }|||}|sj|f|dd  }|dk	rf|f| S |S t|||j|jd	S )
aF  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMForSequenceClassification
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
        >>> model = LayoutLMForSequenceClassification.from_pretrained("microsoft/layoutlm-base-uncased")

        >>> words = ["Hello", "world"]
        >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782]

        >>> token_boxes = []
        >>> for word, box in zip(words, normalized_word_boxes):
        ...     word_tokens = tokenizer.tokenize(word)
        ...     token_boxes.extend([box] * len(word_tokens))
        >>> # add bounding boxes of cls + sep tokens
        >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]]

        >>> encoding = tokenizer(" ".join(words), return_tensors="pt")
        >>> input_ids = encoding["input_ids"]
        >>> attention_mask = encoding["attention_mask"]
        >>> token_type_ids = encoding["token_type_ids"]
        >>> bbox = torch.tensor([token_boxes])
        >>> sequence_label = torch.tensor([1])

        >>> outputs = model(
        ...     input_ids=input_ids,
        ...     bbox=bbox,
        ...     attention_mask=attention_mask,
        ...     token_type_ids=token_type_ids,
        ...     labels=sequence_label,
        ... )

        >>> loss = outputs.loss
        >>> logits = outputs.logits
        ```N
rF   rG   rn   rH   r   ro   rI   rs   r   r   r   Z
regressionZsingle_label_classificationZmulti_label_classificationr   rA   r   )r9   r   r   r3   r  Zproblem_typer   r?   r4   rD   r\   r   squeezer   ri   r   r   rm   r   )r8   rF   rG   rn   rH   r   ro   rI   r   rs   r   r   r   r   r   r   r   r   r<   r<   r=   rM     sX    =




"


z)LayoutLMForSequenceClassification.forward)NNNNNNNNNNN)rN   rO   rP   r"   r   r   r   r   r   r   r   r   r4   r   r   r   r   r   rM   rR   r<   r<   r:   r=   r     s:   

           
r   a3  
    LayoutLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    sequence labeling (information extraction) tasks such as the [FUNSD](https://guillaumejaume.github.io/FUNSD/)
    dataset and the [SROIE](https://rrc.cvc.uab.es/?ch=13) dataset.
    c                       s   e Zd Z fddZdd Zeedee	e
ddeej eej eej eej eej eej eej eej ee ee ee eee	f dd	d
Z  ZS )LayoutLMForTokenClassificationc                    sJ   t  | |j| _t|| _t|j| _t	|j
|j| _|   d S r   r   r7   r:   r<   r=   r"   j  s    
z'LayoutLMForTokenClassification.__init__c                 C   s
   | j jjS r   r   r   r<   r<   r=   r   t  s    z3LayoutLMForTokenClassification.get_input_embeddingsr   r   Nr  c                 C   s   |dk	r|n| j j}| j||||||||	|
|d
}|d }| |}| |}d}|dk	rzt }||d| j|d}|s|f|dd  }|dk	r|f| S |S t|||j	|j
dS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMForTokenClassification
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
        >>> model = LayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased")

        >>> words = ["Hello", "world"]
        >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782]

        >>> token_boxes = []
        >>> for word, box in zip(words, normalized_word_boxes):
        ...     word_tokens = tokenizer.tokenize(word)
        ...     token_boxes.extend([box] * len(word_tokens))
        >>> # add bounding boxes of cls + sep tokens
        >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]]

        >>> encoding = tokenizer(" ".join(words), return_tensors="pt")
        >>> input_ids = encoding["input_ids"]
        >>> attention_mask = encoding["attention_mask"]
        >>> token_type_ids = encoding["token_type_ids"]
        >>> bbox = torch.tensor([token_boxes])
        >>> token_labels = torch.tensor([1, 1, 0, 0]).unsqueeze(0)  # batch size of 1

        >>> outputs = model(
        ...     input_ids=input_ids,
        ...     bbox=bbox,
        ...     attention_mask=attention_mask,
        ...     token_type_ids=token_type_ids,
        ...     labels=token_labels,
        ... )

        >>> loss = outputs.loss
        >>> logits = outputs.logits
        ```Nr  r   r   rA   r   )r9   r   r   r3   r  r   ri   r   r   rm   r   )r8   rF   rG   rn   rH   r   ro   rI   r   rs   r   r   r   r   r   r   r   r   r<   r<   r=   rM   w  s:    ;

z&LayoutLMForTokenClassification.forward)NNNNNNNNNNN)rN   rO   rP   r"   r   r   r   r   r   r   r   r   r4   r   r   r   r   r   rM   rR   r<   r<   r:   r=   r  a  s:   	

           
r  a  
    LayoutLM Model with a span classification head on top for extractive question-answering tasks such as
    [DocVQA](https://rrc.cvc.uab.es/?ch=17) (a linear layer on top of the final hidden-states output to compute `span
    start logits` and `span end logits`).
    c                       s   e Zd Zd fdd	Zdd Zeeeddee	j
 ee	j
 ee	j ee	j
 ee	j
 ee	j ee	j ee	j
 ee	j
 ee ee ee eeef dd	d
Z  ZS )LayoutLMForQuestionAnsweringTc                    s<   t  | |j| _t|| _t|j|j| _| 	  d S r   )
r!   r"   r   r   r   r   r_   r%   
qa_outputsr   )r8   r9   Zhas_visual_segment_embeddingr:   r<   r=   r"     s
    
z%LayoutLMForQuestionAnswering.__init__c                 C   s
   | j jjS r   r   r   r<   r<   r=   r     s    z1LayoutLMForQuestionAnswering.get_input_embeddingsr   N)rF   rG   rn   rH   r   ro   rI   start_positionsend_positionsrs   r   r   rh   c                 C   sR  |dk	r|n| j j}| j||||||||
||d
}|d }| |}|jddd\}}|d }|d }d}|dk	r
|	dk	r
t| dkr|d}t|	 dkr|	d}	|d}|	d|}|		d|}	t
|d}|||}|||	}|| d }|s<||f|dd  }|dk	r8|f| S |S t||||j|jd	S )
a2
  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.

        Returns:

        Example:

        In the example below, we prepare a question + context pair for the LayoutLM model. It will give us a prediction
        of what it thinks the answer is (the span of the answer within the texts parsed from the image).

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMForQuestionAnswering
        >>> from datasets import load_dataset
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("impira/layoutlm-document-qa", add_prefix_space=True)
        >>> model = LayoutLMForQuestionAnswering.from_pretrained("impira/layoutlm-document-qa", revision="1e3ebac")

        >>> dataset = load_dataset("nielsr/funsd", split="train")
        >>> example = dataset[0]
        >>> question = "what's his name?"
        >>> words = example["words"]
        >>> boxes = example["bboxes"]

        >>> encoding = tokenizer(
        ...     question.split(), words, is_split_into_words=True, return_token_type_ids=True, return_tensors="pt"
        ... )
        >>> bbox = []
        >>> for i, s, w in zip(encoding.input_ids[0], encoding.sequence_ids(0), encoding.word_ids(0)):
        ...     if s == 1:
        ...         bbox.append(boxes[w])
        ...     elif i == tokenizer.sep_token_id:
        ...         bbox.append([1000] * 4)
        ...     else:
        ...         bbox.append([0] * 4)
        >>> encoding["bbox"] = torch.tensor([bbox])

        >>> word_ids = encoding.word_ids(0)
        >>> outputs = model(**encoding)
        >>> loss = outputs.loss
        >>> start_scores = outputs.start_logits
        >>> end_scores = outputs.end_logits
        >>> start, end = word_ids[start_scores.argmax(-1)], word_ids[end_scores.argmax(-1)]
        >>> print(" ".join(words[start : end + 1]))
        M. Hamann P. Harper, P. Martinez
        ```Nr  r   r   r   rt   )Zignore_indexrA   )r   start_logits
end_logitsrm   r   )r9   r   r   r  splitr  r~   r   rB   clampr   r   rm   r   )r8   rF   rG   rn   rH   r   ro   rI   r  r	  rs   r   r   r   r   r   r
  r  Z
total_lossZignored_indexr   Z
start_lossZend_lossr   r<   r<   r=   rM     sR    F






z$LayoutLMForQuestionAnswering.forward)T)NNNNNNNNNNNN)rN   rO   rP   r"   r   r   r   r   r   r4   r   r   r   r   r   rM   rR   r<   r<   r:   r=   r    s<   	

            
r  )@rQ   r|   typingr   r   r   r4   Ztorch.utils.checkpointr   Ztorch.nnr   r   r   Zactivationsr
   Zmodeling_outputsr   r   r   r   r   r   Zmodeling_utilsr   Zpytorch_utilsr   r   r   r   r   r   r   r   Zconfiguration_layoutlmr   Z
get_loggerrN   r   r   Z_CHECKPOINT_FOR_DOCr   r0   r.   Moduler   rS   r   r   r   r   r   r   r   r   r   r   r   ZLAYOUTLM_START_DOCSTRINGr   r   r   r   r  r  r<   r<   r<   r=   <module>   sv    
M 2Wc
 / | n