U 9%e@sZdZddlZddlZddlmZmZmZddlZddlZddlm Z ddl m Z m Z m Z ddlmZddlmZmZmZdd lmZmZmZmZmZmZdd lmZmZmZmZdd l m!Z!d d l"m#Z#e!$e%Z&dZ'dZ(ddgZ)Gddde j*Z+d=ddZ,Gddde j*Z-Gddde j*Z.Gddde j*Z/Gddde j*Z0Gddde j*Z1Gdd d e j*Z2Gd!d"d"e j*Z3Gd#d$d$e j*Z4Gd%d&d&e j*Z5Gd'd(d(e j*Z6Gd)d*d*e j*Z7Gd+d,d,e j*Z8Gd-d.d.eZ9d/Z:d0Z;ed1e:Gd2d3d3e9Zed:e:Gd;d<dz PyTorch MarkupLM model.N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)add_start_docstrings%add_start_docstrings_to_model_forwardreplace_return_docstrings))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModelapply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)logging)MarkupLMConfigzmicrosoft/markuplm-baserzmicrosoft/markuplm-largecs*eZdZdZfddZdddZZS)XPathEmbeddingszConstruct the embeddings from xpath tags and subscripts. We drop tree-id in this version, as its info can be covered by xpath. cstt|j|_tj|jj|_t j |_ t |_ tj|jdj|_tdjj|_tfddt|jD|_tfddt|jD|_dS)Ncsg|]}tjjqS)r EmbeddingZmax_xpath_tag_unit_embeddingsxpath_unit_hidden_size.0_configrm/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/models/markuplm/modeling_markuplm.py Psz,XPathEmbeddings.__init__..csg|]}tjjqSr)rrZmax_xpath_subs_unit_embeddingsrr r#rr%r&Ws)superr__init__ max_depthrLinearr hidden_sizeZxpath_unitseq2_embeddingsDropouthidden_dropout_probdropoutZReLU activationxpath_unitseq2_inner inner2emb ModuleListrangexpath_tag_sub_embeddingsxpath_subs_sub_embeddingsselfr$ __class__r#r%r(Cs"   zXPathEmbeddings.__init__Nc Csg}g}t|jD]P}||j||dddd|f||j||dddd|fqtj|dd}tj|dd}||}||| | |}|S)Ndim) r3r)appendr4r5torchcatr1r.r/r0)r7xpath_tags_seqxpath_subs_seqZxpath_tags_embeddingsZxpath_subs_embeddingsixpath_embeddingsrrr%forward]s&(zXPathEmbeddings.forward)NN)__name__ __module__ __qualname____doc__r(rD __classcell__rrr8r%r=s rcCs6||}tj|dd|||}||S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rr;)neintr>ZcumsumZtype_aslong) input_ids padding_idxpast_key_values_lengthmaskZincremental_indicesrrr%"create_position_ids_from_input_idsps rQcs2eZdZdZfddZddZd dd ZZS) MarkupLMEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cstt|||_tj|j|j|jd|_ t|j |j|_ |j |_ t ||_t|j|j|_tj|j|jd|_t|j|_|jdt|j ddd|j|_tj|j |j|jd|_ dS)N)rNZeps position_ids)rr:F) persistent)r'rRr(r$rr vocab_sizer+Z pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsr)rrCZtype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsr,r-r.Zregister_bufferr>arangeexpandrNr6r8rr%r(s( zMarkupLMEmbeddings.__init__cCsN|dd}|d}tj|jd||jdtj|jd}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr:rdtypedevicer)sizer>r]rNrLra unsqueezer^)r7 inputs_embeds input_shapeZsequence_lengthrTrrr%&create_position_ids_from_inputs_embedss  z9MarkupLMEmbeddings.create_position_ids_from_inputs_embedsNrcCs<|dk r|}n|dd}|dk r0|jn|j} |dkr`|dk rVt||j|}n ||}|dkrztj|tj| d}|dkr||}|dkr|j j tj t t ||jgtj| d}|dkr|j jtj t t ||jgtj| d}|} ||} ||} |||} | | | | }||}||}|S)Nr:r_)rbrarQrNrfr>zerosrLrWr$Z tag_pad_idonestuplelistr)Z subs_pad_idrYrZrCr[r.)r7rMr@rAtoken_type_idsrTrdrOreraZwords_embeddingsrYrZrC embeddingsrrr%rDs@           zMarkupLMEmbeddings.forward)NNNNNNr)rErFrGrHr(rfrDrIrrr8r%rRs rRcs4eZdZfddZejejejdddZZS)MarkupLMSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNrS) r'r(rr*r+denser[r\r,r-r.r6r8rr%r(s zMarkupLMSelfOutput.__init__ hidden_states input_tensorreturncCs&||}||}|||}|SNror.r[r7rqrrrrr%rDs  zMarkupLMSelfOutput.forwardrErFrGr(r>TensorrDrIrrr8r%rms rmcs0eZdZfddZejejdddZZS)MarkupLMIntermediatecsBtt|j|j|_t|jt r6t |j|_ n|j|_ dSrt) r'r(rr*r+intermediate_sizero isinstance hidden_actstrr intermediate_act_fnr6r8rr%r(s   zMarkupLMIntermediate.__init__rqrscCs||}||}|Srt)ror~r7rqrrr%rDs  zMarkupLMIntermediate.forwardrwrrr8r%rys rycs4eZdZfddZejejejdddZZS)MarkupLMOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSrn) r'r(rr*rzr+ror[r\r,r-r.r6r8rr%r(s zMarkupLMOutput.__init__rpcCs&||}||}|||}|Srtrurvrrr%rD s  zMarkupLMOutput.forwardrwrrr8r%rs rcs0eZdZfddZejejdddZZS)MarkupLMPoolercs*tt|j|j|_t|_dSrt)r'r(rr*r+roZTanhr/r6r8rr%r(s zMarkupLMPooler.__init__rcCs(|dddf}||}||}|S)Nr)ror/)r7rqZfirst_token_tensor pooled_outputrrr%rDs  zMarkupLMPooler.forwardrwrrr8r%rs rcs0eZdZfddZejejdddZZS)MarkupLMPredictionHeadTransformcsVtt|j|j|_t|jtr6t |j|_ n|j|_ tj |j|j d|_ dSrn) r'r(rr*r+ror{r|r}r transform_act_fnr[r\r6r8rr%r("s   z(MarkupLMPredictionHeadTransform.__init__rcCs"||}||}||}|Srt)rorr[rrrr%rD+s   z'MarkupLMPredictionHeadTransform.forwardrwrrr8r%r!s rcs$eZdZfddZddZZS)MarkupLMLMPredictionHeadcsLtt||_tj|j|jdd|_t t |j|_ |j |j_ dS)NF)bias) r'r(r transformrr*r+rVdecoder Parameterr>rgrr6r8rr%r(4s   z!MarkupLMLMPredictionHead.__init__cCs||}||}|Srt)rrrrrr%rDAs  z MarkupLMLMPredictionHead.forward)rErFrGr(rDrIrrr8r%r3s rcs0eZdZfddZejejdddZZS)MarkupLMOnlyMLMHeadcstt||_dSrt)r'r(r predictionsr6r8rr%r(Is zMarkupLMOnlyMLMHead.__init__)sequence_outputrscCs||}|Srt)r)r7rprediction_scoresrrr%rDMs zMarkupLMOnlyMLMHead.forwardrwrrr8r%rHs rc seZdZd fdd ZejejdddZd ejeejeejeejeejee e ejee e ejdd d Z Z S) MarkupLMSelfAttentionNcst|j|jdkr>t|ds>td|jd|jd|j|_t|j|j|_|j|j|_t |j|j|_ t |j|j|_ t |j|j|_ t |j|_|pt|dd|_|jdks|jd kr|j|_t d |jd |j|_|j|_dS) NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()position_embedding_typeabsolute relative_keyrelative_key_queryr)r'r(r+num_attention_headshasattr ValueErrorrKattention_head_size all_head_sizerr*querykeyvaluer,Zattention_probs_dropout_probr.getattrrrXrdistance_embedding is_decoderr7r$rr8rr%r(Ts* zMarkupLMSelfAttention.__init__)xrscCs6|dd|j|jf}||}|ddddS)Nr:rrrr )rbrrviewpermute)r7rZ new_x_shaperrr%transpose_for_scoresns z*MarkupLMSelfAttention.transpose_for_scoresFrqattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsrscCs||}|dk } | r4|dk r4|d} |d} |}n| r^|||} |||} |}nv|dk r|||} |||} tj|d| gdd} tj|d| gdd} n |||} |||} ||} |dk } |jr| | f}t| | dd}|j dks |j dkr| j d| j d}}| r^tj |dtj |j d dd}ntj|tj |j d dd}tj|tj |j d dd}||}|||jd}|j| jd }|j dkrtd | |}||}n4|j dkrtd | |}td | |}|||}|t|j}|dk r:||}tjj|dd}||}|dk rf||}t|| }|dddd }|dd|jf}||}|r||fn|f}|jr||f}|S)Nrrrr;r:rrr_r`zbhld,lrd->bhlrzbhrd,lrd->bhlrr ) rrrrr>r?rmatmulZ transposershapeZtensorrLrarr]rrXtor`ZeinsummathsqrtrrZ functionalZsoftmaxr.r contiguousrbr)r7rqrrrrrrZmixed_query_layerZis_cross_attentionZ key_layerZ value_layerZ query_layer use_cacheZattention_scoresZ query_lengthZ key_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZ context_layerZnew_context_layer_shapeoutputsrrr%rDssp             zMarkupLMSelfAttention.forward)N)NNNNNF) rErFrGr(r>rxrr FloatTensorrboolrDrIrrr8r%rSs$rc sveZdZd fdd ZddZd ejeejeejeejeejee e ejee e ejddd Z Z S) MarkupLMAttentionNcs.tt||d|_t||_t|_dS)Nr)r'r(rr7rmoutputset pruned_headsrr8rr%r(s  zMarkupLMAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrr;)lenrr7rrrrrrrrrorunion)r7headsindexrrr% prune_headss zMarkupLMAttention.prune_headsFrc Cs<||||||||}||d|} | f|dd} | S)Nrr)r7r) r7rqrrrrrrZ self_outputsattention_outputrrrr%rDs  zMarkupLMAttention.forward)N)NNNNNF) rErFrGr(rr>rxrrrrrDrIrrr8r%rs$rc steZdZfddZd ejeejeejeejeejeeeejee eejdddZ dd Z Z S) MarkupLMLayercsrt|j|_d|_t||_|j|_|j|_|jrZ|jsLt|dt|dd|_ t ||_ t ||_ dS)Nrz> should be used as a decoder model if cross attention is addedrr)r'r(chunk_size_feed_forward seq_len_dimr attentionradd_cross_attentionrcrossattentionry intermediaterrr6r8rr%r( s   zMarkupLMLayer.__init__NFrc Cs|dk r|ddnd}|j|||||d} | d} |jrP| dd} | d} n | dd} d} |jr|dk rt|dstd|d|dk r|d dnd}|| ||||||}|d} | |dd} |d} | | } t|j|j|j| }|f| } |jr| | f} | S) Nrrrrrr:rz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) rrrrrrfeed_forward_chunkrr)r7rqrrrrrrZself_attn_past_key_valueZself_attention_outputsrrZpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputs layer_outputrrr%rDsV         zMarkupLMLayer.forwardcCs||}|||}|Srt)rr)r7rZintermediate_outputrrrr%r\s  z MarkupLMLayer.feed_forward_chunk)NNNNNF) rErFrGr(r>rxrrrrrDrrIrrr8r%r s$  ArcseZdZfddZd ejeejeejeejeejeeeejee ee ee ee e eeje fd ddZ Z S) MarkupLMEncodercs:t|_tfddtjD|_d|_dS)Ncsg|] }tqSr)rr r#rr%r&gsz,MarkupLMEncoder.__init__..F) r'r(r$rr2r3num_hidden_layerslayergradient_checkpointingr6r8r#r%r(ds  zMarkupLMEncoder.__init__NFT) rqrrrrpast_key_valuesrroutput_hidden_states return_dictrsc  st| rdnd} rdnd} r(|jjr(dnd} |jrJ|jrJ|rJtdd}|rRdnd}t|jD]\}}| rv| |f} |dk r||nd}|dk r||nd|jr|jrևfdd}tj j |||||||}n||||||}|d}|r||df7}r`| |df} |jjr`| |d f} q`| r@| |f} | sbt d d ||| | | fDSt ||| | | d S) NrzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fcsfdd}|S)Ncs|fSrtr)inputs)modulerrrr%custom_forwardszNMarkupLMEncoder.forward..create_custom_forward..custom_forwardr)rrr)rr%create_custom_forwardsz6MarkupLMEncoder.forward..create_custom_forwardrr:rrcss|]}|dk r|VqdSrtr)r!vrrr% sz*MarkupLMEncoder.forward..)last_hidden_staterrq attentionscross_attentions) r$rrZtrainingloggerZ warning_once enumeraterr>utils checkpointrir)r7rqrrrrrrrrrZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacherBZ layer_moduleZlayer_head_maskrZ layer_outputsrrr%rDjsv          zMarkupLMEncoder.forward) NNNNNNFFT)rErFrGr(r>rxrrrrrrrDrIrrr8r%rcs. rcsLeZdZdZeZeZdZddZ e e e e ejfdfdd ZZS)MarkupLMPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. markuplmcCst|tjr:|jjjd|jjd|jdk r|jj nft|tj rz|jjjd|jjd|j dk r|jj|j  n&t|tj r|jj |jj ddS)zInitialize the weightsg)ZmeanZstdN?)r{rr*weightdataZnormal_r$Zinitializer_rangerZzero_rrNr[Zfill_)r7rrrr% _init_weightss      z%MarkupLMPreTrainedModel._init_weights)pretrained_model_name_or_pathcstt|j|f||Srt)r'rfrom_pretrained)clsrZ model_argskwargsr8rr%rs z'MarkupLMPreTrainedModel.from_pretrained)rErFrGrHr config_class&MARKUPLM_PRETRAINED_MODEL_ARCHIVE_LISTZpretrained_model_archive_mapZbase_model_prefixr classmethodrrr}osPathLikerrIrrr8r%rsraK This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`MarkupLMConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) xpath_tags_seq (`torch.LongTensor` of shape `({0}, config.max_depth)`, *optional*): Tag IDs for each token in the input sequence, padded up to config.max_depth. xpath_subs_seq (`torch.LongTensor` of shape `({0}, config.max_depth)`, *optional*): Subscript IDs for each token in the input sequence, padded up to config.max_depth. attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: `1` for tokens that are NOT MASKED, `0` for MASKED tokens. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: `0` corresponds to a *sentence A* token, `1` corresponds to a *sentence B* token [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: `1` indicates the head is **not masked**, `0` indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): If set to `True`, the attentions tensors of all attention layers are returned. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): If set to `True`, the hidden states of all layers are returned. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): If set to `True`, the model will return a [`~file_utils.ModelOutput`] instead of a plain tuple. zbThe bare MarkupLM Model transformer outputting raw hidden-states without any specific head on top.cseZdZdfdd ZddZddZdd Zee d e e e d de eje eje eje eje eje eje eje eje ee ee eeee fd ddZdddZddZZS) MarkupLMModelTcsDt|||_t||_t||_|r2t|nd|_| dSrt) r'r(r$rRrlrencoderrpooler post_init)r7r$add_pooling_layerr8rr%r()s    zMarkupLMModel.__init__cCs|jjSrtrlrW)r7rrr%get_input_embeddings5sz"MarkupLMModel.get_input_embeddingscCs ||j_dSrtr)r7rrrr%set_input_embeddings8sz"MarkupLMModel.set_input_embeddingscCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrrr)r7Zheads_to_prunerrrrr% _prune_heads;szMarkupLMModel._prune_headsbatch_size, sequence_length output_typerN) rMr@rArrkrTrrdrrrrsc Cs| dk r | n|jj} | dk r | n|jj} | dk r4| n|jj} |dk rV|dk rVtdn@|dk rt||||} n"|dk r|dd} ntd|dk r|jn|j} |dkrtj | | d}|dkrtj | tj | d}| d d}|j |jd }d |d }|dk r|dkrP| d  d  d d}||jjdddd}n$|dkrt| d d d}|j t|jd }ndg|jj}|j||||||d }|j|||| | | d}|d }|jdk r||nd}| s||f|ddSt|||j|j|jdS)a` Returns: Examples: ```python >>> from transformers import AutoProcessor, MarkupLMModel >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> model = MarkupLMModel.from_pretrained("microsoft/markuplm-base") >>> html_string = " Page Title " >>> encoding = processor(html_string, return_tensors="pt") >>> outputs = model(**encoding) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 4, 768] ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer:z5You have to specify either input_ids or inputs_embeds)rar_rrrrgr)rMr@rArTrkrd)rrrr)rZ pooler_outputrqrr)r$rruse_return_dictrZ%warn_if_padding_and_no_attention_maskrbrar>rhrgrLrcrr`r<r^rnext parametersrlrrrrqrr)r7rMr@rArrkrTrrdrrrreraZextended_attention_maskZembedding_outputZencoder_outputsrrrrr%rDCsn$     zMarkupLMModel.forwardcKsB|j}|dkr||}|dk r4|ddddf}||||dS)Nr:)rMrrr)rZnew_ones)r7rMrrrZ model_kwargsrerrr%prepare_inputs_for_generations z+MarkupLMModel.prepare_inputs_for_generationcs.d}|D] }|tfdd|Df7}q|S)Nrc3s"|]}|d|jVqdS)rN)Z index_selectrra)r!Z past_statebeam_idxrr%rsz/MarkupLMModel._reorder_cache..)ri)r7rrZreordered_pastZ layer_pastrrr%_reorder_caches zMarkupLMModel._reorder_cache)T) NNNNNNNNNNN)NNT)rErFrGr(rrrr MARKUPLM_INPUTS_DOCSTRINGformatr r_CONFIG_FOR_DOCrr>Z LongTensorrrrrrDrrrIrrr8r%r#sH    i rz MarkupLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). cseZdZfddZeedeee dd e e j e e j e e j e e j e e j e e j e e j e e j e e j e e j e e e e e e eee j efdddZZS) MarkupLMForQuestionAnsweringcs@t||j|_t|dd|_t|j|j|_| dSNF)r) r'r( num_labelsrrrr*r+ qa_outputsrr6r8rr%r(s  z%MarkupLMForQuestionAnswering.__init__rrN)rMr@rArrkrTrrdstart_positions end_positionsrrrrsc CsT| dk r | n|jj} |j||||||||| | | d }|d}||}|jddd\}}|d}|d}d}| dk r | dk r t| dkr| d} t| dkr| d} |d}| d|| d|t |d}||| }||| }||d}| s>||f|dd}|dk r:|f|S|St ||||j |j d S) a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. Returns: Examples: ```python >>> from transformers import AutoProcessor, MarkupLMForQuestionAnswering >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base-finetuned-websrc") >>> model = MarkupLMForQuestionAnswering.from_pretrained("microsoft/markuplm-base-finetuned-websrc") >>> html_string = " My name is Niels " >>> question = "What's his name?" >>> encoding = processor(html_string, questions=question, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> answer_start_index = outputs.start_logits.argmax() >>> answer_end_index = outputs.end_logits.argmax() >>> predict_answer_tokens = encoding.input_ids[0, answer_start_index : answer_end_index + 1] >>> processor.decode(predict_answer_tokens).strip() 'Niels' ```N r@rArrkrTrrdrrrrrr:r;)Z ignore_indexr)loss start_logits end_logitsrqr)r$rrr splitsqueezerrrbZclamp_rrrqr)r7rMr@rArrkrTrrdr r rrrrrlogitsrrZ total_lossZ ignored_indexloss_fctZ start_lossZend_lossrrrr%rDsT6          z$MarkupLMForQuestionAnswering.forward) NNNNNNNNNNNNN)rErFrGr(r rrr rrrr>rxrrrrDrIrrr8r%rs@  rz9MarkupLM Model with a `token_classification` head on top.cseZdZfddZeedeee dd e e j e e j e e j e e j e e j e e j e e j e e j e e j e e e e e e eee j efd ddZZS) MarkupLMForTokenClassificationcsbt||j|_t|dd|_|jdk r2|jn|j}t||_ t |j |j|_ | dSr )r'r(r rrclassifier_dropoutr-rr,r.r*r+ classifierrr7r$rr8rr%r(Ks  z'MarkupLMForTokenClassification.__init__rrN rMr@rArrkrTrrdlabelsrrrrsc  Cs| dk r | n|jj} |j||||||||| | | d } | d}||}d}| dk rtt}||d|jj| d}| s|f| dd}|dk r|f|S|St||| j| j dS)a  labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Returns: Examples: ```python >>> from transformers import AutoProcessor, AutoModelForTokenClassification >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> processor.parse_html = False >>> model = AutoModelForTokenClassification.from_pretrained("microsoft/markuplm-base", num_labels=7) >>> nodes = ["hello", "world"] >>> xpaths = ["/html/body/div/li[1]/div/span", "/html/body/div/li[1]/div/span"] >>> node_labels = [1, 2] >>> encoding = processor(nodes=nodes, xpaths=xpaths, node_labels=node_labels, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> loss = outputs.loss >>> logits = outputs.logits ```Nrrr:rrrrqr) r$rrrrrr rrqr)r7rMr@rArrkrTrrdrrrrrrrrrrrrr%rDYs@, z&MarkupLMForTokenClassification.forward) NNNNNNNNNNNN)rErFrGr(r rrr rrrr>rxrrrrDrIrrr8r%rHs<   rz MarkupLM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cseZdZfddZeedeee dd e e j e e j e e j e e j e e j e e j e e j e e j e e j e e e e e e eee j efd ddZZS) !MarkupLMForSequenceClassificationcsdt||j|_||_t||_|jdk r4|jn|j}t ||_ t |j |j|_ |dSrt)r'r(r r$rrrr-rr,r.r*r+rrrr8rr%r(s   z*MarkupLMForSequenceClassification.__init__rrNrc  Cs| dk r | n|jj} |j||||||||| | | d } | d}||}||}d}| dk r<|jjdkr|jdkr~d|j_n4|jdkr| jtj ks| jtj krd|j_nd|j_|jjdkrt }|jdkr|| | }n ||| }nN|jjdkrt }||d|j| d}n|jjdkr 1` a classification loss is computed (Cross-Entropy). Returns: Examples: ```python >>> from transformers import AutoProcessor, AutoModelForSequenceClassification >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> model = AutoModelForSequenceClassification.from_pretrained("microsoft/markuplm-base", num_labels=7) >>> html_string = " Page Title " >>> encoding = processor(html_string, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> loss = outputs.loss >>> logits = outputs.logits ```NrrZ regressionZsingle_label_classificationZmulti_label_classificationr:rr)r$rrr.rZ problem_typer r`r>rLrKrrrrrrrqr)r7rMr@rArrkrTrrdrrrrrrrrrrrrr%rDsZ+      "     z)MarkupLMForSequenceClassification.forward) NNNNNNNNNNNN)rErFrGr(r rrr rrrr>rxrrrrDrIrrr8r%rs<   r)r)@rHrrtypingrrrr>Ztorch.utils.checkpointrZtorch.nnrrrZ activationsr Z file_utilsr r r Zmodeling_outputsrrrrrrZmodeling_utilsrrrrrrZconfiguration_markuplmrZ get_loggerrErZ_CHECKPOINT_FOR_DOCrrModulerrQrRrmryrrrrrrrrrrZMARKUPLM_START_DOCSTRINGrrrrrrrrr%sn      3 c 2Wb" 1$wc