U ,-e@sdZddlZddlZddlmZmZmZddlZddlmZddl m Z ddl m Z ddl mZmZdd lmZdd lmZmZmZd d lmZeeZd ZdZdgZd(ejejej e!dddZ"d)ej#ejee!dddZ$Gdddej%Z&Gdddej'Z(Gdddej'Z)Gdddej'Z*GdddeZ+dZ,Gd d!d!e+Z-ed"e,Gd#d$d$e+Z.ed%e,Gd&d'd'e+Z/dS)*z0 PyTorch TrOCR decoder model (based on RoBERTa).N)OptionalTupleUnion)nn)CrossEntropyLoss)ACT2FN))BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentions)PreTrainedModel)add_start_docstringsloggingreplace_return_docstrings) TrOCRConfigrz microsoft/trocr-base-handwritten)input_ids_shapedtypedevicepast_key_values_lengthcCs|\}}tj||ft|j|d}tj|d|d}|||d|ddkd||}|dkrtj tj ||||d|gdd}|ddddddf |d|||S)zB Make causal mask used for bi-directional self-attention. )rrrrrdimN) torchfullfinfominarangesizeZ masked_fill_viewtocatzerosexpand)rrrrbsztgt_lenmaskZ mask_condr'i/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/trocr/modeling_trocr.py_make_causal_mask.s"  r))r&rr%cCsj|\}}|dk r|n|}|ddddddf|d|||}d|}||tjt|jS)z_ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. Nr?)rr#r Z masked_fillrboolrr)r&rr%r$src_lenZ expanded_maskZ inverted_maskr'r'r( _expand_mask@s  *r-cs@eZdZdZeedfdd Zd ejedfdd ZZ S) TrOCRLearnedPositionalEmbeddingzN This module learns positional embeddings up to a fixed maximum size. )num_embeddings embedding_dimcsd|_t||j|dS)N)offsetsuper__init__)selfr/r0 __class__r'r(r4Tsz(TrOCRLearnedPositionalEmbedding.__init__r input_idsrcsH|jdd\}}tj|||tj|jjd|d}t||j S)z3`input_ids' shape is expected to be [bsz x seqlen].Nr1rr) shaperrlongweightrr#r3forwardr2)r5r9rr$seq_lenZ positionsr6r'r(r=Zsz'TrOCRLearnedPositionalEmbedding.forward)r) __name__ __module__ __qualname____doc__intr4rTensorr= __classcell__r'r'r6r(r.Osr.cseZdZdZdeeeedfdd ZedeeeedddZe de j ed d d Z de j eeed ddZ ZS)"TrOCRSinusoidalPositionalEmbeddingzDThis module produces sinusoidal positional embeddings of any length.N) num_positionsr0 padding_idxcsBtd|_||_||_|||||_|dt ddS)Nr1 _float_tensorr) r3r4r2r0rH get_embeddingweightsZregister_bufferr FloatTensor)r5rGr0rHr6r'r(r4hs  z+TrOCRSinusoidalPositionalEmbedding.__init__)r/r0rHcCs|d}td|d}ttj|tjd| }tj|tjdd|d}tjt|t |gdd |d}|ddkrtj|t |dgdd}|dk rd||ddf<| t S) z Build sinusoidal embeddings. This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of "Attention Is All You Need". r1i'r)rrrrN)mathlogrexprfloatZ unsqueezer!sincosrr"r Zget_default_dtype)r/r0rHZhalf_dimZembr'r'r(rJps & z0TrOCRSinusoidalPositionalEmbedding.get_embeddingrr8cCs|\}}|||j||j}|jd|}|jdksL||jdkr`|||j|j|_|j|j|_|j d| d ||d }|S)Nrrr) r"create_position_ids_from_input_idsrHr rrKrJr0rI index_selectrdetach)r5r9rr$r>Z position_idsZmax_posxr'r'r(r=s "z*TrOCRSinusoidalPositionalEmbedding.forward)r9rHrcCs6||}tj|dd|||}||S)z Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. rr)nerCrZcumsumZtype_asr;)r5r9rHrr&Zincremental_indicesr'r'r(rSszETrOCRSinusoidalPositionalEmbedding.create_position_ids_from_input_ids)N)N)r)r)r?r@rArBrCrr4 staticmethodrJrZno_gradrDr=rSrEr'r'r6r(rFesrFc seZdZdZdeeeeeeeedfdd Zej eed d d Z dej e ej e e ej e ej e ej ee ej e ej e e ej fd d dZ ZS)TrOCRAttentionz>Multi-headed attention from 'Attention Is All You Need' paper.NFT) embed_dim num_headskdimvdimdropout is_decoderbiasis_cross_attentionc st||_|dk r|n||_|dk r.|n||_||_||_|||_|j||jksrtd|jd|d|jd|_ ||_ t j |j||d|_ t j |j||d|_t j |||d|_t j |||d|_dS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g࿩ra)r3r4r[r]r^r\r_head_dim ValueErrorscalingr`rLineark_projv_projq_projout_proj) r5configr[r\r]r^r_r`rarbr6r'r(r4s"   zTrOCRAttention.__init__)tensorr>r$cCs ||||j|jddS)Nrr1)rr\rd transpose contiguous)r5rmr>r$r'r'r(_shapeszTrOCRAttention._shape) hidden_stateskey_value_statespast_key_valueattention_masklayer_head_maskoutput_attentionsreturncCs\|dk }|\}} } |||j} |rD|dk rD|d} |d} n|rr|||d|} |||d|} n|dk r|||d|} |||d|} tj|d| gdd} tj|d| gdd} n(|||d|} |||d|} |jr| | f}||j d|j f}|| | |j |} | j |} | j |} | d}t | | dd}|||j | |fkrtd||j | |fd||dk r||d| |fkrtd |d| |fd|| ||j | ||}| ||j | |}tjj|dd}|dk r||j fkrPtd |j fd|| dddd| ||j | |}| ||j | |}|r| ||j | |}| ||j | |}nd}tjj||j|jd }t || }|||j | |j fkrtd ||j | |j fd|| ||j | |j }| dd}||| | }||}|||fS) z#Input shape: Batch x Time x ChannelNrrrr1rz$Attention weights should be of size z , but is z!Attention mask should be of size z/Head mask for a single layer should be of size ptrainingz `attn_output` should be of size )rrjrfrprhrirr!r`r\rdrZbmmrnrer functionalZsoftmaxr_rzZreshaperk)r5rqrrrsrtrurvrbr$r%r[Z query_statesZ key_statesZ value_statesZ proj_shaper,Z attn_weightsZattn_weights_reshapedZ attn_probsZ attn_outputr'r'r(r=st        "   zTrOCRAttention.forward)NNrZFTF)NNNNF)r?r@rArBrCrPr+r4rrDrprrr=rEr'r'r6r(rYs@! rYc steZdZedfdd Zd ejeejeejeejeejeejeeejee ee d dd Z Z S) TrOCRDecoderLayerrlc st|j|_t||j|j|jdd|_|j|_t |j |_ |j |_ t |j|_|jrt||j|j|j|j|jddd|_t |j|_t |j|j|_t |j|j|_t |j|_dS)NT)r[r\r_r`)r[r\r]r^r_r`rb)r3r4 hidden_sizer[rYZdecoder_attention_headsZattention_dropout self_attnr_rZactivation_function activation_fnactivation_dropoutr LayerNormself_attn_layer_normr`Zcross_attention_hidden_size encoder_attnencoder_attn_layer_normrgZdecoder_ffn_dimfc1fc2final_layer_normr5rlr6r'r(r49s8   zTrOCRDecoderLayer.__init__NFT) rqrtencoder_hidden_statesencoder_attention_maskrucross_attn_layer_head_maskrsrv use_cachec Cs^|} |dk r|ddnd} |j|| |||d\}} } tjj||j|jd}| |}||}d}d}|dk r|} |dk r|ddnd}|j||||||d\}}}tjj||j|jd}| |}||}| |} |} || |}tjj||j |jd}| |}tjj||j|jd}| |}| |}|f}|rJ|| |f7}| rZ|| f7}|S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(batch, seq_len, embed_dim)` encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of size *(decoder_attention_heads,)*. past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. Nr1)rqrsrtrurvrx)rqrrrtrursrv) rrr{r_rzrrrrrrrr)r5rqrtrrrurrsrvrZresidualZself_attn_past_key_valueZself_attn_weightsZpresent_key_valueZcross_attn_present_key_valueZcross_attn_weightsZcross_attn_past_key_valueoutputsr'r'r(r=[sT        zTrOCRDecoderLayer.forward)NNNNNNFT) r?r@rArr4rrDrrr+r=rEr'r'r6r(r|8s(% r|c@s*eZdZeZdZdZddZd ddZdS) TrOCRPreTrainedModelmodelTcCs|jj}t|tjtjfrD|jjjd|d|j dk r~|j j n:t|tj r~|jjjd|d|j dk r~|jj|j  dS)NrZ)Zmeanstd) rlZinit_std isinstancerrgZConv1dr<dataZnormal_raZzero_ EmbeddingrH)r5modulerr'r'r( _init_weightss   z"TrOCRPreTrainedModel._init_weightsFcCst|tr||_dSN)r TrOCRDecodergradient_checkpointing)r5rvaluer'r'r(_set_gradient_checkpointings z0TrOCRPreTrainedModel._set_gradient_checkpointingN)F) r?r@rAr config_classZbase_model_prefixZsupports_gradient_checkpointingrrr'r'r'r(rs  raJ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`TrOCRConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. c sHeZdZdZedfdd ZddZddZd d Zdd d Z Z S)rz Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`TrOCRDecoderLayer`] Args: config: TrOCRConfig r}cstj|_j|_j|_jr6t j nd|_ t jj |j|_jrjtjj |_ntj|jdj |j|_jrt j |_nd|_t fddtjD|_d|_|dS)Nr*rcsg|] }tqSr')r|).0_r}r'r( sz)TrOCRDecoder.__init__..F)r3r4r_Zdecoder_layerdrop layerdropZ pad_token_idrHZscale_embeddingrMsqrtr~ embed_scalerr vocab_size embed_tokensuse_learned_position_embeddingsr.Zmax_position_embeddingsembed_positionsrFlayernorm_embeddingrZ ModuleListrangeZdecoder_layerslayersr post_initrr6r}r(r4s&  zTrOCRDecoder.__init__cCs|jSrrr5r'r'r(get_input_embeddingssz!TrOCRDecoder.get_input_embeddingscCs ||_dSrrr5rr'r'r(set_input_embeddingssz!TrOCRDecoder.set_input_embeddingscCsXd}|ddkr$t||j|j|d}|dk rTt||j|dd}|dkrL|n||}|S)Nrr)rrr%)r)rrr-)r5rt input_shape inputs_embedsrZcombined_attention_maskZexpanded_attn_maskr'r'r(_prepare_decoder_attention_mask s z,TrOCRDecoder._prepare_decoder_attention_maskNc  sdk r n|jj| dk r | n|jj} dk r4n|jj| dk rH| n|jj} |dk rj|dk rjtdnX|dk r|} |d| jd}n8|dk r|dd}|dddddf} ntd|dk r|ddjdnd}|dkr| ||j }|jj r|j | |d}n|j ||d}||}|j dk r>| |}tjj||j|jd}| j}|||||}|dk r|dk rt||j|dd }|jr|jrrtd d | rd nd}rd nd}r|dk rd nd}rd nd}t||gd dgD]V\}}|dk r |dt|jkr td|dt|jd|ddq t|jD]F\}}| r||f7}|jrtg}||jkrql|dk r||nd}|jr"|jr"fdd}tjj |||||||dk r||nd|dk r||ndd}n>||||||dk r>||nd|dk rR||nd|d }|d}r||r|dndf7}rl||df7}|dk rl||df7}ql| r||f7}r|nd}| st!dd|||||fDSt"|||||dS)a Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention on hidden heads. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timerzEYou have to specify either decoder_input_ids or decoder_inputs_embedsrr1)rrxrz^`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`...Fr' head_maskcross_attn_head_maskzThe `z` should be specified for z layers, but it is for .csfdd}|S)Ncs|fSrr')inputs)rrvrr'r(custom_forwardszKTrOCRDecoder.forward..create_custom_forward..custom_forwardr')rrrvr)rr(create_custom_forwardsz3TrOCRDecoder.forward..create_custom_forward)rtrrrurrsrvrrrcss|]}|dk r|VqdSrr')rvr'r'r( sz'TrOCRDecoder.forward..)Zlast_hidden_statepast_key_valuesrq attentionscross_attentions)#rlrvoutput_hidden_statesruse_return_dictrerr:rrrrrrrr{r_rzrr-rrloggerZ warning_onceziplenr enumeraterZrandrutils checkpointtupler )r5r9rtrrrrrrrrvr return_dictinputrrZ embed_posrqZall_hidden_statesZall_self_attnsZall_cross_attentionsZnext_decoder_cacheZ attn_maskZ mask_nameidxZ decoder_layerZdropout_probabilityrsrZ layer_outputsZ next_cacher'rr(r=sP     $        zTrOCRDecoder.forward) NNNNNNNNNNNN) r?r@rArBrr4rrrr=rEr'r'r6r(rs$rzMThe TrOCR Model with a language modeling head. Can be used for summarization.cs(eZdZdZfddZddZZS)TrOCRDecoderWrapperz This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the [`EncoderDecoderModel`] framework. cst|t||_dSr)r3r4rdecoderrr6r'r(r4 s zTrOCRDecoderWrapper.__init__cOs |j||Sr)r)r5argskwargsr'r'r(r= szTrOCRDecoderWrapper.forward)r?r@rArBr4r=rEr'r'r6r(rs rzThe TrOCR Decoder with a language modeling head. Can be used as the decoder part of [`EncoderDecoderModel`] and [`VisionEncoderDecoder`].cseZdZdgZfddZddZddZdd Zd d Zd d Z ddZ e e e ddeejeejeejeejeejeejeeeejeejeejeeeeeeeeeee fdddZdddZeddZZS)TrOCRForCausalLMzoutput_projection.weightcsNt|}d|_d|_t|t||_tj |j |j dd|_ | dS)NTFrc)copydeepcopyr`Zis_encoder_decoderr3r4rrrrgr~routput_projectionrrr6r'r(r4s   zTrOCRForCausalLM.__init__cCs |jjjSrrrrrr'r'r(r%sz%TrOCRForCausalLM.get_input_embeddingscCs||jj_dSrrrr'r'r(r(sz%TrOCRForCausalLM.set_input_embeddingscCs|jSrrrr'r'r(get_output_embeddings+sz&TrOCRForCausalLM.get_output_embeddingscCs ||_dSrr)r5Znew_embeddingsr'r'r(set_output_embeddings.sz&TrOCRForCausalLM.set_output_embeddingscCs ||j_dSrrr)r5rr'r'r( set_decoder1szTrOCRForCausalLM.set_decodercCs|jjSrrrr'r'r( get_decoder4szTrOCRForCausalLM.get_decoder) output_typerN)r9rtrrrrrrlabelsrrvrrrwcCs| dk r | n|jj} | dk r | n|jj} | dk r4| n|jj} |jj||||||||| | | | d }||d}d}| dk rt}||d|jj | d}| s|f|dd}|dk r|f|S|St |||j |j |j |jdS)aO Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional tensors are only required when the model is used as a decoder in a Sequence to Sequence model. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Returns: Example: ```python >>> from transformers import ( ... TrOCRConfig, ... TrOCRProcessor, ... TrOCRForCausalLM, ... ViTConfig, ... ViTModel, ... VisionEncoderDecoderModel, ... ) >>> import requests >>> from PIL import Image >>> # TrOCR is a decoder model and should be used within a VisionEncoderDecoderModel >>> # init vision2text model with random weights >>> encoder = ViTModel(ViTConfig()) >>> decoder = TrOCRForCausalLM(TrOCRConfig()) >>> model = VisionEncoderDecoderModel(encoder=encoder, decoder=decoder) >>> # If you want to start from the pretrained model, load the checkpoint with `VisionEncoderDecoderModel` >>> processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") >>> model = VisionEncoderDecoderModel.from_pretrained("microsoft/trocr-base-handwritten") >>> # load image from the IAM dataset >>> url = "https://fki.tic.heia-fr.ch/static/img/a01-122-02.jpg" >>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB") >>> pixel_values = processor(image, return_tensors="pt").pixel_values >>> text = "industry, ' Mr. Brown commented icily. ' Let us have a" >>> # training >>> model.config.decoder_start_token_id = processor.tokenizer.cls_token_id >>> model.config.pad_token_id = processor.tokenizer.pad_token_id >>> model.config.vocab_size = model.config.decoder.vocab_size >>> labels = processor.tokenizer(text, return_tensors="pt").input_ids >>> outputs = model(pixel_values, labels=labels) >>> loss = outputs.loss >>> round(loss.item(), 2) 5.30 >>> # inference >>> generated_ids = model.generate(pixel_values) >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] >>> generated_text 'industry, " Mr. Brown commented icily. " Let us have a' ```N) r9rtrrrrrrrrvrrrrr)losslogitsrrqrr)rlrvrrrrrrrrr rrqrr)r5r9rtrrrrrrrrrvrrrrrZloss_fctoutputr'r'r(r=7sFzTrOCRForCausalLM.forwardcKs:|dkr||j}|r,|ddddf}||||dS)Nr)r9rtrr)Znew_onesr:)r5r9rrtrrr'r'r(prepare_inputs_for_generations z.TrOCRForCausalLM.prepare_inputs_for_generationcs.d}|D] }|tfdd|Df7}q|S)Nr'c3s"|]}|d|jVqdS)rN)rTr r)rZ past_statebeam_idxr'r(rsz2TrOCRForCausalLM._reorder_cache..)r)rrZreordered_pastZ layer_pastr'rr(_reorder_caches zTrOCRForCausalLM._reorder_cache) NNNNNNNNNNNNN)NNN)r?r@rAZ_tied_weights_keysr4rrrrrrrr _CONFIG_FOR_DOCrrZ LongTensorrDrLrr+rr=rrXrrEr'r'r6r(rsZ   0 r)r)N)0rBrrMtypingrrrrrZtorch.nnrZ activationsrZmodeling_outputsr r Zmodeling_utilsr rr r rZconfiguration_trocrrZ get_loggerr?rrZ_CHECKPOINT_FOR_DOCZ#TROCR_PRETRAINED_MODEL_ARCHIVE_LISTSizerrrCr)rDr-rr.ModulerFrYr|rZTROCR_START_DOCSTRINGrrrr'r'r'r(sZ       >"