U
    9%e                     @   sn  d Z ddlZddlmZmZmZmZ ddlZddlZddlm	Z	 ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ eeZdZdZdgZdZ dZ!d%ej"ej#ej$e%dddZ&d&ej'ej#ee% dddZ(G dd de	j)Z*G dd de	j)Z+G dd de	j)Z,G dd deZ-ede G d d! d!e-Z.ed"e G d#d$ d$e-Z/dS )'z PyTorch XGLM model.    N)ListOptionalTupleUnion)nn)CrossEntropyLoss   )ACT2FN))BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentions)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging   )
XGLMConfigzfacebook/xglm-564Mr   aI  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`XGLMConfig`]):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of
            the decoder.
        encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
            Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
            selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        head_mask (`torch.Tensor` of shape `(num_layers, attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, attention_heads)`, *optional*):
            Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape
            `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you
            can choose to directly pass an embedded representation. This is useful if you want more control over how to
            convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
)input_ids_shapedtypedevicepast_key_values_lengthc                 C   s   | \}}t j||ft |j|d}t j|d|d}|||d |ddk d ||}|dkrt j	t j
||||d|gdd}|ddddddf |d||| S )zB
    Make causal mask used for bi-directional self-attention.
    r   r   r   r   r   dimN)torchfullfinfominarangesizemasked_fill_viewtocatzerosexpand)r   r   r   r   bsztgt_lenmaskZ	mask_cond r+   e/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/models/xglm/modeling_xglm.py_make_causal_mask   s    "
 r-   )r*   r   r)   c                 C   sj   |   \}}|dk	r|n|}| ddddddf |d|||}d| }||tjt|jS )z_
    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
    Nr         ?)r!   r'   r$   Zmasked_fillr   boolr   r   )r*   r   r)   r(   src_lenZexpanded_maskZinverted_maskr+   r+   r,   _expand_mask   s
    *r1   c                       s   e Zd ZdZdeeee d fddZdeeee dddZedeeee dd	d
Z	e
 de
jedddZ  ZS )!XGLMSinusoidalPositionalEmbeddingzDThis module produces sinusoidal positional embeddings of any length.N)num_positionsembedding_dimpadding_idxc                    s4   t    d| _|| _|| _| || j || d S )N   )super__init__offsetr4   r5   make_weights)selfr3   r4   r5   	__class__r+   r,   r8      s
    
z*XGLMSinusoidalPositionalEmbedding.__init__)num_embeddingsr4   r5   c                 C   sB   |  |||}t| dr.|j| jj| jjd}| jd|dd d S )Nweightsr   F)
persistent)get_embeddinghasattrr$   r?   r   r   Zregister_buffer)r;   r>   r4   r5   Zemb_weightsr+   r+   r,   r:      s    
z.XGLMSinusoidalPositionalEmbedding.make_weightsc                 C   s   |d }t d|d  }ttj|tjd|  }tj| tjdd|d }tjt|t	|gdd
| d}|d dkrtj|t| dgdd}|dk	rd||ddf< |t S )	z
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        r6   i'  r   )r   r   r   r   N)mathlogr   expr    float	unsqueezer%   sincosr#   r&   r$   Zget_default_dtype)r>   r4   r5   Zhalf_dimZembr+   r+   r,   rA      s     &z/XGLMSinusoidalPositionalEmbedding.get_embeddingr   )position_idsr   c                 C   sn   |  \}}|| j7 }d| | }|| j dkrD| || j| j | jd|d||| jjd 	 S )Nr6   r   r   )
r!   r9   r?   r:   r4   r5   index_selectr#   shapedetach)r;   rJ   r   r(   seq_lenZmax_posr+   r+   r,   forward   s    
z)XGLMSinusoidalPositionalEmbedding.forward)N)N)N)Nr   )__name__
__module____qualname____doc__intr   r8   r:   staticmethodrA   r   Zno_gradTensorrO   __classcell__r+   r+   r<   r,   r2      s   r2   c                       s   e Zd ZdZdeeeeed fddZej	eedd	d
Z
dej	eej	 eeej	  eej	 eej	 eeej	eej	 eeej	  f dddZ  ZS )XGLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paper        FT)	embed_dim	num_headsdropout
is_decoderbiasc                    s   t    || _|| _|| _|| | _| j| | jkrNtd| j d| d| jd | _|| _t	j
|||d| _t	j
|||d| _t	j
|||d| _t	j
|||d| _d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      ࿩r^   )r7   r8   rZ   r[   r\   head_dim
ValueErrorscalingr]   r   Lineark_projv_projq_projout_proj)r;   rZ   r[   r\   r]   r^   r<   r+   r,   r8      s    

zXGLMAttention.__init__)tensorrN   r(   c                 C   s    | ||| j| jdd S )Nr   r6   )r#   r[   r`   	transpose
contiguous)r;   rh   rN   r(   r+   r+   r,   _shape   s    zXGLMAttention._shapeN)hidden_stateskey_value_statespast_key_valueattention_masklayer_head_maskoutput_attentionsreturnc                 C   s  |dk	}|  \}}	}
| || j }|rD|dk	rD|d }|d }n|rr| | |d|}| | |d|}n|dk	r| | |d|}| | |d|}tj|d |gdd}tj|d |gdd}n(| | |d|}| | |d|}| jr||f}|| j	 d| j
f}| ||	|j| }|j| }|j| }| d}t||dd}|  || j	 |	|fkrtd|| j	 |	|f d|   |dk	r*|  |d|	|fkrtd	|d|	|f d|   ||| j	|	|| }t|tjt|jj|jd
}||| j	 |	|}|jtjkrVtjj|dtjdtj}ntjj|dd}|dk	r|  | j	fkrtd| j	f d|   |dddd||| j	|	| }||| j	 |	|}|r||| j	|	|}||| j	 |	|}nd}tjj|| j| jd}t||}|  || j	 |	| j
fkrhtd|| j	|	| j
f d|   ||| j	|	| j
}|dd}|||	| j}| |}|||fS )z#Input shape: Batch x Time x ChannelNr   r   r   r6   r   z$Attention weights should be of size z	, but is z!Attention mask should be of size r   )r   r   z/Head mask for a single layer should be of size ptrainingz `attn_output` should be of size ) r!   rf   rb   rk   rd   re   r   r%   r]   r[   r`   r#   Zbmmri   ra   maxrh   r   r   r   r   Zfloat16r   
functionalZsoftmaxZfloat32r$   r\   ru   ZreshaperZ   rg   )r;   rl   rm   rn   ro   rp   rq   Zis_cross_attentionr(   r)   _Zquery_statesZ
key_statesZvalue_statesZ
proj_shaper0   Zattn_weightsZattn_weights_reshapedZ
attn_probsZattn_outputr+   r+   r,   rO      s    




 
"
zXGLMAttention.forward)rY   FT)NNNNF)rP   rQ   rR   rS   rT   rF   r/   r8   r   rV   rk   r   r   rO   rW   r+   r+   r<   r,   rX      s4           rX   c                       sx   e Zd Zed fddZd
ejeej eej eej eej eej eeej  ee	 ee	 ejd
dd	Z
  ZS )XGLMDecoderLayerconfigc                    s   t    |j| _t| j|j|jdd| _|j| _t	|j
 | _|j| _|jrrt| j|j|jdd| _t| j| _t| j| _t| j|j| _t|j| j| _t| j| _d S )NT)rZ   r[   r\   r]   )r7   r8   d_modelrZ   rX   Zattention_headsZattention_dropout	self_attnr\   r	   Zactivation_functionactivation_fnactivation_dropoutZadd_cross_attentionencoder_attnr   	LayerNormencoder_attn_layer_normself_attn_layer_normrc   Zffn_dimfc1fc2final_layer_normr;   r{   r<   r+   r,   r8   r  s.    
zXGLMDecoderLayer.__init__NFT)
rl   ro   encoder_hidden_statesencoder_attention_maskrp   cross_attn_layer_head_maskrn   rq   	use_cacherr   c
                 C   s^  |}
|  |}|dk	r"|dd nd}| j|||||d\}}}tjj|| j| jd}|
| }d}d}|dk	r|}
| |}|dk	r|dd nd}| j||||||d\}}}tjj|| j| jd}|
| }|| }|}
| |}| 	| 
|}tjj|| j| jd}| |}tjj|| j| jd}|
| }|f}|rJ|||f7 }|	rZ||f7 }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            encoder_hidden_states (`torch.FloatTensor`):
                cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
            encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
                `(encoder_attention_heads,)`.
            cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of
                size `(decoder_attention_heads,)`.
            past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        Nr6   )rl   rn   ro   rp   rq   rs   )rl   rm   ro   rp   rn   rq   )r   r}   r   rw   r\   ru   r   r   r   r~   r   r   r   )r;   rl   ro   r   r   rp   r   rn   rq   r   ZresidualZself_attn_past_key_valueZself_attn_weightsZpresent_key_valueZcross_attn_present_key_valueZcross_attn_weightsZcross_attn_past_key_valueoutputsr+   r+   r,   rO     sT    




zXGLMDecoderLayer.forward)NNNNNNFT)rP   rQ   rR   r   r8   r   rV   r   r   r/   rO   rW   r+   r+   r<   r,   ry   q  s*            ry   c                   @   s0   e Zd ZeZdZdZdgZdd Zd
ddZ	d	S )XGLMPreTrainedModelmodelTry   c                 C   s|   | j j}t|tjr>|jjjd|d |jd k	rx|jj	  n:t|tj
rx|jjjd|d |jd k	rx|jj|j 	  d S )NrY   )Zmeanstd)r{   Zinit_std
isinstancer   rc   weightdataZnormal_r^   Zzero_	Embeddingr5   )r;   moduler   r+   r+   r,   _init_weights  s    

z!XGLMPreTrainedModel._init_weightsFc                 C   s   t |tr||_d S N)r   	XGLMModelgradient_checkpointing)r;   r   valuer+   r+   r,   _set_gradient_checkpointing  s    
z/XGLMPreTrainedModel._set_gradient_checkpointingN)F)
rP   rQ   rR   r   config_classbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesr   r   r+   r+   r+   r,   r     s   r   z^The bare XGLM Model transformer outputting raw hidden-states without any specific head on top.c                       s   e Zd ZdZdeeej d fddZdd Z	dd	 Z
d
d Zeeeeeeddeej eej eej eej eej eej eej eeej  eej ee ee ee ee eeej ef dddZ  ZS )r   z
    Transformer decoder consisting of *config.num_layers* layers. Each layer is a [`XGLMDecoderLayer`]

    Args:
        config: XGLMConfig
        embed_tokens (nn.Embedding): output embedding
    N)r{   embed_tokensc                    s   t     j| _ j| _ j| _ j| _ jr>t	
 jnd| _|d k	rT|| _nt j j| j| _t j j j| _t fddt jD | _t j| _d| _|   d S )Nr.   c                    s   g | ]}t  qS r+   )ry   ).0rx   rz   r+   r,   
<listcomp>  s     z&XGLMModel.__init__.<locals>.<listcomp>F)r7   r8   r\   	layerdroppad_token_idr5   Zmax_position_embeddingsZmax_target_positionsZscale_embeddingrC   sqrtr|   embed_scaler   r   r   
vocab_sizer2   embed_positionsZ
ModuleListrangeZ
num_layerslayersr   
layer_normr   	post_init)r;   r{   r   r<   rz   r,   r8     s$     zXGLMModel.__init__c                 C   s   | j S r   r   r;   r+   r+   r,   get_input_embeddings%  s    zXGLMModel.get_input_embeddingsc                 C   s
   || _ d S r   r   r;   r   r+   r+   r,   set_input_embeddings(  s    zXGLMModel.set_input_embeddingsc                 C   sX   d }|d dkr$t ||j|j|d}|d k	rTt||j|d d}|d krL|n|| }|S )Nr   r   )r   r   r)   )r-   r   r   r1   )r;   ro   input_shapeinputs_embedsr   Zcombined_attention_maskZexpanded_attn_maskr+   r+   r,   _prepare_decoder_attention_mask+  s    z)XGLMModel._prepare_decoder_attention_mask
checkpointoutput_typer   )	input_idsro   rJ   r   r   	head_maskcross_attn_head_maskpast_key_valuesr   r   rq   output_hidden_statesreturn_dictrr   c                    s4   d k	r n| j j |d k	r |n| j j}d k	r4n| j j|d k	rH|n| j j}|d k	rj|	d k	rjtdnP|d k	r| || | }|d|d }n"|	d k	r|	 d d }ntd|d k	r|d d j	d nd}|d kr(t
j||d | t
j|d k	r|jn|	jd}|dd|d }n|d|d }|	d krR| || j }	| |||	|}|d k	r|d k	rt||	j|d d}|	| || }tjj|t| j| jd}| jr| jr؈rtd	 d
|rdnd } rdnd } r|d k	rdnd }rdnd }t||gddgD ]V\}}|d k	r,| d t| jkr,td| dt| j d| d  dq,t | jD ]F\}}|r||f7 }| jrt
!g }|| j"k rʐq|d k	r|| nd }| jrD| jrD fdd}t
j#j$$|||||||d k	r&|| nd |d k	r:|| nd d }n>||||||d k	r`|| nd |d k	rt|| nd | d	}|d }r|| rdnd f7 } r||d f7 }|d k	r||d f7 }q| %|}|r||f7 }r|nd }|s"t&dd |||||fD S t'|||||dS )NzDYou cannot specify both input_ids and inputs_embeds at the same timer   z5You have to specify either input_ids or inputs_embedsr   r6   r   r   rs   z_`use_cache = True` is incompatible with gradient checkpointing`. Setting `use_cache = False`...Fr+   r   r   zThe `z` should be specified for z layers, but it is for .c                    s    fdd}|S )Nc                     s    | f S r   r+   )inputs)r   rq   r   r+   r,   custom_forward  s    zHXGLMModel.forward.<locals>.create_custom_forward.<locals>.custom_forwardr+   )r   r   rq   r   )r   r,   create_custom_forward  s    z0XGLMModel.forward.<locals>.create_custom_forward)ro   r   r   rp   r   rn   rq   r   r   r   c                 s   s   | ]}|d k	r|V  qd S r   r+   )r   vr+   r+   r,   	<genexpr>  s   z$XGLMModel.forward.<locals>.<genexpr>)Zlast_hidden_stater   rl   
attentionscross_attentions)(r{   rq   r   r   use_return_dictra   Z%warn_if_padding_and_no_attention_maskr!   r#   rL   r   r    longr   rG   r   r   r   r1   r   r   r   rw   r\   rF   ru   r   loggerZwarning_onceziplenr   	enumerateZrandr   utilsr   r   tupler
   )r;   r   ro   rJ   r   r   r   r   r   r   r   rq   r   r   r   r   rl   Zall_hidden_statesZall_self_attnsZall_cross_attentionsZnext_decoder_cacheZ	attn_maskZ	mask_nameidxZdecoder_layerZdropout_probabilityrn   r   Zlayer_outputsZ
next_cacher+   r   r,   rO   @  s    



   
$




zXGLMModel.forward)N)NNNNNNNNNNNNN)rP   rQ   rR   rS   r   r   r   r   r8   r   r   r   r   XGLM_INPUTS_DOCSTRINGr   _CHECKPOINT_FOR_DOCr
   _CONFIG_FOR_DOCr   rV   r   FloatTensorr/   r   r   rO   rW   r+   r+   r<   r,   r     sP                r   z
    The XGLM Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                       s   e Zd ZdZdgZ fddZdd Zdd Zd	d
 Zdd Z	e
eeeeeddeej eej eej eej eej eej eej eeej  eej eej ee ee ee ee eeej ef dddZdddZedd Z  ZS )XGLMForCausalLMr   zlm_head.weightc                    s8   t  | t|| _tj|j|jdd| _| 	  d S )NFr_   )
r7   r8   r   r   r   rc   Zhidden_sizer   lm_headr   r   r<   r+   r,   r8     s    
zXGLMForCausalLM.__init__c                 C   s   | j jS r   r   r   r   r+   r+   r,   r     s    z$XGLMForCausalLM.get_input_embeddingsc                 C   s   || j _d S r   r   r   r+   r+   r,   r     s    z$XGLMForCausalLM.set_input_embeddingsc                 C   s   | j S r   r   r   r+   r+   r,   get_output_embeddings  s    z%XGLMForCausalLM.get_output_embeddingsc                 C   s
   || _ d S r   r   )r;   Znew_embeddingsr+   r+   r,   set_output_embeddings  s    z%XGLMForCausalLM.set_output_embeddingsr   N)r   ro   rJ   r   r   r   r   r   r   labelsr   rq   r   r   rr   c                 C   s0  |dk	r|n| j j}|dk	r |n| j j}|dk	r4|n| j j}| j|||||||||	||||d}| |d }d}|
dk	r|
|
j}|
ddddf  |ddddf< | j j	|dddf< t
 }||d| j j|d}|s|f|dd  }|dk	r|f| S |S t|||j|j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)r   ro   rJ   r   r   r   r   r   r   r   rq   r   r   r   r   r   )losslogitsr   rl   r   r   )r{   rq   r   r   r   r   Z	new_zerosrL   cloner   r   r#   r   r   r   rl   r   r   )r;   r   ro   rJ   r   r   r   r   r   r   r   r   rq   r   r   r   r   r   Zshift_labelsZloss_fctoutputr+   r+   r,   rO     sL    (zXGLMForCausalLM.forwardc                 K   s   | dd }|d k	rZ|d krZ| dd }||dkd |rr|d d df d}nd }|d krr||j}|r|d d dd f }|||||dS )NrJ   r   r   r   )r   ro   rJ   r   r   )getr   Zcumsumr"   rG   Znew_onesrL   )r;   r   r   ro   r   kwargsrJ   r+   r+   r,   prepare_inputs_for_generationU  s"    z-XGLMForCausalLM.prepare_inputs_for_generationc                    s.   d}| D ] }|t  fdd|D f7 }q|S )Nr+   c                 3   s"   | ]}| d  |jV  qdS )r   N)rK   r$   r   )r   Z
past_statebeam_idxr+   r,   r   u  s     z1XGLMForCausalLM._reorder_cache.<locals>.<genexpr>)r   )r   r   Zreordered_pastZ
layer_pastr+   r   r,   _reorder_cachep  s    zXGLMForCausalLM._reorder_cache)NNNNNNNNNNNNNN)NNN)rP   rQ   rR   r   Z_tied_weights_keysr8   r   r   r   r   r   r   r   r   r   r   r   r   rV   r   r   r/   r   r   rO   r   rU   r   rW   r+   r+   r<   r,   r     sd                 I     
r   )r   )N)0rS   rC   typingr   r   r   r   r   Ztorch.utils.checkpointr   Ztorch.nnr   Zactivationsr	   Zmodeling_outputsr
   r   Zmodeling_utilsr   r   r   r   r   r   Zconfiguration_xglmr   Z
get_loggerrP   r   r   r   Z"XGLM_PRETRAINED_MODEL_ARCHIVE_LISTZXGLM_START_DOCSTRINGr   Sizer   r   rT   r-   rV   r1   Moduler2   rX   ry   r   r   r   r+   r+   r+   r,   <module>   sT   
H    4 x f