U
    ,-e                     @   s  d Z ddlZddlmZmZmZmZ ddlZddlZddlm	Z	 ddl
mZmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ eeZdZ d0ej!ej"ej#e$dddZ%d1ej&ej"ee$ dddZ'G dd de	j(Z)G dd de)Z*G dd de)Z+dd Z,dd Z-G dd de	j(Z.G d d! d!e	j(Z/G d"d# d#e	j(Z0d$Z1ed%e1G d&d' d'eZ2d(Z3ed%e1G d)d* d*e2Z4G d+d, d,e2Z5ed-e1G d.d/ d/e2Z6dS )2z PyTorch Persimmon model.    N)ListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)BaseModelOutputWithPastCausalLMOutputWithPast SequenceClassifierOutputWithPast)PreTrainedModel)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )PersimmonConfigr   )input_ids_shapedtypedevicepast_key_values_lengthc                 C   s   | \}}t j||ft |j|d}t j|d|d}|||d |ddk d ||}|dkrt j	t j
||||d|gdd}|ddddddf |d||| S )zB
    Make causal mask used for bi-directional self-attention.
    r   r   r   r   r   dimN)torchfullfinfominarangesizemasked_fill_viewtocatZzerosexpand)r   r   r   r   bsztgt_lenmaskZ	mask_cond r-   q/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/persimmon/modeling_persimmon.py_make_causal_mask*   s    "
 r/   )r,   r   r+   c                 C   sj   |   \}}|dk	r|n|}| ddddddf |d|||}d| }||tjt|jS )z_
    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
    Nr         ?)r$   r)   r'   Zmasked_fillr   boolr!   r"   )r,   r   r+   r*   Zsrc_lenZexpanded_maskZinverted_maskr-   r-   r.   _expand_mask<   s
    *r2   c                       s0   e Zd Zd
 fdd	Zdd Zddd	Z  ZS )PersimmonRotaryEmbedding   '  Nc                    sr   t    || _|| _|| _d| jtd| jd || j   }| j	d|dd | j
|| jjt d d S )Nr0   r      inv_freqF
persistentseq_lenr   r   )super__init__r   max_position_embeddingsbaser   r#   floatr'   register_buffer_set_cos_sin_cacher7   r   Zget_default_dtype)selfr   r>   r?   r   r7   	__class__r-   r.   r=   L   s    
*  z!PersimmonRotaryEmbedding.__init__c                 C   s   || _ tj| j || jjd}td|| j}tj||fdd}| jd| d d d d d d f 	|dd | jd|
 d d d d d d f 	|dd d S 	Nr   r   i,j->ijr   r   
cos_cachedFr8   
sin_cached)max_seq_len_cachedr   r#   r7   r   einsumr(   rA   cosr'   sinrC   r;   r   r   tfreqsembr-   r-   r.   rB   Z   s    .z+PersimmonRotaryEmbedding._set_cos_sin_cachec                 C   sn   || j kr| j||j|jd | jd d d d d |df j|jd| jd d d d d |df j|jdfS )Nr:   .)r   )rK   rB   r   r   rI   r'   rJ   )rC   xr;   r-   r-   r.   forwardd   s
    
&&z PersimmonRotaryEmbedding.forward)r4   r5   N)N)__name__
__module____qualname__r=   rB   rT   __classcell__r-   r-   rD   r.   r3   K   s   
r3   c                       s*   e Zd ZdZd
 fdd	Zdd	 Z  ZS )%PersimmonLinearScalingRotaryEmbeddingz_PersimmonRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendevr4   r5   Nr0   c                    s   || _ t |||| d S Nscaling_factorr<   r=   rC   r   r>   r?   r   r\   rD   r-   r.   r=   s   s    z.PersimmonLinearScalingRotaryEmbedding.__init__c                 C   s   || _ tj| j || jjd}|| j }td|| j}tj||fdd}| jd|	 d d d d d d f 
|dd | jd| d d d d d d f 
|dd d S rF   )rK   r   r#   r7   r   r\   rL   r(   rA   rM   r'   rN   rO   r-   r-   r.   rB   w   s    
.z8PersimmonLinearScalingRotaryEmbedding._set_cos_sin_cache)r4   r5   Nr0   rU   rV   rW   __doc__r=   rB   rX   r-   r-   rD   r.   rY   p   s   rY   c                       s*   e Zd ZdZd
 fdd	Zdd	 Z  ZS ))PersimmonDynamicNTKScalingRotaryEmbeddingzqPersimmonRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozillar4   r5   Nr0   c                    s   || _ t |||| d S rZ   r[   r]   rD   r-   r.   r=      s    z2PersimmonDynamicNTKScalingRotaryEmbedding.__init__c           	      C   s  || _ || jkrx| j| j| | j | jd  | j| jd    }d|td| jd || j   }| j	d|dd tj| j || j
jd}td	|| j
}tj||fd
d}| j	d| d d d d d d f |dd | j	d| d d d d d d f |dd d S )Nr   r6   r0   r   r7   Fr8   rG   rH   r   r   rI   rJ   )rK   r>   r?   r\   r   r   r#   r@   r'   rA   r7   r   rL   r(   rM   rN   )	rC   r;   r   r   r?   r7   rP   rQ   rR   r-   r-   r.   rB      s    
(.z<PersimmonDynamicNTKScalingRotaryEmbedding._set_cos_sin_cache)r4   r5   Nr0   r^   r-   r-   rD   r.   r`      s   r`   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nr   r6   r   )shaper   r(   )rS   x1Zx2r-   r-   r.   rotate_half   s    rc   c                 C   sl   | d d}| d d}|| d}|| d}| | t| |  }|| t||  }||fS )Nr   r   )squeeze	unsqueezerc   )qkrM   rN   position_idsZq_embedZk_embedr-   r-   r.   apply_rotary_pos_emb   s    ri   c                       s$   e Zd Z fddZdd Z  ZS )PersimmonMLPc                    s>   t    t|j|j| _t|j|j| _t|j	 | _
d S rZ   )r<   r=   r   Linearhidden_sizeZintermediate_sizedense_h_to_4hdense_4h_to_hr   Z
hidden_actactrC   configrD   r-   r.   r=      s    
zPersimmonMLP.__init__c                 C   s"   |  |}| |}| |}|S rZ   )rm   ro   rn   )rC   hidden_statesr-   r-   r.   rT      s    


zPersimmonMLP.forward)rU   rV   rW   r=   rT   rX   r-   r-   rD   r.   rj      s   rj   c                       s   e Zd ZdZed fddZdd Zeje	ejejejf ddd	Z
dejeej eej ee	ej  eee	ejeej ee	ej  f dddZ  ZS )PersimmonAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrq   c                    s  t    || _|j| _|j| _| j| j | _|j| _|j| _|j	| _	| j| j | jkrrt
d| j d| j dtj| jd| j dd| _tj| j| j | jdd| _|j| _| jrtj|j| j |jdd| _tj|j| j |jdd| _t|j| _|   d S )Nz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r
   Tbias)epsZelementwise_affine)r<   r=   rq   rl   Znum_attention_heads	num_headshead_dimr>   
rope_thetapartial_rotary_factor
ValueErrorr   rk   query_key_valuedenseqk_layernorm	LayerNormlayer_norm_epsq_layernormk_layernormDropoutattention_dropout
_init_roperp   rD   r-   r.   r=      s8    

  
  zPersimmonAttention.__init__c                 C   s   | j jd kr.tt| j| j | j| jd| _n~| j jd }| j jd }|dkrrt	t| j| j | j|| jd| _n:|dkrt
t| j| j | j|| jd| _ntd| d S )N)r>   r?   typefactorZlinear)r>   r\   r?   ZdynamiczUnknown RoPE scaling type )rq   Zrope_scalingr3   intr{   ry   r>   rz   
rotary_embrY   r`   r|   )rC   Zscaling_typer\   r-   r-   r.   r      s.    


zPersimmonAttention._init_rope)	fused_qkvreturnc                 C   sV   |j \}}}|||| jd| j}|ddddf |ddddf |ddddf fS )a  
        Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory
        storage as `fused_qkv`

        Args:
            fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim]

        Returns:
            query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim]
            value: [batch_size, seq_length, num_heads, head_dim]
        r
   .r   Nr   r6   )ra   r&   rx   ry   )rC   r   
batch_size
seq_lengthZthree_times_hidden_sizer-   r-   r.   _split_heads   s    zPersimmonAttention._split_headsNFrr   attention_maskrh   past_key_valueoutput_attentions	use_cacher   c                 C   s  |  \}}}	| |}
| |
\}}}| jrB| |}| |}|dd}|dd}|dd}|jd }|d k	r||d jd 7 }| j||d\}}|dd | jj	f |d| jj	d f  }}|dd | jj	f |d| jj	d f  }}t
|||||\}}tj||fdd}tj||fdd}|d k	r^tj|d |gdd}tj|d |gdd}|rl||fnd }t||dd	t| j }|  || j||fkrtd
|| j||f d|   |d k	r|  |d||fkr
td|d||f d|   || }tjj|tjdd|j}| |}t||}|  || j|| jfkrtd|| j|| jf d|   |dd }|||| j}| |}|sd }|||fS )Nr   r6   r   )r;   .r   r   r
   z$Attention weights should be of size z	, but is z!Attention mask should be of size )r   r   z `attn_output` should be of size )r$   r}   r   r   r   r   Z	transposera   r   r   ri   r   r(   matmulmathsqrtry   rx   r|   r   Z
functionalZsoftmaxZfloat32r'   r   r   
contiguousZreshaperl   r~   )rC   rr   r   rh   r   r   r   r*   Zq_len_r   Zquery_statesZ
key_statesZvalue_statesZ
kv_seq_lenrM   rN   Z	query_rotZ
query_passZkey_rotZkey_passZattn_weightsZattn_outputr-   r-   r.   rT     sd    	




 


zPersimmonAttention.forward)NNNFF)rU   rV   rW   r_   r   r=   r   r   Tensorr   r   r   
LongTensorr1   rT   rX   r-   r-   rD   r.   rs      s$   "     rs   c                       sx   e Zd Zed fddZd	ejeej eej ee	ej  ee
 ee
 e	ejee	ejejf  f dddZ  ZS )
PersimmonDecoderLayerrt   c                    sb   t    |j| _t|d| _t|| _tj|j|j	d| _
tj|j|j	d| _t|j| _d S )Nrt   rw   )r<   r=   rl   rs   	self_attnrj   mlpr   r   r   input_layernormpost_attention_layernormr   Zhidden_dropoutdropoutrp   rD   r-   r.   r=   j  s    

zPersimmonDecoderLayer.__init__NFr   c                 C   s   |}|  |}| j||||||d\}}}	|| }|}| |}| |}| |}|| }|f}
|rp|
|f7 }
|r~|
|	f7 }
|
S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
                Indices of positions of each input sequence tokens in the position embeddings. Selected in the range
                `[0, config.n_positions - 1]`.

                [What are position IDs?](../glossary#position-ids)
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*):
                cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        )rr   r   rh   r   r   r   )r   r   r   r   r   )rC   rr   r   rh   r   r   r   ZresidualZself_attn_weightsZpresent_key_valueoutputsr-   r-   r.   rT   s  s,    





zPersimmonDecoderLayer.forward)NNNFF)rU   rV   rW   r   r=   r   r   r   r   r   r1   FloatTensorrT   rX   r-   r-   rD   r.   r   i  s        r   aN  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`PersimmonConfig`]):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
zWThe bare Persimmon Model outputting raw hidden-states without any specific head on top.c                   @   s4   e Zd ZeZdZdZdgZdZdd Z	ddd	Z
d
S )PersimmonPreTrainedModelmodelTr   past_key_valuesc                 C   s|   | j j}t|tjr>|jjjd|d |jd k	rx|jj	  n:t|tj
rx|jjjd|d |jd k	rx|jj|j 	  d S )Ng        )Zmeanstd)rq   Zinitializer_range
isinstancer   rk   weightdataZnormal_rv   Zzero_	Embeddingpadding_idx)rC   moduler   r-   r-   r.   _init_weights  s    

z&PersimmonPreTrainedModel._init_weightsFc                 C   s   t |tr||_d S rZ   )r   PersimmonModelgradient_checkpointing)rC   r   valuer-   r-   r.   _set_gradient_checkpointing  s    
z4PersimmonPreTrainedModel._set_gradient_checkpointingN)F)rU   rV   rW   r   config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ_skip_keys_device_placementr   r   r-   r-   r-   r.   r     s   r   aV  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
            information on the default strategy.

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`.

            [What are position IDs?](../glossary#position-ids)
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                       s   e Zd ZdZed fddZdd Zdd Zd	d
 Ze	e
dejeej eej eeej  eej ee ee ee ee eeef d
ddZ  ZS )r   z
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`PersimmonDecoderLayer`]

    Args:
        config: PersimmonConfig
    rt   c                    sx   t     j| _ j| _t j j| j| _t	 fddt
 jD | _tj j jd| _d| _|   d S )Nc                    s   g | ]}t  qS r-   )r   ).0r   rt   r-   r.   
<listcomp>1  s     z+PersimmonModel.__init__.<locals>.<listcomp>r   F)r<   r=   pad_token_idr   
vocab_sizer   r   rl   embed_tokensZ
ModuleListrangeZnum_hidden_layerslayersr   r   final_layernormr   	post_initrp   rD   rt   r.   r=   +  s     zPersimmonModel.__init__c                 C   s   | j S rZ   r   rC   r-   r-   r.   get_input_embeddings8  s    z#PersimmonModel.get_input_embeddingsc                 C   s
   || _ d S rZ   r   rC   r   r-   r-   r.   set_input_embeddings;  s    z#PersimmonModel.set_input_embeddingsc                 C   s`   d }|d dkr$t ||j|j|d}|d k	r\t||j|d d|j}|d krT|n|| }|S )Nr   r   )r   r   )r+   )r/   r   r   r2   r'   )rC   r   Zinput_shapeinputs_embedsr   Zcombined_attention_maskZexpanded_attn_maskr-   r-   r.   _prepare_decoder_attention_mask?  s    z.PersimmonModel._prepare_decoder_attention_maskN)
	input_idsr   rh   r   r   r   r   output_hidden_statesreturn_dictr   c
              	      s   d k	r n| j j |d k	r |n| j j}|d k	r4|n| j j}|	d k	rH|	n| j j}	|d k	rj|d k	rjtdn2|d k	r~|j\}
}n|d k	r|j\}
}}ntd|}d}|d k	r|d d jd }|| }|d kr|d k	r|jn|j}tj	||| tj
|d}|dd|}n|d|
 }|d kr4| |}|d krVtj|
|ftj|jd}| ||
|f||}|}| jr| jr|rtd d}|rd	nd } rd	nd }|rd	nd }t| jD ]\}}|r||f7 }|d k	r|| nd | jr,| jr, fd
d}tjj|||||}n|||| |d}|d }|rf|| r\dnd f7 } r||d f7 }q| |}|r||f7 }|r|nd }|	stdd ||||fD S t||||dS )NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timezEYou have to specify either decoder_input_ids or decoder_inputs_embedsr   r6   r   r   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr-   c                    s    fdd}|S )Nc                     s    | f S rZ   r-   )inputs)r   r   r   r-   r.   custom_forward  s    zMPersimmonModel.forward.<locals>.create_custom_forward.<locals>.custom_forwardr-   )r   r   r   r   )r   r.   create_custom_forward  s    z5PersimmonModel.forward.<locals>.create_custom_forward)r   rh   r   r   r   r   c                 s   s   | ]}|d k	r|V  qd S rZ   r-   )r   vr-   r-   r.   	<genexpr>  s      z)PersimmonModel.forward.<locals>.<genexpr>)Zlast_hidden_stater   rr   
attentions)rq   r   r   r   use_return_dictr|   ra   r   r   r#   longre   r&   r   Zonesr1   r   r   ZtrainingloggerZwarning_once	enumerater   utils
checkpointr   tupler   )rC   r   r   rh   r   r   r   r   r   r   r   r   r   Zseq_length_with_pastr   r   rr   Zall_hidden_statesZall_self_attnsZnext_decoder_cacheidxZdecoder_layerr   Zlayer_outputsZ
next_cacher-   r   r.   rT   V  s    

   


     
	

zPersimmonModel.forward)	NNNNNNNNN)rU   rV   rW   r_   r   r=   r   r   r   r   PERSIMMON_INPUTS_DOCSTRINGr   r   r   r   r   r   r1   r   r   r   rT   rX   r-   r-   rD   r.   r     s6            
r   c                       s   e Zd ZdgZ fddZdd Zdd Zdd	 Zd
d Zdd Z	dd Z
eeeeeddejeej eej eeej  eej eej ee ee ee ee eeef dddZdddZedd Z  ZS )PersimmonForCausalLMzlm_head.weightc                    s@   t  | t|| _|j| _tj|j|jdd| _| 	  d S NFru   )
r<   r=   r   r   r   r   rk   rl   lm_headr   rp   rD   r-   r.   r=     s
    
zPersimmonForCausalLM.__init__c                 C   s   | j jS rZ   r   r   r   r-   r-   r.   r     s    z)PersimmonForCausalLM.get_input_embeddingsc                 C   s   || j _d S rZ   r   r   r-   r-   r.   r     s    z)PersimmonForCausalLM.set_input_embeddingsc                 C   s   | j S rZ   r   r   r-   r-   r.   get_output_embeddings  s    z*PersimmonForCausalLM.get_output_embeddingsc                 C   s
   || _ d S rZ   r   )rC   Znew_embeddingsr-   r-   r.   set_output_embeddings  s    z*PersimmonForCausalLM.set_output_embeddingsc                 C   s
   || _ d S rZ   r   )rC   decoderr-   r-   r.   set_decoder  s    z PersimmonForCausalLM.set_decoderc                 C   s   | j S rZ   r   r   r-   r-   r.   get_decoder  s    z PersimmonForCausalLM.get_decoder)output_typer   Nr   r   rh   r   r   labelsr   r   r   r   r   c                 C   s"  |dk	r|n| j j}|	dk	r |	n| j j}	|
dk	r4|
n| j j}
| j||||||||	|
d	}|d }| |}d}|dk	r|dddddf  }|dddf  }t }|d| j j	}|d}|
|j}|||}|
s
|f|dd  }|dk	r|f| S |S t|||j|j|jdS )u  
        Args:
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, PersimmonForCausalLM

        >>> model = PersimmonForCausalLM.from_pretrained("adept/persimmon-8b-base")
        >>> tokenizer = AutoTokenizer.from_pretrained("adept/persimmon-8b-base")

        >>> prompt = "human: Hey, what should I eat for dinner?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        'human: Hey, what should I eat for dinner?\n\ncat: 🐱\n\nhuman: 😐\n\n'
        ```N)	r   r   rh   r   r   r   r   r   r   r   .r   r   losslogitsr   rr   r   )rq   r   r   r   r   r   r   r   r&   r   r'   r   r   r   rr   r   )rC   r   r   rh   r   r   r   r   r   r   r   r   rr   r   r   Zshift_logitsZshift_labelsloss_fctoutputr-   r-   r.   rT     sH    )


zPersimmonForCausalLM.forwardc                 K   s   |r|d d dd f }| dd }|d k	rp|d krp| dd }||dkd |rp|d d df d}|d k	r|d krd|i}nd|i}|||| d|d |S )	Nr   rh   r   r   r   r   r   )rh   r   r   r   )getr   Zcumsumr%   re   update)rC   r   r   r   r   kwargsrh   Zmodel_inputsr-   r-   r.   prepare_inputs_for_generationT  s&    
z2PersimmonForCausalLM.prepare_inputs_for_generationc                    s.   d}| D ] }|t  fdd|D f7 }q|S )Nr-   c                 3   s"   | ]}| d  |jV  qdS )r   N)Zindex_selectr'   r   )r   Z
past_statebeam_idxr-   r.   r   w  s     z6PersimmonForCausalLM._reorder_cache.<locals>.<genexpr>)r   )r   r   Zreordered_pastZ
layer_pastr-   r   r.   _reorder_cacher  s    z#PersimmonForCausalLM._reorder_cache)
NNNNNNNNNN)NNN)rU   rV   rW   Z_tied_weights_keysr=   r   r   r   r   r   r   r   r   r   r   _CONFIG_FOR_DOCr   r   r   r   r   r   r1   r   r   rT   r   staticmethodr   rX   r-   r-   rD   r.   r     sN   

          
X     
r   a  
    The Persimmon transformer with a sequence classification head on top (linear layer).

    [`PersimmonForSequenceClassification`] uses the last token in order to do the classification, as other causal
    models (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                       s   e Zd Z fddZdd Zdd Zeedej	e
ej e
ej	 e
eej  e
ej e
ej	 e
e e
e e
e e
e eeef dd	d
Z  ZS )"PersimmonForSequenceClassificationc                    s@   t  | |j| _t|| _tj|j| jdd| _| 	  d S r   )
r<   r=   
num_labelsr   r   r   rk   rl   scorer   rp   rD   r-   r.   r=     s
    
z+PersimmonForSequenceClassification.__init__c                 C   s   | j jS rZ   r   r   r-   r-   r.   r     s    z7PersimmonForSequenceClassification.get_input_embeddingsc                 C   s   || j _d S rZ   r   r   r-   r-   r.   r     s    z7PersimmonForSequenceClassification.set_input_embeddingsNr   c                 C   s(  |
dk	r|
n| j j}
| j||||||||	|
d	}|d }| |}|dk	rV|jd }n
|jd }| j jdkr||dkr|td| j jdkrd}n4|dk	rt|| j j	 
dd |j}nd}|tj||jd|f }d}|dk	r||j}| j jdkrR| jdkrd| j _n:| jdkrJ|jtj	ks@|jtjkrJd	| j _nd
| j _| j jdkrt }| jdkr|| | }n
|||}nN| j jd	krt }||d| j|d}n| j jd
krt }|||}|
s|f|dd  }|dk	r|f| S |S t|||j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   rh   r   r   r   r   r   r   r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r   r   Z
regressionZsingle_label_classificationZmulti_label_classificationr   )rq   r   r   r   ra   r   r|   r   eqr   Zargmaxr'   r   r#   Zproblem_typer   r   r   r	   rd   r   r&   r   r   r   rr   r   )rC   r   r   rh   r   r   r   r   r   r   r   Ztransformer_outputsrr   r   r   Zsequence_lengthsZpooled_logitsr   r   r   r-   r-   r.   rT     sr    



(

z*PersimmonForSequenceClassification.forward)
NNNNNNNNNN)rU   rV   rW   r=   r   r   r   r   r   r   r   r   r   r   r1   r   r   r   rT   rX   r-   r-   rD   r.   r   |  s6   	          
r   )r   )N)7r_   r   typingr   r   r   r   r   Ztorch.utils.checkpointr   Ztorch.nnr   r   r	   Zactivationsr   Zmodeling_outputsr   r   r   Zmodeling_utilsr   r   r   r   r   r   Zconfiguration_persimmonr   Z
get_loggerrU   r   r   Sizer   r   r   r/   r   r2   Moduler3   rY   r`   rc   ri   rj   rs   r   ZPERSIMMON_START_DOCSTRINGr   r   r   r   r   r-   r-   r-   r.   <module>   sd   
    % )I@ 5 '