U
    9%e3                     @   s   d dl Z d dlmZ d dlmZmZmZmZmZ d dl	Z
ddlmZmZ ddlmZ dZdd	iZdd
diiZd
diZeeZG dd deZdS )    N)copyfile)AnyDictListOptionalTuple   )
AddedTokenPreTrainedTokenizer)loggingu   ▁
vocab_filezspiece.modelzgoogle/pegasus-xsumzDhttps://huggingface.co/google/pegasus-xsum/resolve/main/spiece.modeli   c                       s  e Zd ZdZeZeZeZ	ddgZ
d/eeeef  d	d fddZeedddZeeef dddZdd Zdd Zeee dddZeedddZeedddZd d! Zd0d#d$Zd%d& Zd1eee eee d'd(d)Zd2ee dd*d+Zd3eee e e d,d-d.Z!  Z"S )4PegasusTokenizera  
    Construct a PEGASUS tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        mask_token (`str`, *optional*, defaults to `"<mask_2>"`):
            The token used for masking single token values. This is the token used when training this model with masked
            language modeling (MLM). This is the token that the PEGASUS encoder will try to predict during pretraining.
            It corresponds to *[MASK2]* in [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive
            Summarization](https://arxiv.org/pdf/1912.08777.pdf).
        mask_token_sent (`str`, *optional*, defaults to `"<mask_1>"`):
            The token used for masking whole target sentences. This is the token used when training this model with gap
            sentences generation (GSG). This is the sentence that the PEGASUS decoder will try to predict during
            pretraining. It corresponds to *[MASK1]* in [PEGASUS: Pre-training with Extracted Gap-sentences for
            Abstractive Summarization](https://arxiv.org/pdf/1912.08777.pdf).
        additional_special_tokens (`List[str]`, *optional*):
            Additional special tokens used by the tokenizer. If no additional_special_tokens are provided <mask_2> and
            <unk_2, ..., unk_102> are used as additional special tokens corresponding to the [original PEGASUS
            tokenizer](https://github.com/google-research/pegasus/blob/939830367bcf411193d2b5eca2f2f90f3f9260ca/pegasus/ops/pretrain_parsing_ops.cc#L66)
            that uses the tokens 2 - 104 only for pretraining
        sp_model_kwargs (`dict`, *optional*):
            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
            to set:

            - `enable_sampling`: Enable subword regularization.
            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.

              - `nbest_size = {0,1}`: No sampling is performed.
              - `nbest_size > 1`: samples from the nbest_size results.
              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
                using forward-filtering-and-backward-sampling algorithm.

            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
              BPE-dropout.
    Z	input_idsZattention_mask<pad></s><unk><mask_2><mask_1>Ng   )sp_model_kwargsreturnc
                    s  || _ |d k	rt|ts4tdtt dt| ||krN|d k	rN|g| n|}|dd tt|| j d D 7 }tt|t|krtd| d|}n0g }|d k	r|gng }|dd td	| j D 7 }|	d kri n|	| _	|| _
|| _tjf | j	| _| j| tt|d
d
dtt|d
d
dd| _| j
d k	r\t|| jd	< tt|| jd< td| j d D ]"}td| d| jt| j< qlt jf |||||||| j	d|
 d S )Nz,additional_special_tokens should be of type z	, but is c                 S   s   g | ]}d | dqS <unk_> .0ir   r   o/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/models/pegasus/tokenization_pegasus.py
<listcomp>   s    z-PegasusTokenizer.__init__.<locals>.<listcomp>   zPlease make sure that the provided additional_special_tokens do not contain an incorrectly shifted list of <unk_x> tokens. Found .c                 S   s   g | ]}d | dqS r   r   r   r   r   r   r      s        T)lstriprstrip)r   r   r   r   r   )	eos_token	unk_token
mask_token	pad_tokenmask_token_sentoffsetadditional_special_tokensr   )r)   
isinstancelist	TypeErrortyperangelenset
ValueErrorr   r(   r   spmSentencePieceProcessorsp_modelLoadr	   strZ_added_tokens_decodersuper__init__)selfr   r'   r$   r%   r&   r(   r*   r)   r   kwargsZ"additional_special_tokens_extendedr   	__class__r   r   r9   h   s`    

 	zPegasusTokenizer.__init__)r   c                 C   s   t | j| j S )N)r0   r5   r)   r:   r   r   r   
vocab_size   s    zPegasusTokenizer.vocab_sizec                    s(    fddt  jD }| j |S )Nc                    s   i | ]}  ||qS r   )Zconvert_ids_to_tokensr   r>   r   r   
<dictcomp>   s      z.PegasusTokenizer.get_vocab.<locals>.<dictcomp>)r/   r?   updateZadded_tokens_encoder)r:   Zvocabr   r>   r   	get_vocab   s    zPegasusTokenizer.get_vocabc                 C   s   | j  }d |d< |S )Nr5   )__dict__copy)r:   stater   r   r   __getstate__   s    
zPegasusTokenizer.__getstate__c                 C   s8   || _ t| dsi | _tjf | j| _| j| j d S )Nr   )rC   hasattrr   r3   r4   r5   r6   r   )r:   dr   r   r   __setstate__   s
    
zPegasusTokenizer.__setstate__)textr   c                 C   s   | j j|tdS )zPTake as input a string and return a list of strings (tokens) for words/sub-words)Zout_type)r5   encoder7   )r:   rJ   r   r   r   	_tokenize   s    zPegasusTokenizer._tokenize)tokenr   c                 C   s   | j |}|| j S )z0Converts a token (str) to an id using the vocab.)r5   Zpiece_to_idr)   )r:   rM   Zsp_idr   r   r   _convert_token_to_id   s    z%PegasusTokenizer._convert_token_to_id)indexr   c                 C   s,   || j k r| j|S | j|| j  }|S )z=Converts an index (integer) to a token (str) using the vocab.)r)   r5   Z	IdToPiece)r:   rO   rM   r   r   r   _convert_id_to_token   s    
z%PegasusTokenizer._convert_id_to_tokenc                 C   sX   g }d}|D ]2}|| j kr4|| j|| 7 }g }q|| q|| j|7 }| S )z:Converts a sequence of tokens (string) in a single string. )Zall_special_tokensr5   decodeappendstrip)r:   tokensZcurrent_sub_tokensZ
out_stringrM   r   r   r   convert_tokens_to_string   s    
z)PegasusTokenizer.convert_tokens_to_stringFc                 C   s   dS )zJust EOSr   r   )r:   pairr   r   r   num_special_tokens_to_add   s    z*PegasusTokenizer.num_special_tokens_to_addc                    s(   t | j  | j  fdd|D S )Nc                    s   g | ]}| krd ndqS )r   r   r   )r   xall_special_idsr   r   r      s     z8PegasusTokenizer._special_token_mask.<locals>.<listcomp>)r1   r[   removeZunk_token_id)r:   seqr   rZ   r   _special_token_mask   s    
z$PegasusTokenizer._special_token_mask)token_ids_0token_ids_1already_has_special_tokensr   c                 C   s>   |r|  |S |dkr&|  |dg S |  || dg S dS )zCGet list where entries are [1] if a token is [eos] or [pad] else 0.Nr   )r^   )r:   r_   r`   ra   r   r   r   get_special_tokens_mask   s
    
z(PegasusTokenizer.get_special_tokens_maskc                 C   s$   |dkr|| j g S || | j g S )a@  
        Build model inputs from a sequence or a pair of sequences for sequence classification tasks by concatenating
        and adding special tokens. A PEGASUS sequence has the following format, where `X` represents the sequence:

        - single sequence: `X </s>`
        - pair of sequences: `A B </s>` (not intended use)

        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
        separator.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        N)Zeos_token_id)r:   r_   r`   r   r   r    build_inputs_with_special_tokens   s    z1PegasusTokenizer.build_inputs_with_special_tokens)save_directoryfilename_prefixr   c              	   C   s   t j|s"td| d d S t j||r6|d ndtd  }t j| jt j|krzt j	| jrzt
| j| n8t j	| jst|d}| j }|| W 5 Q R X |fS )NzVocabulary path (z) should be a directory-rQ   r   wb)ospathisdirloggererrorjoinVOCAB_FILES_NAMESabspathr   isfiler   openr5   Zserialized_model_protowrite)r:   rd   re   Zout_vocab_filefiZcontent_spiece_modelr   r   r   save_vocabulary  s     (
z PegasusTokenizer.save_vocabulary)r   r   r   r   r   Nr   N)F)NF)N)N)#__name__
__module____qualname____doc__rn   Zvocab_files_namesPRETRAINED_VOCAB_FILES_MAPZpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESZmax_model_input_sizesZmodel_input_namesr   r   r7   r   r9   propertyintr?   rB   rF   rI   r   rL   rN   rP   rV   rX   r^   boolrb   rc   r   rt   __classcell__r   r   r<   r   r   *   sL   8        
G

     r   )rh   shutilr   typingr   r   r   r   r   Zsentencepiecer3   Ztokenization_utilsr	   r
   utilsr   ZSPIECE_UNDERLINErn   ry   rz   Z
get_loggerru   rk   r   r   r   r   r   <module>   s     
