U ,-e8@s$dZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z m Z mZddlmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZmZm Z m!Z!e!"e#Z$dZ%d Z&d Z'Gd d d Z(d dZ)ddZ*ddZ+ddZ,ddZ-e e.e.dddZ/e eGdddeZ0dS)z Tokenization classes for python tokenizers. For fast tokenizers (provided by HuggingFace's tokenizers library) see tokenization_utils_fast.py N) OrderedDict)AnyDictListOptionalTupleUnionoverload) ENCODE_KWARGS_DOCSTRING'ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRINGINIT_TOKENIZER_DOCSTRING AddedToken BatchEncoding EncodedInputEncodedInputPairPreTokenizedInputPreTokenizedInputPairPreTrainedTokenizerBase TextInput TextInputPairTruncationStrategy)PaddingStrategy TensorTypeadd_end_docstringsloggingzspecial_tokens_map.jsonzadded_tokens.jsonztokenizer_config.jsonc@sBeZdZdZddZedddZeeeddd Zd d Z d S) Triez Trie in Python. Creates a Trie out of a list of words. The trie is used to split on `added_tokens` in one pass Loose reference https://en.wikipedia.org/wiki/Trie cCsi|_t|_dSN)dataset_tokensselfr#`/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/tokenization_utils.py__init__:sz Trie.__init__)wordcCsP|sdS|j||j}|D]$}||kr2||p4i||<||}qd|d<dS)uy Passes over every char (utf-8 char) on word and recursively adds it to the internal `data` trie representation. The special key `""` is used to represent termination. This function is idempotent, adding twice the same word will leave the trie unchanged Example: ```python >>> trie = Trie() >>> trie.add("Hello 友達") >>> trie.data {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}} >>> trie.add("Hello") >>> trie.data {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}} ``` Nr )r addr)r"r&refcharr#r#r$r(>s  zTrie.addtextreturncCst}dg}d}t|D]x\}}|r0||kr0qt}d}|D]\} } d| kr,|D]\} } | | krvq n"| | kr|d} |d}n|} |}| t|kr|| nd}d| kr| } | }| }|| kr^| |} | d7} d| kr| } | }| }| t|krq^|| }qq^|| ||d}qTqB|| krH| |} | || <qB|| qB|r`i}n|D] } || =qd||kr||jkr|j|||<q|D]6\} } d| krt|}|| ||qԐq|||S)a\ Will look for the words added to the trie within `text`. Output is the original string splitted along the boundaries of the words found. This trie will match the longest possible word first ! Example: ```python >>> trie = Trie() >>> trie.split("[CLS] This is a extra_id_100") ["[CLS] This is a extra_id_100"] >>> trie.add("[CLS]") >>> trie.add("extra_id_1") >>> trie.add("extra_id_100") >>> trie.split("[CLS] This is a extra_id_100") ["[CLS]", " This is a ", "extra_id_100"] ``` rFr'r NT) r enumerateritemslenappendr(rcut_text)r"r,ZstatesoffsetsskipcurrentZ current_charZ to_removeresetstartZ trie_pointerZ lookstartZlooktrie_pointerZlookahead_indexendZ next_charr#r#r$split]sl!              z Trie.splitcCsZ|t|g}d}|D]:}||kr4tdqn ||kr>q|||||}q|S)NrzbThere was a bug in Trie algorithm in tokenization. Attempting to recover. Please report it anyway.)r1r0loggererror)r"r,r3tokensr7r8r#r#r$r2sz Trie.cut_textN) __name__ __module__ __qualname____doc__r%strr(rr9r2r#r#r#r$r4s rcCs>|dks |dks |dks |dkr$dSt|}|dkr:dSdS)z0Checks whether `char` is a whitespace character.    TZsF) unicodedatacategoryr*catr#r#r$_is_whitespaces   rKcCs8|dks|dks|dkrdSt|}|dr4dSdS)z-Checks whether `char` is a control character.rCrDrEFCT)rGrH startswithrIr#r#r$ _is_controls   rNcCsht|}|dkr|dksH|dkr(|dksH|dkr8|dksH|dkrL|dkrLd St|}|d rdd Sd S) z1Checks whether `char` is a punctuation character.!/:@[`{~TPF)ordrGrHrM)r*cprJr#r#r$_is_punctuation's@  rZcCs$|d}tt|t|Bt|BS)zcChecks whether the last character in text is one of a punctuation, control or whitespace character.boolrNrZrK)r, last_charr#r#r$_is_end_of_word6sr_cCs$|d}tt|t|Bt|BS)zdChecks whether the first character in text is one of a punctuation, control or whitespace character.rr\)r, first_charr#r#r$_is_start_of_word<sra) token_list new_tokencCs8t||}|t|kr(|||kr(dS|||dS)zm Inserts one token to an ordered list if it does not already exist. Note: token_list must be sorted. N)bisect bisect_leftr0insert)rbrcZ insertion_idxr#r#r$!_insert_one_token_to_ordered_listBs rgcseZdZdZfddZeedddZeedddZ ee e efdd d Z ee ee fdd d Zeje eee e ffe ee fd dd Ze e efdddZddZdHeee ee feedddZgfee dddZdIeedddZeee dddZd d!Zee ee feeeefd"d#d$Zd%d&Zd'd(Zd)d*eje j!d)d+dd)d)d)d)ddddd*feee"e#feeee"e#feee eeeeeeeee e$feeeeeeeeee%d,d-d.Z&d*eje j!d)d+dd)d)d)d)ddddd*feeeee'ee"ee(ee#ee)feee eeeeeeeee e$feeeeeeeeee%d/d0d1Z*e+e,e-d*eje j!d)d+d)d)d)d)dddd*f eee(e.eed)ffeee eeeeeee eeeeeeeee%d2d3d4Z/dJe ee.e e e e0ffd5d6d7Z1dKeeeeeed8fd9d: Z2e3dLeee d;dd=Z4dNeeeefeee ee fd;d?d=Z4ee d@dAdBZ5ee e d"dCdDZ6dOeeeeee dEdFdGZ7Z8S)PPreTrainedTokenizera Base class for all slow tokenizers. Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`]. Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading pretrained tokenizers as well as adding tokens to the vocabulary. This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...). c sptjf|t|_t|ds&i|_d|kr@|j|ddd|jD|_ |j |j ddd|_ dS)N_added_tokens_decoderadded_tokens_decodercSsi|]\}}|j|qSr#content.0vkr#r#r$ jsz0PreTrainedTokenizer.__init__..T)special_tokensF) superr%r tokens_triehasattrriupdategetr/_added_tokens_encoder _add_tokensZall_special_tokens_extended_decode_use_source_tokenizer)r"kwargs __class__r#r$r%]s zPreTrainedTokenizer.__init__)r-cCsdS)NFr#r!r#r#r$is_fastrszPreTrainedTokenizer.is_fastcCstdS)zP `int`: Size of the base vocabulary (without the added tokens). NNotImplementedErrorr!r#r#r$ vocab_sizevszPreTrainedTokenizer.vocab_sizecCs ddt|jdddDS)z Returns the sorted mapping from string to index. The added tokens encoder is cached for performance optimisation in `self._added_tokens_encoder` for the slow tokenizers. cSsi|]\}}|j|qSr#rkrmr#r#r$rqsz.cSs|dSNrr#itemr#r#r$z:PreTrainedTokenizer.added_tokens_encoder..key)sortedrir/r!r#r#r$added_tokens_encoder}sz(PreTrainedTokenizer.added_tokens_encodercCstt|jdddS)z Returns the added tokens in the vocabulary as a dictionary of index to AddedToken. Returns: `Dict[str, int]`: The added tokens. cSs|dSrr#rr#r#r$rrz:PreTrainedTokenizer.added_tokens_decoder..r)dictrrir/r!r#r#r$rjsz(PreTrainedTokenizer.added_tokens_decoder)valuer-c Cs|D]r\}}t|ttfr(t|tsPtd|j|jfdttttfft|trbt|n||j|<||j t|<qdS)Nz;The provided `added_tokens_decoder` has an element of type z, should be a dict of ) r/ isinstancerArint ValueErrorr}rrirx)r"rindextokenr#r#r$rjs"cCs|jS)aX Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from the fast call because for now we always add the tokens even if they are already in the vocabulary. This is something we should change. Returns: `Dict[str, int]`: The added tokens. )rxr!r#r#r$get_added_vocabs z#PreTrainedTokenizer.get_added_vocabcCstt|S)z Size of the full vocabulary with the added tokens. Counts the `keys` and not the `values` because otherwise if there is a hole in the vocab, we will add tokenizers at a wrong index. )r0r get_vocabkeysr!r#r#r$__len__szPreTrainedTokenizer.__len__F) new_tokensrrr-cCsNd}|dkr|S|}t|}|D]}t|ttfsVtd|dt|dt|dkrdq(t|trt|| ddd}|rd|_||j krq(|js|j rt |d r|j r|j |_ |j |kr||}|||j <|d 7}n ||j }|jrt||jkr|j|||j |<||j|j <|jr(td |d q(||S) am Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to it with indices starting from length of the current vocabulary. Special tokens are sometimes already in the vocab which is why they have to be handled specifically. Args: new_tokens (`List[str]`or `List[tokenizers.AddedToken]`): Token(s) to add in vocabulary. A token is counted as added if it's not already in the vocabulary (tested by checking if the tokenizer assign the index of the `unk_token` to them). If a token is part of the vocabulary then we simply mark this token as an `AddedToken` which allows to control the stripping and normalization of this token. This is NOT possible in `tokenizers`. special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the tokens should be added as special tokens. Returns: `int`: The number of tokens actually added to the vocabulary. Examples: ```python # Let's see how to increase the vocabulary of Bert model and tokenizer tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") model = BertModel.from_pretrained("bert-base-uncased") num_added_toks = tokenizer.add_tokens(["new_tok1", "my_new-tok2"]) print("We have added", num_added_toks, "tokens") # Note: resize_token_embeddings expects to receive the full size of the new vocabulary, i.e. the length of the tokenizer. model.resize_token_embeddings(len(tokenizer)) ```rNzToken z is not a string but a .r'T) normalizedrstriplstrip do_lower_caser zAdding z to the vocabulary)rcopyr0rrAr TypeErrortypespecialrirrurrllowerall_special_tokensZ_additional_special_tokensr1rxverboser:info _update_trie)r"rrrZ added_tokensZ current_vocabZnew_idxrZ token_indexr#r#r$rys>             zPreTrainedTokenizer._add_tokens)unique_no_split_tokenscCsP|jD]}||jjkr |j|jq |D]}||jjkr.|j|q.dSr)rivaluesrtr r(rl)r"rrr#r#r$rs   z PreTrainedTokenizer._update_trie)pairr-cCs g}g}t|||r|ndS)aG Returns the number of added tokens when encoding a sequence with special tokens. This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put this inside your training loop. Args: pair (`bool`, *optional*, defaults to `False`): Whether the number of added tokens should be computed in the case of a sequence pair or a single sequence. Returns: `int`: Number of special tokens added to sequences. N)r0Z build_inputs_with_special_tokens)r"r token_ids_0 token_ids_1r#r#r$num_special_tokens_to_addsz-PreTrainedTokenizer.num_special_tokens_to_addr+cKsF|d|j}|j|f|\}}|r6td|dt|dr|jrdd|jD}|dd|j D7}dd |d d }t |d d |}|rg}|g}nt |j}|j|}t|D]6\}} | |kr|j|j| d} |dkr||dnd} |t|dkr&||dnd} t| tr| jrT| rT| ||d<| jrr| rr| ||d<| jr| r| ddkr||d| 7<d||<n<| jr| r| ddkr| ||d||d<d||<qt| dt| qg} |D]6} | sq | |kr.| | n| || q | S)a" Converts a string in a sequence of tokens, using the tokenizer. Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). Takes care of added tokens. Args: text (`str`): The sequence to be encoded. **kwargs (additional keyword arguments): Passed along to the model-specific `prepare_for_tokenization` preprocessing method. Returns: `List[str]`: The list of tokens. split_special_tokenszKeyword arguments z not recognized.rcSsg|]}t|qSr#)reescapernZs_tokr#r#r$ *sz0PreTrainedTokenizer.tokenize..cSs$g|]}|js|jrt|jqSr#)rrrrrlrr#r#r$r+s(|z)|z(.+?)cSs|dp|dS)Nrr )groupsr)mr#r#r$r1rz.PreTrainedTokenizer.tokenize..Nrr r[rBr'zy cannot be tokenized because it was not properly added to the tokenizer. This means that it is not an `AddedToken` but a )poprprepare_for_tokenizationr:warningrurrrirjoinrsubrrxrrtr9r.rwr0rrrrZ single_wordrrr1extend _tokenize)r"r,r{rZescaped_special_tokspatternZno_split_tokenr<irZ tok_extendedleftrightZtokenized_textr#r#r$tokenizesV "     zPreTrainedTokenizer.tokenizecKstdS)a Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). Do NOT take care of added tokens. Nr)r"r,r{r#r#r$rbszPreTrainedTokenizer._tokenize)r<r-cCsB|dkr dSt|tr ||Sg}|D]}|||q(|S)aT Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the vocabulary. Args: tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s). Returns: `int` or `List[int]`: The token id or list of token ids. N)rrA#_convert_token_to_id_with_added_vocr1)r"r<idsrr#r#r$convert_tokens_to_idsks   z)PreTrainedTokenizer.convert_tokens_to_idscCs*|dkr dS||jkr |j|S||Sr)rx_convert_token_to_idr"rr#r#r$rs   z7PreTrainedTokenizer._convert_token_to_id_with_added_voccCstdSrrrr#r#r$rsz(PreTrainedTokenizer._convert_token_to_idNTr)r, text_pairadd_special_tokenspadding_strategytruncation_strategy max_lengthstrideis_split_into_wordspad_to_multiple_ofreturn_tensorsreturn_token_type_idsreturn_attention_maskreturn_overflowing_tokensreturn_special_tokens_maskreturn_offsets_mapping return_lengthrr-c sffdd}|rtd||}|dk r4||nd}j||||j|j||| | d| | | |||dS)Ncst|tr"j|f}|St|ttfrt|dkrt|dtrrvttjfdd|D}|S|SnRt|ttfrt|dkrt|dt r|Srt d|dnt d|ddS)Nrc3s$|]}j|fddiVqdSrTNrrntr{r"r#r$ szJPreTrainedTokenizer._encode_plus..get_input_ids..zInput z] is not valid. Should be a string or a list/tuple of strings when `is_split_into_words=True`.zW is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers. rrArrlisttupler0 itertoolschainrrr,r<rr{r"r#r$ get_input_idss&  (  (  z7PreTrainedTokenizer._encode_plus..get_input_idsa return_offset_mapping is not available when using Python tokenizers. To use this feature, change your tokenizer to one deriving from transformers.PreTrainedTokenizerFast. More information on available tokenizers at https://github.com/huggingface/transformers/pull/2674T)pair_idsrpadding truncationrrrrprepend_batch_axisrrrrrr)rprepare_for_modelr)r"r,rrrrrrrrrrrrrrrrr{r first_ids second_idsr#rr$ _encode_pluss2z PreTrainedTokenizer._encode_plus)batch_text_or_text_pairsrrrrrrrrrrrrrrrr-c sfdd}|rtdg}|D]r}t|ttfsB|d}}n*rdt|dttfsd|d}}n|\}}||}|dk r||nd}|||fq$j|||||||| | | | || |d}t|S)Ncst|tr"j|f}|St|ttfrt|dkrt|dtrrvttjfdd|D}|S|Sn4t|ttfrt|dkrt|dt r|St ddS)Nrc3s$|]}j|fddiVqdSrrrrr#r$rszPPreTrainedTokenizer._batch_encode_plus..get_input_ids..z\Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.rrrr#r$rs  (  (z=PreTrainedTokenizer._batch_encode_plus..get_input_idszreturn_offset_mapping is not available when using Python tokenizers. To use this feature, change your tokenizer to one deriving from transformers.PreTrainedTokenizerFast.r) rrrrrrrrrrrrr)rrrrr1_batch_prepare_for_modelr)r"rrrrrrrrrrrrrrrrr{rZ input_idsZids_or_pair_idsrrrr batch_outputsr#rr$_batch_encode_pluss@  z&PreTrainedTokenizer._batch_encode_plus)batch_ids_pairsrrrrrrrrrrrrrr-cCsi}|D]h\}}|j|||tjj|j||dd| | | | dd|d}|D]&\}}||kr`g||<|||qHq|j||j||| d}t||d}|S)a Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates sequences if overflowing while taking into account the special tokens and manages a moving window (with user defined stride) for overflowing tokens Args: batch_ids_pairs: list of tokenized input ids or input ids pairs NF)rrrrrrrrrrrrrr)rrrr)Z tensor_type)rr DO_NOT_PADrr/r1padr)r"rrrrrrrrrrrrrrrrroutputsrrr#r#r$r/sB  z,PreTrainedTokenizer._batch_prepare_for_model)r,rr-cKs||fS)a Performs any necessary transformations before tokenization. This method should pop the arguments from kwargs and return the remaining `kwargs` as well. We test the `kwargs` at the end of the encoding process to be sure all the arguments have been used. Args: text (`str`): The text to prepare. is_split_into_words (`bool`, *optional*, defaults to `False`): Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) which it will tokenize. This is useful for NER or token classification. kwargs (`Dict[str, Any]`, *optional*): Keyword arguments to use for the tokenization. Returns: `Tuple[str, Dict[str, Any]]`: The prepared text and the unused kwargs. r#)r"r,rr{r#r#r$rpsz,PreTrainedTokenizer.prepare_for_tokenization)rralready_has_special_tokensr-csD|r&|dk rtdtj||ddSdg|r6t|ndt|S)a Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. Args: token_ids_0 (`List[int]`): List of ids of the first sequence. token_ids_1 (`List[int]`, *optional*): List of ids of the second sequence. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. NzYou should not supply a second sequence if the provided sequence of ids is already formatted with special tokens for the model.T)rrrr)rrsget_special_tokens_maskr0)r"rrrr|r#r$rsz+PreTrainedTokenizer.get_special_tokens_mask)rskip_special_tokensr-cCsdSrr#r"rrr#r#r$convert_ids_to_tokenssz)PreTrainedTokenizer.convert_ids_to_tokenscCsdSrr#rr#r#r$rscCst|tr*||jkr |j|jS||Sg}|D]J}t|}|rN||jkrNq2||jkrl||j|jq2|||q2|S)a Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and added tokens. Args: ids (`int` or `List[int]`): The token id (or token ids) to convert to tokens. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. Returns: `str` or `List[str]`: The decoded token(s). )rrrirl_convert_id_to_tokenall_special_idsr1)r"rrr<rr#r#r$rs     )rr-cCstdSrr)r"rr#r#r$rsz(PreTrainedTokenizer._convert_id_to_tokencCs d|S)NrB)r)r"r<r#r#r$convert_tokens_to_stringsz,PreTrainedTokenizer.convert_tokens_to_string) token_idsrclean_up_tokenization_spacesspaces_between_special_tokensr-c s |dd_j||d}tjtjfddjDB}g}g} |D]Z} |rh| jkrhqT| |kr| r | } t | dkr| | g} | | qT| | qT| r| | |rd |} n d |} |dk r|nj }|r| } | S| SdS) NZuse_source_tokenizerF)rcs h|]}|jkr|qSr#)rr)rnrr!r#r$ sz.PreTrainedTokenizer._decode..rrBr')rrzrrrxrrZadditional_special_tokensrrr0r1rrZclean_up_tokenization)r"rrrrr{Zfiltered_tokensZlegacy_added_tokensZ sub_textsZcurrent_sub_textrstringr,Z clean_textr#r!r$_decodes>         zPreTrainedTokenizer._decode)F)F)F)NF)F)F)F)FNT)9r=r>r?r@r%propertyr]r~rrrrArrrjsetterrrrrryrrrrrrrrrrrrZDO_NOT_TRUNCATErrrrrrrrrrr r rrrrrr rrrr __classcell__r#r#r|r$rhOsN  *  $EQ (  X UA  rh)1r@rdrrrG collectionsrtypingrrrrrrr Ztokenization_utils_baser r r rrrrrrrrrrutilsrrrrZ get_loggerr=r:ZSPECIAL_TOKENS_MAP_FILEZADDED_TOKENS_FILEZTOKENIZER_CONFIG_FILErrKrNrZr_rarArgrhr#r#r#r$s, $< \