U ,-e['@sdZddlZddlZddlmZmZmZddlmZm Z erFddl Z ddl m Z ddl mZeeZdd iZdd d iiZd d iZd dZGdddeZGddde ZdS)z Tokenization classes for CPMAnt.N)ListOptionalTuple)is_jieba_availablerequires_backends)PreTrainedTokenizer)logging vocab_filez vocab.txtzopenbmb/cpm-ant-10bz>https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txtic CsRt}t|ddd}|}W5QRXt|D]\}}|d}|||<q2|S)z*Loads a vocabulary file into a dictionary.rutf-8encoding ) collections OrderedDictopen readlines enumeraterstrip)r vocabreadertokensindextokenro/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/cpmant/tokenization_cpmant.py load_vocab-s  rc@seZdZdddZddZdS) WordpieceTokenizercCs||_||_||_dSN)r unk_tokenmax_input_chars_per_word)selfrr"r#rrr__init__9szWordpieceTokenizer.__init__cCst|}t||jkr|jgSd}g}|t|krt|}d}||krrd|||}||jkrh|}qr|d8}q>|dkr||j|d7}q&|||}q&|S)Nr)listlenr#r"joinrappend)r$rcharsstartZ sub_tokensendZ cur_substrsubstrrrrtokenize>s(      zWordpieceTokenizer.tokenizeN)rr )__name__ __module__ __qualname__r%r0rrrrr8s rc seZdZdZeZeZeZ ddgZ dZ d2fdd Z e ddZe ddZe ddZe edddZddZddZfddZdd Zeeed!d"d#Zd$d%Zd&d'Zd3eeeeed)d*d+Zd4eeeeeed,d-d.Zd5eeeeee eed/fd0d1 Z!Z"S)6CpmAntTokenizera Construct a CPMAnt tokenizer. Based on byte-level Byte-Pair-Encoding. Args: vocab_file (`str`): Path to the vocabulary file. bod_token (`str`, *optional*, defaults to `""`): The beginning of document token. eod_token (`str`, *optional*, defaults to `""`): The end of document token. bos_token (`str`, *optional*, defaults to `""`): The beginning of sequence token. eos_token (`str`, *optional*, defaults to `""`): The end of sequence token. pad_token (`str`, *optional*, defaults to `""`): The token used for padding. unk_token (`str`, *optional*, defaults to `""`): The unknown token. line_token (`str`, *optional*, defaults to `""`): The line token. space_token (`str`, *optional*, defaults to `""`): The space token. Z input_idsZattention_maskFrleftc st|dg||_||_t||_|j| |jd<|j||jd<|j| =|j|=tt|jddd|_dd|jD|_ t |j|d |_ t j f|||||||| | d | dS) Njieba rcSs|dSNr'rxrrrz*CpmAntTokenizer.__init__..keycSsi|]\}}||qSrr).0kvrrr sz,CpmAntTokenizer.__init__..)rr") bod_token eod_token bos_token eos_token pad_tokenr" line_token space_token padding_side)rrJrKrencoderrrsorteditemsdecoderrwordpiece_tokenizersuperr%) r$r rJrKrLrMrNr"rOrPrQkwargs __class__rrr%ws0   zCpmAntTokenizer.__init__cCs |j|jSr!)rRrJr$rrr bod_token_idszCpmAntTokenizer.bod_token_idcCs |j|jSr!)rRrKr[rrr eod_token_idszCpmAntTokenizer.eod_token_idcCs |jdS)NrrRr[rrr newline_idszCpmAntTokenizer.newline_id)returncCs t|jSr!)r)rRr[rrr vocab_sizeszCpmAntTokenizer.vocab_sizecCst|jf|jSr!)dictrRZadded_tokens_encoderr[rrr get_vocabszCpmAntTokenizer.get_vocabcCs.g}tj|ddD]}||j|q|S)zTokenize a string.F)Zcut_all)r=cutextendrVr0)r$textZ output_tokensrArrr _tokenizeszCpmAntTokenizer._tokenizec s0dd|D}fdd|D}tj|f|S)zDecode ids into a string.cSsg|]}|dkr|qS)rr)rFirrr sz+CpmAntTokenizer._decode..cs.g|]&}|jkr|jkr|jkr|qSr)Z pad_token_idZ eos_token_id bos_token_id)rFrAr[rrris  )rW_decode)r$Z token_idsrXrYr[rrks  zCpmAntTokenizer._decodecCs ||jkSr!r^r$rrrrcheckszCpmAntTokenizer.check)rr`cCs d|S)Nr&)r*)r$rrrrconvert_tokens_to_stringsz(CpmAntTokenizer.convert_tokens_to_stringcCs|j||j|jS)z0Converts a token (str) in an id using the vocab.)rRgetr"rlrrr_convert_token_to_idsz$CpmAntTokenizer._convert_token_to_idcCs|j||jS)z=Converts an index (integer) in a token (str) using the vocab.)rUror")r$rrrr_convert_id_to_tokensz$CpmAntTokenizer._convert_id_to_tokenN)save_directoryfilename_prefixr`c Cstj|r0tj||r |dndtd}n|r<|dnd|}d}d|jkrj|jd|jd<|jd=d|jkr|jd|jd<|jd=tt|j d d d |_t |d d dN}|j D]<\}}||krt d|d|}| |d|d7}qW5QRX|fS)N-r&r rr>r;rr:cSs|dSr?rr@rrrrBrCz1CpmAntTokenizer.save_vocabulary..rDwr r zSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!r')ospathisdirr*VOCAB_FILES_NAMESrRrrrSrTrloggerwarningwrite)r$rrrsr rwriterrZ token_indexrrrsave_vocabularys0    zCpmAntTokenizer.save_vocabulary) token_ids_0 token_ids_1r`cCs,|dkr|jg|S|jg||jg|S)a1 Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A CPMAnt sequence has the following format: - single sequence: `[BOS] Sequence`. Args: token_ids_0 (`List[int]`): The first tokenized sequence that special tokens will be added. token_ids_1 (`List[int]`): The optional second tokenized sequence that special tokens will be added. Returns: `List[int]`: The model input with special tokens. N)rj)r$rrrrr build_inputs_with_special_tokenss z0CpmAntTokenizer.build_inputs_with_special_tokens)rralready_has_special_tokensr`csZ|rtj||ddS|dk rFdgdgt|dgdgt|Sdgdgt|S)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)rrrNr'r)rWget_special_tokens_maskr))r$rrrrYrrrs(z'CpmAntTokenizer.get_special_tokens_mask) r5r6r7r8r9rr:r;r<)N)N)NF)#r1r2r3__doc__ryZvocab_files_namesPRETRAINED_VOCAB_FILES_MAPZpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESZmax_model_input_sizesZmodel_input_namesZadd_prefix_spacer%propertyr\r]r_intrarcrgrkrmrstrrnrprqrrr~rboolr __classcell__rrrYrr4XsR*      r4)rrrvtypingrrrZtransformers.utilsrrr=Ztokenization_utilsrutilsr Z get_loggerr1rzryrrrobjectrr4rrrrs*