U ,-ed@s.dZddlZddlZddlmZddlmZmZmZm Z ddl Z ddl Z ddlmZmZddlmZdd lmZd d d d dZddiddiddidZddiZddZeeZddZdddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|cZd}d~eDdddddddddddd ZddgZ GdddeZ!ddZ"dddZ#ddZ$dee%e&e&e&dddZ'ee%dddZ(ee%dddZ)ddZ*dS)z!Tokenization classes for Whisper.N) lru_cache)ListOptionalTupleUnion) AddedTokenPreTrainedTokenizer)logging)EnglishTextNormalizerz vocab.jsonztokenizer.jsonz merges.txtznormalizer.json) vocab_fileZtokenizer_file merges_filenormalizer_filezopenai/whisper-basezBhttps://huggingface.co/openai/whisper-base/resolve/main/vocab.jsonzGhttps://huggingface.co/openai/whisper-base/resolve/main/merges_file.txtzGhttps://huggingface.co/openai/whisper-base/resolve/main/normalizer.json)r rricCstttdtddtttdtddtttdtdd}|dd}d }td D],}||krf|||d ||d7}qfd d |D}tt||S) a8 Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control characters the bpe code barfs on. The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup tables between utf-8 bytes and unicode strings. !~r ¡¬®ÿNrcSsg|] }t|qS)chr).0nrrq/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/whisper/tokenization_whisper.py Hsz$bytes_to_unicode..)listrangeordappenddictzip)bscsrbrrrbytes_to_unicode4s L    r&cCs6t}|d}|ddD]}|||f|}q|S)z Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings). rr N)setadd)wordpairsZ prev_charcharrrr get_pairsPs r,englishchinesegermanspanishrussiankoreanfrenchjapanese portugueseturkishpolishcatalandutcharabicswedishitalianZ indonesianZhindifinnishZ vietnamesehebrewZ ukrainiangreekZmalayczechromaniandanish hungarianZtamil norwegianthaiZurducroatian bulgarian lithuanianlatinZmaoriZ malayalamZwelshslovakZteluguZpersianZlatvianZbengaliZserbianZ azerbaijani slovenianZkannadaestonianZ macedonianZbretonZbasque icelandicZarmenianZnepaliZ mongolianZbosnianZkazakhZalbanianZswahiligalicianZmarathiZpunjabisinhalaZkhmerZshonaZyorubaZsomaliZ afrikaansZoccitanZgeorgianZ belarusianZtajikZsindhiZgujaratiZamharicZyiddishlaoZuzbekZfaroesezhaitian creoleZpashtoZturkmennynorskZmalteseZsanskritZ luxembourgishmyanmarZtibetanZtagalogZmalagasyZassameseZtatarZhawaiianZlingalaZhausaZbashkirZjavaneseZ sundanese)cenzhdeesrukofrjapttrplcanlarsvitidhifiviheukelmsr$rodahutanothurhrbgltZlamimlcysktefalvZbnsrazslknetmkbreuishyneZmnr#kksqswglmrpasikmZsnZyosoafockabetgsdguamyilouzfohtpsZtknnmtsalbmyZbotlZmgasttZhawlnZhabaZjwZsucCsi|]\}}||qSrr)rcodelanguagerrr srrr^r_rrrrrkrrV) ZburmeseZ valencianZflemishZhaitianZ letzeburgeschZpushtoZpanjabiZ moldavianZmoldovanZ sinhaleseZ castilian translate transcribec seZdZdZeZeZeZ ddgZ dJfdd Z e e d d d Zd dZddZdKeeedddZe ee d ddZdLee d ddZdMee eee eee dfdd ZddZddZdd Zd!d"ZdNed d$d%ZdOd&d'ZedPd(d)Z dQed*d+d,Z!d-d.Z"dReeeeed/fd0d1 Z#dSe$e ee feeeed2d3d4Z%d5d6Z&dTeeee'ed7d8d9Z(dUd:d;Z)e d"`): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`): The beginning of sequence token. The `decoder_start_token_id` is used to set the first token as `"<|startoftranscript|>"` when generating. eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`): The end of sequence token. add_prefix_space (`bool`, *optional*, defaults to `False`): Whether or not to add an initial space to the input. This allows to treat the leading word just as any other word. language (`str`, *optional*): The language of the transcription text. The corresponding language id token is appended to the start of the sequence for multilingual speech recognition and speech translation tasks, e.g. for Spanish the token `"<|es|>"` is appended to the start of sequence. This should be used for multilingual fine-tuning only. task (`str`, *optional*): Task identifier to append at the start of sequence (if any). This should be used for mulitlingual fine-tuning, with `"transcribe"` for speech recognition and `"translate"` for speech translation. predict_timestamps (`bool`, *optional*, defaults to `False`): Whether to omit the `<|notimestamps|>` token at the start of the sequence. input_idsZattention_maskNreplace <|endoftext|>Fc  st|trt|dddddn|}t|tr.cSsi|]\}}||qSrrrrrrr-s r cSsg|]}t|qSr)tuplesplit)rmergerrrr0sz-WhisperTokenizer.__init__..zJ's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+z<\|(\d+\.\d+)\|>)errors unk_token bos_token eos_token pad_tokenadd_prefix_space) isinstancestrropenjsonloadencoderitemsdecoderrr& byte_encoder byte_decoderreadrr!r"rlen bpe_rankscacherenglish_spelling_normalizerrecompilepat timestamp_patrsuper__init__taskpredict_timestamps)selfr rrrrrrrrrrrkwargsZ vocab_handleZ merges_handleZ bpe_merges __class__rrrs^     zWhisperTokenizer.__init__)returncCs t|jSN)rrrrrr vocab_sizeMszWhisperTokenizer.vocab_sizecs(fddtjD}|j|S)Ncsi|]}||qSr)convert_ids_to_tokensrirrrrRsz.WhisperTokenizer.get_vocab..)rrupdateadded_tokens_encoder)rZvocabrrr get_vocabQs zWhisperTokenizer.get_vocabc sd|jkrj|St|}t|}|s,|St|fddd}|jkrNqL|\}}g}d}|t|kr"z|||} Wn,tk r|||dYq"YnX|||| | }|||kr |t|dkr ||d|kr | |||d7}q^| |||d7}q^t|}|}t|dkrBqLq,t|}q,d |}|j|<|S)Ncsj|tdS)Ninf)rgetfloat)pairrrraz&WhisperTokenizer.bpe..keyrr  ) rrr,minrrindex ValueErrorextendr join) rtokenr)r*ZbigramfirstsecondZnew_wordrjrrrbpeWsB    2     zWhisperTokenizer.bperrrcCs@|dk r |n|j|_|dk r |n|j|_|dk r4|n|j|_dS)a Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to update the prefix tokens as required when fine-tuning. Example: ```python >>> # instantiate the tokenizer and set the prefix token to Spanish >>> tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny", language="spanish") >>> # now switch the prefix token from Spanish to French >>> tokenizer.set_prefix_tokens(language="french") ``` Args: language (`str`, *optional*, defaults to `None`): The language of the transcription text. task (`str`, *optional*, defaults to `None`): Task identifier to append at the start of sequence (if any). predict_timestamps (`bool`, *optional*, defaults to `None`): Whether to omit the `<|notimestamps|>` token at the start of the sequence. Nr)rrrrrrrset_prefix_tokenssz"WhisperTokenizer.set_prefix_tokensc CsD|d}|d}|d}|d}tt}|jdk r|j|_|jtkr`t|j}nT|jtkrv|j}n>t|jdk}t d|jd|rt tn t td|j dk r|j t krt d |j d t |g}|jdk r| |d |||j dk r.| |j d kr(|n||js@| ||S) N<|startoftranscript|>z <|translate|>z<|transcribe|><|notimestamps|>rzUnsupported language: z. Language should be one of: .zUnsupported task: z. Task should be in: r r)convert_tokens_to_idsr LANGUAGESkeysrlowerTO_LANGUAGE_CODEvaluesrrrrTASK_IDSr rr) rZ bos_token_idZtranslate_token_idZtranscribe_token_idZnotimestamps_token_idZlangsZ language_idZis_language_codeZ bos_sequencerrr prefix_tokenss4         *     zWhisperTokenizer.prefix_tokenscCs0|dkr|j||jgS|j|||jgS)z=Build model inputs from a sequence by appending eos_token_id.N)r  eos_token_id)r token_ids_0 token_ids_1rrr build_inputs_with_special_tokenssz1WhisperTokenizer.build_inputs_with_special_tokens)r ralready_has_special_tokensrcsn|rtj||ddSdgt|j}dg}|dkrJ|dgt||S|dgt|dgt||S)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)r rrr Nr)rget_special_tokens_maskrr )rr rrZ prefix_onesZ suffix_onesrrrrsz(WhisperTokenizer.get_special_tokens_maskcsZg}tj|D]B}dfdd|dD}|dd|dDq|S)zTokenize a string.c3s|]}j|VqdSr)r)rr%rrr sz-WhisperTokenizer._tokenize..rcss|] }|VqdSrr)rZ bpe_tokenrrrrsr)rfindallrrencoderrr)rtext bpe_tokensrrrr _tokenizes"zWhisperTokenizer._tokenizecCs|j||j|jS)z0Converts a token (str) in an id using the vocab.)rrr)rrrrr_convert_token_to_idsz%WhisperTokenizer._convert_token_to_idcCs|j|dS)z Converts an index (integer) in a token (str) using the vocab. Whisper's base tokenizer always decodes OOV tokens as "", thus we do not use the `unk_token` here. r)rr)rrrrr_convert_id_to_tokensz%WhisperTokenizer._convert_id_to_tokencCst|j}||S)z Normalize a given string using the `EnglishTextNormalizer` class, which preforms commons transformation on english text. )r r)rrZ normalizerrrr _normalizes zWhisperTokenizer._normalize{Gz?cs~jdd}gg}|D]F}||krPd|||dd}|||gq|d|qfdd|D}d|S) z Timestamp tokens are above the special tokens' id range and are ignored by `decode()`. This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>". rr z<|z.2fz|>cs(g|] }t|tr|n j|dqS)skip_special_tokens)rrdecode)rsrrrrrsz.r)all_special_idsr r)r token_idsrtime_precisiontimestamp_beginoutputsr timestamprr!r_decode_with_timestampss   z(WhisperTokenizer._decode_with_timestampsc Cs\g}t|}|jddkr2t|jdkr2td|jdd}||k}t|dd|dd@dd}|jddkr|dkrgSt|ddd|krt|t|ddd}t|dd}|D]|}|||} t| dkrR| d |} | d |} | | } | | } | | } || | || |fd|}q|S)a~ Compute offsets for a given tokenized input Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Can be obtained using the `__call__` method. time_precision (`float`, `optional`, defaults to 0.02): The time ratio to convert from token to time. rr z)Can only process a single input at a timerNrr') nparrayshaperrr"wheresumr item_preprocess_token_ids_decode_filter_timestamp_ids) rr#r$offsetsr%Ztimestamp_tokensZ consecutiveZ last_sliceZ current_sliceZ sliced_tokensZstart_timestamp_positionZend_timestamp_positionrrrr_compute_offsetss:  &     z!WhisperTokenizer._compute_offsetscs|fddtdDS)a Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache. Args: time_precision (`float`, `optional`, defaults to 0.02): The time ratio to convert from token to time. csg|]}d|qS)z<|%.2f|>rrr$rrrMsz2WhisperTokenizer.timestamp_ids..i)rr)rr$rr5r timestamp_idsDs zWhisperTokenizer.timestamp_idsrcCs*|r&|d}|d}||||}|S)a Pre-process the token ids for decoding by removing the prompt tokens ids and timestamp token ids. Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Typically, obtained using the `__call__` method of the tokenizer. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens from the token ids. If `True`, the prompt token ids will be removed. <|startofprev|>r)r _strip_prompt)rr#rprompt_token_iddecoder_start_token_idrrrr0Os   z&WhisperTokenizer._preprocess_token_idscCst|jd|S)Nr)rsubr)rr#rrrr2asz&WhisperTokenizer._filter_timestamp_ids)rclean_up_tokenization_spacesoutput_offsetsdecode_with_timestampsrc  sj|j||d}tj|f|||d|} |r@|j|||d} n || } |rf|j||d} | | dS| S)a Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`. Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. If `None`, will default to `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. output_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output the offsets of the tokens. This should only be set if the model predicted timestamps. time_precision (`float`, `optional`, defaults to 0.02): The time ratio to convert from token to time. decode_with_timestamps (`bool`, *optional*, defaults to `False`): Whether or not to decode with timestamps included in the raw text. Returns: `str`: The decoded sentence. r)rr<r>)r$rr5)rr3)r0rrr(r2r4) rr#rr<r=r$r>rZ filtered_idsrr3rrrrds0$  zWhisperTokenizer.decode)r#r normalizer>rc Ks|dd|_|j||d}g}g}|D]L} |r<| |jkrrZfiltered_tokensZ sub_textsZcurrent_sub_textrrZ clean_textrrrr1s(     zWhisperTokenizer._decodecs0d|}tfdd|Djdjd}|S)z:Converts a sequence of tokens (string) in a single string.rcsg|]}j|qSr)r)rcrrrrsz=WhisperTokenizer.convert_tokens_to_string..r)r)r bytearrayrr)rtokensrrrrrAs "z)WhisperTokenizer.convert_tokens_to_string)save_directoryfilename_prefixrc Cstj|s"td|ddStj||r6|dndtd}tj||rX|dndtd}tj||rz|dndtd}t|dd d $}|t j |j d d d ddW5QRXd}t|dd d l}|dt |j dddD]D\} } || krtd|d| }|d| d|d7}qW5QRX|jdk rt|dd d $}|t j |jd d d ddW5QRX|||fS)NzVocabulary path (z) should be a directory-rr rrwrrrTF)indent sort_keys ensure_asciirrz#version: 0.2 cSs|dS)Nr r)kvrrrrrz2WhisperTokenizer.save_vocabulary..rzSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!rr )ospathisdirloggererrorrVOCAB_FILES_NAMESrwriterdumpsrsortedrrwarningr) rrErFr Z merge_filerfrwriterrZ token_indexrrrsave_vocabularysD (    z WhisperTokenizer.save_vocabularycKs&|d|j}|s|rd|}||fS)Nrr)r@r)rrZis_split_into_wordsrrrrrprepare_for_tokenizationsz)WhisperTokenizer.prepare_for_tokenizationcCsdS)zv A simple chat template that ignores role information and just concatenates messages with EOS tokens. zM{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}rrrrrdefault_chat_templatesz&WhisperTokenizer.default_chat_templateTcCs6|j||| d|jdd}ddt|D}|S)N)rrrr cSsg|]\}}|d|fqS)r r)rZrankrrrrr sz;WhisperTokenizer.get_decoder_prompt_ids..)rr  enumerate)rrrZ no_timestampsZ forced_tokensZforced_decoder_idsrrrget_decoder_prompt_idssz'WhisperTokenizer.get_decoder_prompt_idscCst|||||dS)N)return_timestampsreturn_languager$) _decode_asr)r model_outputsr^r_r$rrrr` szWhisperTokenizer._decode_asrr*)rcstdd|dd}|ddd}tfdd |Dd}|dk r`|}td |d |j|d |dS) z`Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`].r7rF)Zadd_special_tokensrr Nc3s |]}|jdkr|VqdS)rN)r")rxrrrrsz2WhisperTokenizer.get_prompt_ids..zJEncountered text in the prompt corresponding to disallowed special token: r)Z tensor_type)stripnextrrZconvert_to_tensors)rrZreturn_tensorsZbatch_encodingZprompt_text_idsZspecial_token_idrrrrget_prompt_idss  zWhisperTokenizer.get_prompt_ids)r#r9r:cCs@t|to|o|d|k}|r<||kr8|||dSgS|S)Nr)rrr)r#r9r:Z has_promptrrrr8$s zWhisperTokenizer._strip_prompt) NrrrrNFNNF)NNN)N)NF)Fr)r)r)F)FNFrF)FFF)N)F)NNT)r*)1__name__ __module__ __qualname____doc__rRZvocab_files_namesPRETRAINED_VOCAB_FILES_MAPZpretrained_vocab_files_mapMAX_MODEL_INPUT_SIZESZmax_model_input_sizesZmodel_input_namesrpropertyintrrrrboolrrr rrrrrrrr(r4rr6r0r2rrr1rArrYrZr[r]r`re staticmethodr8 __classcell__rrrrrs&J*"    0 A &'    rc'sdfdd}g}|}d}|dd} g} g} d} d} t|j}t|D]:\}}|dd }|d kr|d d }d}| }d |kr|d \}}}||8}||} |r||| }|rt|D],}|| kr|dk r|| || krq|}qg}g}t|D]\}}||kr||g}|d d}t|d}|dk r"r|kr|s| |t | }||}||d<| |g} g}|}||d<|nq|| kr|| ||} t | d } |r||krd} n| s| r||krd} n|dd dkr$| |dd <n| |dd kr8n| |dd<| ||d krb| |t | | \}}!||}||d<|d krt |||!|d<| |g} g}g} g}|}nb| ||d krt |||d }"|dt |krt ||d|d }#nd}#| |"|#fqd |kr<|||7}|rb| ||d kr| |qNtdd| DsN|}g} g}g} g}qN| r|rtdt | | \}}!||}||d<|d krt |||!|d<| |ddd|D}$|s|r|D]8}|s"|dnt|d|d<|s |dq |d krxg}%|D]}|%|dqXd|%i}&nd|i}&ni}&|$|&fS)z Internal method meant to only be used by asr pipeline. Handles all the little quirks specific to whisper to handle the various options not allowed in other seq2seq models NcsddgddS)Nr)rr'rrrZ last_languagerr new_chunkDsz_decode_asr..new_chunkrr FrDrr)token_timestampsZstriderrrTr'wordscss|] }|VqdSrr)rprrrrsz_decode_asr..zWhisper did not predict an ending timestamp, which can happen if audio is cut off in the middle of a word. Also make sure WhisperTimeStampLogitsProcessor was used during generation.rcss|]}|dVqdS)rNr)rchunkrrrrschunks)rr'r"r\tolistreversedrrrr _find_longest_common_sequenceround_collate_word_timestampsranyrPrVrr@rr)' tokenizerrar^r_r$rrryrxZ time_offsetr%Zprevious_tokensZprevious_token_timestampsskipZright_stride_startr"Zchunk_idoutputr#rtZlast_timestampZfirst_timestampZ chunk_lenZ stride_leftZ stride_rightrcurrent_tokensZcurrent_token_timestampsrrrZresolved_tokensZ resolved_texttimeZresolved_token_timestamps start_timeend_time full_textZ new_chunksoptionalrrqrr`0s                                    r`cCs|d}t|}g}|r$|d}g}t|ddD]^\}}d} ||ddf} t|} td|| D]} | d} td|| }t||| | }t|||}td| |}t| | }t|||}t|t|krtdt||k}|| | }|dkrd|| krd|} ||||f} qd| \}}}}||d}||d}| |d|||d}t|}|r4| |d|||d|d}q4| ||dkr|St|dkr| |||fS|gfSdS)Nrr rsg@ziThere is a bug within whisper `decode_asr` function, please report it. Dropping to prevent bad inference.r) rr\rmaxrr*r+ RuntimeErrorr.r) sequencesZtoken_timestamp_sequencesZ left_sequenceZ left_lengthZtotal_sequenceZleft_token_timestamp_sequenceZtotal_token_timestamp_sequenceZseq_idxZright_sequenceZmax_Z max_indicesZ right_lengthrZepsZ left_startZ left_stopleftZ right_startZ right_stoprightmatchesZmatchingZleft_midZ right_midrrrr|sV          r|cs.t|||\}}}fddt||D}|S)Ncs6g|].\}}||dd|ddfdqS)rrr r)r)rr)indicesrtrrr|sz,_collate_word_timestamps..)_combine_tokens_into_wordsr")rrDrtrrv_ token_indicesZtimingsrrrr~zs  r~ "'“¡¿([{-"'.。,，!！?？:：”)]}、)rDrprepend_punctuationsappend_punctuationscCs^|dkr|j}|dkrd}|dkr4t||\}}}nt||\}}}t||||||||fS)z Groups tokens by word. Returns a tuple containing a list of strings with the words, and a list of `token_id` sequences with the tokens making up each word. Nr->rEr4rRr.rP)r_split_tokens_on_unicode_split_tokens_on_spaces_merge_punctuations)rrDrrrrv word_tokensrrrrrs r)rDc Cs|j|dd}d}g}g}g}g}g}d} t|D]z\} } || || |j|dd} || ksz|| | ||kr2|| ||||g}g}| t| 7} q2|||fS)zlCombine tokens into words by splitting at any position where the tokens are decoded as valid unicode points.T)r>u�r)rr\r rr) rrDZ decoded_fullreplacement_charrvrrrZcurrent_indicesZunicode_offsetZ token_idxrdecodedrrrrs.     rcCst||\}}}g}g}g}t|||D]\}} } | d|jk} |d} |dk} | sn| sn| snt|dkr|||| || q(|d||d<|d| |d| q(|||fS)zLCombine tokens into words by splitting at whitespace and punctuation tokens.rrz !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~r)rr"r  startswithrcrr r)rrDZsubwordsZsubword_tokens_listZsubword_indices_listrvrrZsubwordZsubword_tokensZsubword_indicesrZ with_space punctuationrrrrs      rcCst|d}t|d}|dkr||dr|||kr||||||<||||||<||||||<d||<g||<g||<n|}|d8}qd}d}|t|kr:||ds,|||kr,||||7<||||7<||||7<d||<g||<g||<n|}|d7}qdd|D|dd<d d|D|dd<d d|D|dd<dS) z1Merges punctuation tokens with neighboring words.rr rrrcSsg|] }|r|qSrr)rr)rrrrsz'_merge_punctuations..NcSsg|] }|r|qSrr)rrrrrrscSsg|] }|r|qSrr)ridxrrrrs)rrrcendswith)rvrDrZ prependedZappendedrrrrrrs6      r)N)Nrr)+rirrM functoolsrtypingrrrrnumpyr*regexrZtokenization_utilsrr utilsr Zenglish_normalizerr rRrjrkr&Z get_loggerrfrPr,rrrr rr`r|r~rmrrrrrrrrrsF      h]b i