U ,-e5@sdZddlZddlmZmZmZddlmZddlm Z ddl m Z e e Zdd iZdd d d iZd d d ZddZGdddeZdS)zTokenization classes for ESM.N)ListOptionalUnion)PreTrainedTokenizer) AddedToken)logging vocab_file vocab.txtzGhttps://huggingface.co/facebook/esm2_t6_8M_UR50D/resolve/main/vocab.txtzIhttps://huggingface.co/facebook/esm2_t12_35M_UR50D/resolve/main/vocab.txt)zfacebook/esm2_t6_8M_UR50Dzfacebook/esm2_t12_35M_UR50Dic Cs<t|d(}|}dd|DW5QRSQRXdS)NrcSsg|] }|qS)strip).0lr r i/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/transformers/models/esm/tokenization_esm.py ,sz#load_vocab_file..)openread splitlines)r flinesr r rload_vocab_file)s  rcseZdZdZeZeZeZ ddgZ d+fd d Z e e d d d Ze e dddZddZd,ddZddZe e dddZe e d ddZd-ee eee ee dddZd.eeeeee d d!d"Zd#d$Zee d%d&d'Zd/eee eefee d(fd)d* ZZ S)0 EsmTokenizerz& Constructs an ESM tokenizer. Z input_idsZattention_maskc sft||_tt|j|_ddt|jD|_tjf|||||d||j|_| |jdS)NcSsi|]\}}||qSr r )rindtokr r r Esz)EsmTokenizer.__init__..) unk_token cls_token pad_token mask_token eos_token) r all_tokensdict enumerate _id_to_token _token_to_idsuper__init__Zunique_no_split_tokensZ _update_trie)selfr r!r"r#r$r%kwargs __class__r rr,9s  zEsmTokenizer.__init__)indexreturncCs|j||jSNr)getr!r-r1r r r_convert_id_to_tokenUsz!EsmTokenizer._convert_id_to_token)tokenr2cCs|j||j|jSr3r*r5r!r-r8r r r_convert_token_to_idXsz!EsmTokenizer._convert_token_to_idcKs|Sr3)split)r-textr.r r r _tokenize[szEsmTokenizer._tokenizeFcCs t|jSr3)lenr))r-with_added_tokensr r rget_vocab_size^szEsmTokenizer.get_vocab_sizecCsddt|jDS)NcSsi|]\}}||qSr r )rir8r r rr bsz*EsmTokenizer.get_vocab..)r(r&r-r r r get_vocabaszEsmTokenizer.get_vocabcCs|j||j|jSr3r9r:r r r token_to_iddszEsmTokenizer.token_to_idcCs|j||jSr3r4r6r r r id_to_tokengszEsmTokenizer.id_to_tokenN) token_ids_0 token_ids_1r2cCs^|jg}|jg}|dkr8|jdkr*||S|||Sn|jdkrJtd|||||S)Nz=Cannot tokenize multiple sequences when EOS token is not set!)Z cls_token_idZ eos_token_id ValueError)r-rGrHclssepr r r build_inputs_with_special_tokensjs  z-EsmTokenizer.build_inputs_with_special_tokens)rGrHalready_has_special_tokensr2csd|r&|dk rtdfdd|DSdgdgt|dg}|dk r`|dgt|dg7}|S)a Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. Args: token_ids_0 (`List[int]`): List of ids of the first sequence. token_ids_1 (`List[int]`, *optional*): List of ids of the second sequence. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. NzYou should not supply a second sequence if the provided sequence of ids is already formatted with special tokens for the model.csg|]}|jkrdndqS)r)Zall_special_ids)rr8rCr rrsz8EsmTokenizer.get_special_tokens_mask..rNr)rIr?)r-rGrHrMmaskr rCrget_special_tokens_maskxsz$EsmTokenizer.get_special_tokens_maskc CsLtj||r|dndd}t|d}|d|jW5QRX|fS)N-r w )ospathjoinrwriter&)r-Zsave_directoryZfilename_prefixr rr r rsave_vocabularys zEsmTokenizer.save_vocabulary)r2cCs |jddS)NF)r@)rArCr r r vocab_sizeszEsmTokenizer.vocab_size) new_tokensspecial_tokensr2cstj|ddS)NT)r\)r+ _add_tokens)r-r[r\r/r rr]szEsmTokenizer._add_tokens)rrrrr)F)N)NF)F)!__name__ __module__ __qualname____doc__VOCAB_FILES_NAMESZvocab_files_namesPRETRAINED_VOCAB_FILES_MAPZpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESZmax_model_input_sizesZmodel_input_namesr,intstrr7r;r>rArDrErFrrrLboolrPrYpropertyrZrrr] __classcell__r r r/rr/sH    r)rarUtypingrrrZtokenization_utilsrZtokenization_utils_baserutilsrZ get_loggerr^loggerrbrcrdrrr r r rs"