U 9%e&@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z m Z mZmZmZmZmZmZddlZddlmZmZmZmZddlmZmZmZmZm Z m!Z!ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(e'rddl)Z*e(+e,Z-d d iZ.d d d d ddddddddddddddddddd d!d"d#iZ/d$d%e/0DZ1d&d%e/0DZ2Gd'd(d(e#Z3e4d)d*d+d,gZ5e d-d.Gd/d0d0Z6e Gd1d2d2Z7e d-d.Gd3d4d4Z8ed5d6d7Z9d8d9Z:d:d;Z;dd>eZ=Gd?d@d@e>Z?GdAdBdBe>Z@GdCdDdDejAZBe GdEdFdFZCe GdGdHdHZDe GdIdJdJZEe GdKdLdLZFe GdMdNdNZGe4dOdPdQdRgZHeHd-dSdSZIeHd-d-dSZJeHd-d-d-ZKeHdSd-dSZLeHdSd-d-ZMdTeLfdUeIfdVeIfdWeJfdXeJfdYeMfdZeMfd[eMfd\eMfd]eKfd^eKfd_eKfd`eKfdaeJfdXeJfdbeKfd`eKfdceKfddeKfdeeMfdfeMffZNdgZOdhdiZPdjdkZQeQZRdlZSdmdndodpdqdrdsdtdudvdwdxdyg ZTdzd{d|d}d~ddddddddg ZUddddgZVeWdZXdZYdZZe[dZ\ddZ]ddZ^ddZ_ddZ`ddZaddZbddZcddZdddZeee[eee[ee[ee[ffZfe eDgee[egffZhdZidZjdZkdZlddZmddZnddZoddZpddZqddZrddZsddZtddZuddZvddZwdddZxdS)z$ Tokenization class for TAPAS model.N) dataclass)CallableDict GeneratorListOptionalTextTupleUnion)PreTrainedTokenizer _is_control_is_punctuation_is_whitespace)ENCODE_KWARGS_DOCSTRINGVERY_LARGE_INTEGER BatchEncoding EncodedInputPreTokenizedInput TextInput) ExplicitEnumPaddingStrategy TensorTypeadd_end_docstringsis_pandas_availablelogging vocab_filez vocab.txtzNhttps://huggingface.co/google/tapas-large-finetuned-sqa/resolve/main/vocab.txtzNhttps://huggingface.co/google/tapas-large-finetuned-wtq/resolve/main/vocab.txtz]https://huggingface.co/google/tapas-large-finetuned-wikisql-supervised/resolve/main/vocab.txtzRhttps://huggingface.co/google/tapas-large-finetuned-tabfact/resolve/main/vocab.txtzMhttps://huggingface.co/google/tapas-base-finetuned-sqa/resolve/main/vocab.txtzMhttps://huggingface.co/google/tapas-base-finetuned-wtq/resolve/main/vocab.txtz\https://huggingface.co/google/tapas-base-finetuned-wikisql-supervised/resolve/main/vocab.txtzQhttps://huggingface.co/google/tapas-base-finetuned-tabfact/resolve/main/vocab.txtzOhttps://huggingface.co/google/tapas-medium-finetuned-sqa/resolve/main/vocab.txtzOhttps://huggingface.co/google/tapas-medium-finetuned-wtq/resolve/main/vocab.txtz^https://huggingface.co/google/tapas-medium-finetuned-wikisql-supervised/resolve/main/vocab.txtzShttps://huggingface.co/google/tapas-medium-finetuned-tabfact/resolve/main/vocab.txtzNhttps://huggingface.co/google/tapas-small-finetuned-sqa/resolve/main/vocab.txtzNhttps://huggingface.co/google/tapas-small-finetuned-wtq/resolve/main/vocab.txtz]https://huggingface.co/google/tapas-small-finetuned-wikisql-supervised/resolve/main/vocab.txtzRhttps://huggingface.co/google/tapas-small-finetuned-tabfact/resolve/main/vocab.txtzMhttps://huggingface.co/google/tapas-tiny-finetuned-sqa/resolve/main/vocab.txtzMhttps://huggingface.co/google/tapas-tiny-finetuned-wtq/resolve/main/vocab.txtz\https://huggingface.co/google/tapas-tiny-finetuned-wikisql-supervised/resolve/main/vocab.txtzQhttps://huggingface.co/google/tapas-tiny-finetuned-tabfact/resolve/main/vocab.txtzMhttps://huggingface.co/google/tapas-mini-finetuned-sqa/resolve/main/vocab.txtzMhttps://huggingface.co/google/tapas-mini-finetuned-wtq/resolve/main/vocab.txtz\https://huggingface.co/google/tapas-mini-finetuned-wikisql-supervised/resolve/main/vocab.txtzQhttps://huggingface.co/google/tapas-mini-finetuned-tabfact/resolve/main/vocab.txt)z google/tapas-large-finetuned-sqaz google/tapas-large-finetuned-wtqz/google/tapas-large-finetuned-wikisql-supervisedz$google/tapas-large-finetuned-tabfactzgoogle/tapas-base-finetuned-sqazgoogle/tapas-base-finetuned-wtqz.google/tapas-base-finetuned-wikisql-supervisedz#google/tapas-base-finetuned-tabfactz!google/tapas-medium-finetuned-sqaz!google/tapas-medium-finetuned-wtqz0google/tapas-medium-finetuned-wikisql-supervisedz%google/tapas-medium-finetuned-tabfactz google/tapas-small-finetuned-sqaz google/tapas-small-finetuned-wtqz/google/tapas-small-finetuned-wikisql-supervisedz$google/tapas-small-finetuned-tabfactzgoogle/tapas-tiny-finetuned-sqazgoogle/tapas-tiny-finetuned-wtqz.google/tapas-tiny-finetuned-wikisql-supervisedz#google/tapas-tiny-finetuned-tabfactzgoogle/tapas-mini-finetuned-sqazgoogle/tapas-mini-finetuned-wtqz.google/tapas-mini-finetuned-wikisql-supervisedz#google/tapas-mini-finetuned-tabfactcCsi|] }|dqS).0namerrk/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/models/tapas/tokenization_tapas.py sr#cCsi|]}|ddiqS) do_lower_caseTrrrrr"r#sc@seZdZdZdZdZdS)TapasTruncationStrategyz} Possible values for the `truncation` argument in [`~TapasTokenizer.__call__`]. Useful for tab-completion in an IDE. drop_rows_to_fitZdo_not_truncateN)__name__ __module__ __qualname____doc__DROP_ROWS_TO_FITDO_NOT_TRUNCATErrrr"r%sr%Z TokenValuetoken column_idrow_idT)frozenc@s&eZdZUeed<eed<eed<dS)TokenCoordinates column_index row_index token_indexN)r'r(r)int__annotations__rrrr"r1s r1c@s.eZdZUeeeeed<eeed<dS)TokenizedTablerowsselected_tokensN)r'r(r)rrr6r1rrrr"r7s r7c@s>eZdZUeeed<eeed<eeed<eeed<dS)SerializedExampletokens column_idsrow_ids segment_idsN)r'r(r)rrr6r5rrrr"r:s    r:r-cCs |dS)N##) startswithr?rrr"_is_inner_wordpiecesrBc CsRt}t|ddd}|}W5QRXt|D]\}}|d}|||<q2|S)z*Loads a vocabulary file into a dictionary.rutf-8encoding ) collections OrderedDictopen readlines enumeraterstrip)rvocabreaderr;indexr-rrr" load_vocabs  rQcCs|}|sgS|}|S)z@Runs basic whitespace cleaning and splitting on a piece of text.)stripsplit)textr;rrr"whitespace_tokenizes rUa add_special_tokens (`bool`, *optional*, defaults to `True`): Whether or not to encode the sequences with the special tokens relative to their model. padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`): Activates and controls padding. Accepts the following values: - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence if provided). - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum acceptable input length for the model if that argument is not provided. - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different lengths). truncation (`bool`, `str` or [`TapasTruncationStrategy`], *optional*, defaults to `False`): Activates and controls truncation. Accepts the following values: - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum acceptable input length for the model if that argument is not provided. This will truncate row by row, removing rows from the table. - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths greater than the model maximum admissible input size). max_length (`int`, *optional*): Controls the maximum length to use by one of the truncation/padding parameters. If left unset or set to `None`, this will use the predefined model maximum length if a maximum length is required by one of the truncation/padding parameters. If the model has no specific maximum input length (like XLNet) truncation/padding to a maximum length will be deactivated. is_split_into_words (`bool`, *optional*, defaults to `False`): Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) which it will tokenize. This is useful for NER or token classification. pad_to_multiple_of (`int`, *optional*): If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta). return_tensors (`str` or [`~utils.TensorType`], *optional*): If set, will return tensors instead of list of python integers. Acceptable values are: - `'tf'`: Return TensorFlow `tf.constant` objects. - `'pt'`: Return PyTorch `torch.Tensor` objects. - `'np'`: Return Numpy `np.ndarray` objects. csVeZdZdZeZeZeZ de e e e e e e e ed fdd ZeddZeddZddZddZddZddZddZdee eeeddd Ze e e ee e d!d"d#Ze e e ee e d!d$d%Ze e e ee e d!d&d'Ze e e ee e d!d(d)Zde e e e e e e d*d+d,Zde e e e e e e e d-fd.d/ Z e!e"dd0e e#e$e%e&e e$e e%e e&fe e#e ee e efe e#e e$e e e$fe e#e ee'fe#e ee(fe e e e e e#ee)fe e e e e e e e e e*d1d2d3Z+e!e,e"dd0e e#e e$e e%e e&fe e e ee e e e$e e#e ee'fe#e ee(fe e e e e e#ee)fe e e e e e e e e e*d1d4d5Z-d6d7Z.de#e e$e e%e e&fe e e ee e e e$e e#e ee'fe#e ee(fe e e e e e#ee)fe e e e e e e e e e*d8d9d:Z/dd0e#e e$e e%e e&fe e0e e e ee e e ee e e e$e e#e ee'fe#e ee(fe e e e e e#ee)fe e e e e e e e e e*d;dd?d@Z2e!e,e"dd0e e#e$e%e&fe e ee e e$e e#e ee'fe#e ee(fe e e e e e#ee)fe e e e e e e e e*dAdBdCZ3dd0e#e$e%e&fe e ee e e$e e#e ee'fe#e ee(fe e e e e e#ee)fe e e e e e e e dDdEdFZ4e!e,e"dd0e#e$e%e&fe e0e e0e e ee e e$e e#e ee'fe#e ee(fe e e e e e#ee)fe e e e e e e e e e*dGdHdIZ5e ee0e e e e#ee(fee e fdJdKdLZ6ddMdNZ7dOdPZ8ddQdRZ9e:eddfdSdTdUZ;dVdWZd\d]Z?d^d_Z@d`daZAdbdcZBdddeZCdfdgZDdhdiZEdjdkZFdldmZGdndoZHdpdqZIdrdsZJdtduZKdvdwZLdxdyZMdzd{ZNd|d}ZOd~dZPddZQde'jRddfe#eSee&fe*fe e e'e e e e eTdddZUddZVddZWdddZXZYS)TapasTokenizera Construct a TAPAS tokenizer. Based on WordPiece. Flattens a table and one or more related sentences to be used by TAPAS models. This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. [`TapasTokenizer`] creates several token type ids to encode tabular structure. To be more precise, it adds 7 token type ids, in the following order: `segment_ids`, `column_ids`, `row_ids`, `prev_labels`, `column_ranks`, `inv_column_ranks` and `numeric_relations`: - segment_ids: indicate whether a token belongs to the question (0) or the table (1). 0 for special tokens and padding. - column_ids: indicate to which column of the table a token belongs (starting from 1). Is 0 for all question tokens, special tokens and padding. - row_ids: indicate to which row of the table a token belongs (starting from 1). Is 0 for all question tokens, special tokens and padding. Tokens of column headers are also 0. - prev_labels: indicate whether a token was (part of) an answer to the previous question (1) or not (0). Useful in a conversational setup (such as SQA). - column_ranks: indicate the rank of a table token relative to a column, if applicable. For example, if you have a column "number of movies" with values 87, 53 and 69, then the column ranks of these tokens are 3, 1 and 2 respectively. 0 for all question tokens, special tokens and padding. - inv_column_ranks: indicate the inverse rank of a table token relative to a column, if applicable. For example, if you have a column "number of movies" with values 87, 53 and 69, then the inverse column ranks of these tokens are 1, 3 and 2 respectively. 0 for all question tokens, special tokens and padding. - numeric_relations: indicate numeric relations between the question and the tokens of the table. 0 for all question tokens, special tokens and padding. [`TapasTokenizer`] runs end-to-end tokenization on a table and associated sentences: punctuation splitting and wordpiece. Args: vocab_file (`str`): File containing the vocabulary. do_lower_case (`bool`, *optional*, defaults to `True`): Whether or not to lowercase the input when tokenizing. do_basic_tokenize (`bool`, *optional*, defaults to `True`): Whether or not to do basic tokenization before WordPiece. never_split (`Iterable`, *optional*): Collection of tokens which will never be split during tokenization. Only has an effect when `do_basic_tokenize=True` unk_token (`str`, *optional*, defaults to `"[UNK]"`): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. sep_token (`str`, *optional*, defaults to `"[SEP]"`): The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. pad_token (`str`, *optional*, defaults to `"[PAD]"`): The token used for padding, for example when batching sequences of different lengths. cls_token (`str`, *optional*, defaults to `"[CLS]"`): The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. mask_token (`str`, *optional*, defaults to `"[MASK]"`): The token used for masking values. This is the token used when training this model with masked language modeling. This is the token which the model will try to predict. empty_token (`str`, *optional*, defaults to `"[EMPTY]"`): The token used for empty cell values in a table. Empty cell values include "", "n/a", "nan" and "?". tokenize_chinese_chars (`bool`, *optional*, defaults to `True`): Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)). strip_accents (`bool`, *optional*): Whether or not to strip all accents. If this option is not specified, then it will be determined by the value for `lowercase` (as in the original BERT). cell_trim_length (`int`, *optional*, defaults to -1): If > 0: Trim cells so that the length is <= this value. Also disables further cell trimming, should thus be used with `truncation` set to `True`. max_column_id (`int`, *optional*): Max column id to extract. max_row_id (`int`, *optional*): Max row id to extract. strip_column_names (`bool`, *optional*, defaults to `False`): Whether to add empty strings instead of column names. update_answer_coordinates (`bool`, *optional*, defaults to `False`): Whether to recompute the answer coordinates from the answer text. min_question_length (`int`, *optional*): Minimum length of each question in terms of tokens (will be skipped otherwise). max_question_length (`int`, *optional*): Maximum length of each question in terms of tokens (will be skipped otherwise). TN[UNK][SEP][PAD][CLS][MASK][EMPTY]Fr)cell_trim_length max_column_id max_row_idstrip_column_namesupdate_answer_coordinatesmodel_max_lengthadditional_special_tokensc s<tstd|dk r*| |kr0|| n| g}tj|sLtd|dt||_t dd|j D|_ ||_ |rt||| | d|_t|jt|d|_| |_|dk r|n|dk r|nt|_|dk r|n|dk r|nt|_||_||_||_||_tjf|||||||| | | | | ||||||||d|dS) Nz+Pandas is required for the TAPAS tokenizer.z&Can't find a vocabulary file at path 'z'. To load the vocabulary from a Google pretrained model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`cSsg|]\}}||fqSrr)r tokidsrrr" isz+TapasTokenizer.__init__..)r$ never_splittokenize_chinese_chars strip_accents)rN unk_token)r$do_basic_tokenizerhrk sep_token pad_token cls_token mask_token empty_tokenrirjr^r_r`rarbmin_question_lengthmax_question_lengthrcrd)r ImportErrorappendospathisfile ValueErrorrQrNrHrIitems ids_to_tokensrlBasicTokenizerbasic_tokenizerWordpieceTokenizerstrwordpiece_tokenizerr^rr_r`rarbrrrssuper__init__)selfrr$rlrhrkrmrnrorprqrirjr^r_r`rarbrrrsrcrdkwargs __class__rr"rAs|     zTapasTokenizer.__init__cCs|jjSN)r}r$rrrr"r$szTapasTokenizer.do_lower_casecCs t|jSr)lenrNrrrr" vocab_sizeszTapasTokenizer.vocab_sizecCst|jf|jSr)dictrNZadded_tokens_encoderrrrr" get_vocabszTapasTokenizer.get_vocabcCstt|tkr|jdgSg}|jrd|jj||jdD],}||jjkrP||q4||j |7}q4n |j |}|S)Nr)rh) format_text EMPTY_TEXTrdrlr}tokenizeZall_special_tokensrhrur)rrT split_tokensr-rrr" _tokenizes     zTapasTokenizer._tokenizecCs|j||j|jS)z0Converts a token (str) in an id using the vocab.)rNgetrk)rr-rrr"_convert_token_to_idsz#TapasTokenizer._convert_token_to_idcCs|j||jS)z=Converts an index (integer) in a token (str) using the vocab.)r{rrk)rrPrrr"_convert_id_to_tokensz#TapasTokenizer._convert_id_to_tokencCsd|dd}|S)z:Converts a sequence of tokens (string) in a single string. z ##)joinreplacerR)rr;Z out_stringrrr"convert_tokens_to_stringsz'TapasTokenizer.convert_tokens_to_string)save_directoryfilename_prefixreturnc Csd}tj|r4tj||r$|dndtd}n|r@|dnd|}t|dddZ}t|jdd d D]<\}}||krt d |d |}| |d |d7}qnW5QRX|fS)Nr-rrwrDrEcSs|dSNr)kvrrr"z0TapasTokenizer.save_vocabulary..keyzSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!rGr) rvrwisdirrVOCAB_FILES_NAMESrJsortedrNrzloggerwarningwrite)rrrrPrwriterr-r4rrr"save_vocabularys"  zTapasTokenizer.save_vocabulary) query_ids table_valuesrcCsdgdt|dt|S)a Creates the attention mask according to the query token IDs and a list of table values. Args: query_ids (`List[int]`): list of token IDs corresponding to the ID. table_values (`List[TableValue]`): lift of table values, which are named tuples containing the token value, the column ID and the row ID of said token. Returns: `List[int]`: List of ints containing the attention mask values. rr)rrrrrr"$create_attention_mask_from_sequencess z3TapasTokenizer.create_attention_mask_from_sequencescCs<|rtt|dng}dgdt|ddgt|S)a Creates the segment token type IDs according to the query token IDs and a list of table values. Args: query_ids (`List[int]`): list of token IDs corresponding to the ID. table_values (`List[TableValue]`): lift of table values, which are named tuples containing the token value, the column ID and the row ID of said token. Returns: `List[int]`: List of ints containing the segment token type IDs values. rrlistzipr)rrr table_idsrrr",create_segment_token_type_ids_from_sequencessz;TapasTokenizer.create_segment_token_type_ids_from_sequencescCs6|rtt|dng}dgdt|dt|S)a Creates the column token type IDs according to the query token IDs and a list of table values. Args: query_ids (`List[int]`): list of token IDs corresponding to the ID. table_values (`List[TableValue]`): lift of table values, which are named tuples containing the token value, the column ID and the row ID of said token. Returns: `List[int]`: List of ints containing the column token type IDs values. rrr)rrrZtable_column_idsrrr"+create_column_token_type_ids_from_sequencessz:TapasTokenizer.create_column_token_type_ids_from_sequencescCs6|rtt|dng}dgdt|dt|S)a Creates the row token type IDs according to the query token IDs and a list of table values. Args: query_ids (`List[int]`): list of token IDs corresponding to the ID. table_values (`List[TableValue]`): lift of table values, which are named tuples containing the token value, the column ID and the row ID of said token. Returns: `List[int]`: List of ints containing the row token type IDs values. rrr)rrrZ table_row_idsrrr"(create_row_token_type_ids_from_sequences sz7TapasTokenizer.create_row_token_type_ids_from_sequences) token_ids_0 token_ids_1rcCs(|dkrtd|jg||jg|S)a Build model inputs from a question and flattened table for question answering or sequence classification tasks by concatenating and adding special tokens. Args: token_ids_0 (`List[int]`): The ids of the question. token_ids_1 (`List[int]`, *optional*): The ids of the flattened table. Returns: `List[int]`: The model input with special tokens. Nz=With TAPAS, you must provide both question IDs and table IDs.)ryZ cls_token_idZ sep_token_id)rrrrrr" build_inputs_with_special_tokenssz/TapasTokenizer.build_inputs_with_special_tokens)rralready_has_special_tokensrcs`|rtj||ddS|dk rFdgdgt|dgdgt|Sdgdgt|dgS)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`List[int]`): List of question IDs. token_ids_1 (`List[int]`, *optional*): List of flattened table IDs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)rrrNrr)rget_special_tokens_maskr)rrrrrrr"r0s(z&TapasTokenizer.get_special_tokens_maskz pd.DataFrame)tablequeriesanswer_coordinates answer_textadd_special_tokenspadding truncation max_lengthpad_to_multiple_ofreturn_tensorsreturn_token_type_idsreturn_attention_maskreturn_overflowing_tokensreturn_special_tokens_maskreturn_offsets_mapping return_lengthverbosercKst|tjstdd}|dks*t|tr0d}n,t|ttfr\t|dksXt|dtr\d}|shtdt|ttf}|r|j f||||||||| | | | | ||||d|S|j f||||||||| | | | | ||||d|SdS) a Main method to tokenize and prepare for the model one or several sequence(s) related to a table. Args: table (`pd.DataFrame`): Table containing tabular data. Note that all cell values must be text. Use *.astype(str)* on a Pandas dataframe to convert it to string. queries (`str` or `List[str]`): Question or batch of questions related to a table to be encoded. Note that in case of a batch, all questions must refer to the **same** table. answer_coordinates (`List[Tuple]` or `List[List[Tuple]]`, *optional*): Answer coordinates of each table-question pair in the batch. In case only a single table-question pair is provided, then the answer_coordinates must be a single list of one or more tuples. Each tuple must be a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The first column has index 0. In case a batch of table-question pairs is provided, then the answer_coordinates must be a list of lists of tuples (each list corresponding to a single table-question pair). answer_text (`List[str]` or `List[List[str]]`, *optional*): Answer text of each table-question pair in the batch. In case only a single table-question pair is provided, then the answer_text must be a single list of one or more strings. Each string must be the answer text of a corresponding answer coordinate. In case a batch of table-question pairs is provided, then the answer_coordinates must be a list of lists of strings (each list corresponding to a single table-question pair). z"Table must be of type pd.DataFrameFNTrzgqueries input must of type `str` (single example), `List[str]` (batch or single pretokenized example). rrrrrrrrrrrrrrrrr)rqueryrrrrrrrrrrrrrrr) isinstancepdZ DataFrameAssertionErrorrrtuplerrybatch_encode_plus encode_plus)rrrrrrrrrrrrrrrrrrrZ valid_queryZ is_batchedrrr"__call__Lsn7zTapasTokenizer.__call__cKs| dk r|std|r|r$|s.|r.tdn"|dkrP|dkrPdgt|}}d|kr`td|rltd|jf||||||||| | | | | ||||d|S)a Prepare a table and a list of strings for the model. This method is deprecated, `__call__` should be used instead. Args: table (`pd.DataFrame`): Table containing tabular data. Note that all cell values must be text. Use *.astype(str)* on a Pandas dataframe to convert it to string. queries (`List[str]`): Batch of questions related to a table to be encoded. Note that all questions must refer to the **same** table. answer_coordinates (`List[Tuple]` or `List[List[Tuple]]`, *optional*): Answer coordinates of each table-question pair in the batch. Each tuple must be a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The first column has index 0. The answer_coordinates must be a list of lists of tuples (each list corresponding to a single table-question pair). answer_text (`List[str]` or `List[List[str]]`, *optional*): Answer text of each table-question pair in the batch. In case a batch of table-question pairs is provided, then the answer_coordinates must be a list of lists of strings (each list corresponding to a single table-question pair). Each string must be the answer text of a corresponding answer coordinate. NAsking to return token_type_ids while setting add_special_tokens to False results in an undefined behavior. Please set add_special_tokens to True or set return_token_type_ids to None.WIn case you provide answers, both answer_coordinates and answer_text should be providedis_split_into_words.special_tokens_maskz4sequence-length-is-longer-than-the-specified-maximumzcToken indices sequence length is longer than the specified maximum sequence length for this model (z > zJ). Running this sequence through the model will result in indexing errors.)rrrrlength)rr)(rboolrZ MAX_LENGTHrr%r+r, _get_num_rows_get_num_columns_get_table_boundaries_get_truncated_table_rowsr_get_table_valuesZconvert_tokens_to_idsrrryrrrrget_answer_idsadd_numeric_table_valuesadd_numeric_values_to_question_get_numeric_column_ranks_get_numeric_relationsmodel_input_namesr_get_numeric_values_get_numeric_values_scalerrcZdeprecation_warningsrrrrrr).rrrrrrrrrrrrrrrrrrrrrrZis_part_of_batchrrnum_rows num_columns_ num_tokensZ table_datarrrr>r<r= prev_labels column_ranksinv_column_ranksnumeric_relationsrrrrrrrrr"res1        *         $ z TapasTokenizer.prepare_for_model)rrrrrrrcCst|tst|}|dkr |j}|tjkr^|j|||||d}|dk rHqx|d8}|dkr*qxq*n|tjkrxtd|d||pdfS)a Truncates a sequence pair in-place following the strategy. Args: query_tokens (`List[str]`): List of strings corresponding to the tokenized query. tokenized_table (`TokenizedTable`): Tokenized table num_rows (`int`): Total number of table rows num_columns (`int`): Total number of table columns max_length (`int`): Total maximum length. truncation_strategy (`str` or [`TapasTruncationStrategy`]): Truncation strategy to use. Seeing as this method should only be called when truncating, the only available strategy is the `"drop_rows_to_fit"` strategy. Returns: `Tuple(int, int)`: tuple containing the number of rows after truncation, and the number of tokens available for each table element. N)rrrrzUnknown truncation strategy .)rr%rcr+_get_max_num_tokensr,ry)rrrrrrrrrrr"r&s(   z(TapasTokenizer._get_truncated_table_rowsc Csg}g}|D],}|jr(||dq |||q |||D]0\}}g}|D]}|||q\||qLg}t|D]@\} }t|D].\} }t|D]\} } |t| | | dqqqt||dS)z Tokenizes column headers and cell texts of a table. Args: table (`pd.Dataframe`): Table. Returns: `TokenizedTable`: TokenizedTable object. rr3r2r4)r8r9)raruriterrowsrLr1r7) rrZtokenized_rowsZ tokenized_rowcolumnrrowcellZtoken_coordinatesr3r2r4rrrr"r_s6    zTapasTokenizer._tokenize_tablecCs t|dS)Nrr)rquestion_tokensrrr"_question_encoding_costsz&TapasTokenizer._question_encoding_costcCs|dk r |n|j||S)al Computes the number of tokens left for the table after tokenizing a question, taking into account the max sequence length of the model. Args: question_tokens (`List[String]`): List of question tokens. Returns: `int`: the number of tokens left for the table, given the model max length. N)rcr)rrrrrr"_get_token_budgets z TapasTokenizer._get_token_budget)rc cs|jD]}|j|dkrq|j|kr&q|j|j|j}||j}|j}|dkrft||rf|d8}qH||krpqt||jd|jVqdS)zFIterates over partial table and returns token, column and row indexes.rrN)r9r3r2r8r4rB TableValue) rrrrrtcrr-Zword_begin_indexrrr"rs    z TapasTokenizer._get_table_valuescCsjd}d}d}|jD]L}t||jd}t||jd}t||jd}t|j|}t|j|}q|||fS)z2Return maximal number of rows, columns and tokens.rr)r9maxr2r3r4minr_r`)rrmax_num_tokensZmax_num_columnsZ max_num_rowsr"rrr"rs  z$TapasTokenizer._get_table_boundariescCstdd|||||DS)Ncss|] }dVqdS)rNr)r rrrr" sz1TapasTokenizer._get_table_cost..)sumr)rrrrrrrr"_get_table_costszTapasTokenizer._get_table_costc Cs|||}||\}}}|jdkr6||jkr6|j}d} t|dD]$} ||||| d} | |krFqlqF| |kr|jdkrdS| dkrdS| S)zCComputes max number of tokens that can be squeezed into the budget.rrN)r rr^ranger() rrrrrrZ token_budgetrr%rZcostrrr"rs  z"TapasTokenizer._get_max_num_tokenscCs |jd}||jkrtd|S)NrzToo many columns)shaper_ry)rrrrrr"rs  zTapasTokenizer._get_num_columnscCs0|jd}||jkr,|r$|jd}ntd|S)Nrrz Too many rows)r*r`ry)rrr&rrrr"rs    zTapasTokenizer._get_num_rowscCsxg}g}g}g}||j|d|d|d|D],}|||d|d|dq>||||fS)z!Serializes texts in index arrays.r)ruro)rrr;r>r<r=r-rrr"_serialize_texts        zTapasTokenizer._serialize_textc Cs||\}}}} ||j|d|d| d|||||D]2\} } } || |d|| | | qLt|||| dS)zSerializes table and text.rr)r;r>r<r=)r+rurmrr:) rrrrrrr;r>r<r=r-r.r/rrr" _serializes         zTapasTokenizer._serializecCs6i}|D]$\}}||}|jdk r |j||<q |Sr)r numeric_value)rr col_indextable_numeric_valuesr3rrrrr"_get_column_valuess   z!TapasTokenizer._get_column_valuesccs<tt|D]*}||d|kr ||d|kr |Vq dSr)r)r)rr<r=r.r/rPrrr"_get_cell_token_indexess z&TapasTokenizer._get_cell_token_indexesc sdgt|}dgt|}|dk rtt|jD]}|||}|sJq4zt|Wntk rrYq4YnXfdd|D}t t }|D]\} } ||  | qt | } t| D]H\} } || D]6} ||||| D] } | d|| <t| | || <qqqq4||fS)z-Returns column ranks for all numeric columns.rNcsi|]\}}||qSrr)r r3rZkey_fnrr"r#6sz.r)rr)columnsr0get_numeric_sort_key_fnvaluesryrzrH defaultdictrrurkeysrLr1)rr<r=rZranksZ inv_ranksr.r/Ztable_numeric_values_invr3rZ unique_valuesZrankrPrr2r"r #s,       z(TapasTokenizer._get_numeric_column_rankscCsD|sdSt|}||z t|WStk r>YdSXdS)a Returns the sort key function for comparing value to table values. The function returned will be a suitable input for the key param of the sort(). See number_annotation_utils._get_numeric_sort_key_fn for details Args: table_numeric_values: Numeric values of a column value: Numeric value in the question Returns: A function key function to compare column and question values. N)rr5rur4ry)rr/rZ all_valuesrrr"_get_numeric_sort_key_fnFs    z'TapasTokenizer._get_numeric_sort_key_fncCsdgt|}tt}|dk r|dk r|jD]~}|jD]r}tt|jD]^} ||| } | | |} | dkrpqJ| D].\} } t || | }|dk rx|| | f |qxqJq8q.| D]b\\} } }d}|D],}|j tjj kst|d|j tjj 7}q|||| | D]}|||<qq|S)a% Returns numeric relations embeddings Args: question: Question object. column_ids: Maps word piece position to column id. row_ids: Maps word piece position to row id. table: The table containing the numeric cell values. rNr)rrHr6set numeric_spansr5r)r3r0r8rzget_numeric_relationaddrRelationEQrr1)rquestionr<r=rrZcell_indices_to_relationsZnumeric_value_spanrr2r/ sort_key_fnr3Z cell_valueZrelationZ relationsZrelation_set_indexZcell_token_indexrrr"r [s,       z%TapasTokenizer._get_numeric_relationsc Cstdgt|}|dk r|jd}|jd}t|D]h}t|D]Z}|j||fj} | dk rB| jdkrjqB| j} | tdkr~qB|||||D] } | || <qqBq6|S)z6Returns numeric values for computation of answer loss.nanNrrinf)floatrr*r)ilocr- float_valuer1) rrr<r=rrrr.r3r-rErPrrr"r s       z"TapasTokenizer._get_numeric_valuesc Csdgt|}|dkr|S|jd}|jd}t|D]L}t|D]>}t|||||} t| } | dkrB| D]} t| || <qnqBq6|S)zDReturns a scale to each token to down weigh the value of long words.?Nrr)rr*r)rr1rC) rrr<r=rrrr.r3indicesZ num_indicesrPrrr"rs    z(TapasTokenizer._get_numeric_values_scalecCs6t||jkr|qt||jkr2|dqdSNr)rrcpopru)rinputsrrr"_pad_to_seq_lengths z!TapasTokenizer._pad_to_seq_lengthc Cs~dgt|}t}t}|D]F}|\}} ||| f||||| D]} ||| fd|| <qHqt|t|} || fS)z2Maps lists of answer coordinates to token indexes.rr)rr9r<r1) rr<r= answers_list answer_idsZ found_answersZ all_answersZanswersr2r3rP missing_countrrr"$_get_all_answer_ids_from_coordinatess z3TapasTokenizer._get_all_answer_ids_from_coordinatescCsdd}|j||||dS)a  Maps answer coordinates of a question to token indexes. In the SQA format (TSV), the coordinates are given as (row, column) tuples. Here, we first swap them to (column, row) format before calling _get_all_answer_ids_from_coordinates. cSsdd|DS)NcSsg|]}|d|dfqS)rrr)r coordsrrr"rgszOTapasTokenizer._get_all_answer_ids.._to_coordinates..r)answer_coordinates_questionrrr"_to_coordinatessz;TapasTokenizer._get_all_answer_ids.._to_coordinates)rL)rO)rr<r=rrRrrr"_get_all_answer_idss z"TapasTokenizer._get_all_answer_idscCsjtd|d|tdt|t|D]6}t|D] \}}|||j|jkr:q.q:|Sq.dS)z.Return start index of segment in text or None.ztext: rrN)rinfor)rrLZpiece)rrTsegmentrPZ seg_indexZ seg_tokenrrr" _find_tokenss zTapasTokenizer._find_tokensccshtd|t|jD]H\}}|dkr,qt|D],\}}|||}|dk r4t|||dVq4qdS)z4Returns all occurrences of answer_text in the table.z answer text: rNr)rrTrLr8rVr1)rrrr3rr.rr4rrr")_find_answer_coordinates_from_answer_texts z8TapasTokenizer._find_answer_coordinates_from_answer_textc Csdgt|}|D]}|||D]}t|j|||j|jdd}|g} |r|j|d} | t|} |D]} | | krr| | krr| | qrt| t|kr"| D] } d|| <qqq"q|S)zDMaps question with answer texts to the first matching token indexes.rr)r.r/) rrWrr1r2r3sortr4ru) rr<r=r answer_textsrMrZ coordinatesZindexesZcoordinate_answer_ids begin_index end_indexrPrrr""_find_answer_ids_from_answer_textss6   z1TapasTokenizer._find_answer_ids_from_answer_textscCs"||||\}}|rtd|S)z7Maps answer coordinates of a question to token indexes.zCouldn't find all answers)rSry)rr<r=rrMrNrrr"_get_answer_idsszTapasTokenizer._get_answer_idscs4jr&j|||fdd|DdS|||S)Ncsg|]}|qSr)r)r atrrr"rg,sz1TapasTokenizer.get_answer_ids..)rY)rbr\r])rr<r=rZanswer_texts_questionrQrrr"r&szTapasTokenizer.get_answer_ids)rrpadding_strategyrrrcCs|dkrd|jk}|tjkr(t|d}|dk rT|dk rT||dkrT||d|}|tjkolt|d|k}|rd|krdgt|d|d<|r|t|d}|jdkr|r|ddg||d<d|kr|d|jgdg||d<d |kr|d dg||d <d |kr6|d td g||d <d |krV|d d g||d <d|krv|ddg||d<|d|jg||d<n|jdkr|rdg||d|d<d|kr|jgdg||d|d<d |krdg||d |d <d |kr&td g||d |d <d |krFd g||d |d <d|krfdg||d|d<|jg||d|d<nt dt |j|S)a? Pad encoded inputs (on left/right and up to predefined length or max length in the batch) Args: encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). max_length: maximum length of the returned list and optionally padding length (see below). Will truncate by taking into account the special tokens. padding_strategy: PaddingStrategy to use for padding. - PaddingStrategy.LONGEST Pad to the longest sequence in the batch - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) - PaddingStrategy.DO_NOT_PAD: Do not pad The tokenizer padding sides are defined in self.padding_side: - 'left': pads on the left of the sequences - 'right': pads on the right of the sequences pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability `>= 7.5` (Volta). return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics) NrrrrrightrrrrArrFrleftzInvalid padding strategy:) r rZLONGESTrrZ padding_sideZpad_token_type_idrCZ pad_token_idryr)rrrr_rrZneeds_to_be_padded differencerrr"_pad0s`                   zTapasTokenizer._padc csXt|D]J\}}||}||d}||d} |dkr| dkr|dkr||fVqdS)Nrr)rL) r probabilitiesr>r=r<ipZ segment_idcolrrrr"_get_cell_token_probss   z$TapasTokenizer._get_cell_token_probsc Cs`tt}|||||D]2\}}||d}||d} ||| f|qdd|DS)z?Computes average probability per cell, aggregating over tokens.rcSs i|]\}}|t|qSr)nparrayZmean)r rPZ cell_probsrrr"r#sz7TapasTokenizer._get_mean_cell_probs..)rHr6rrirurz) rrer>r=r<Zcoords_to_probsrfZprobrhrrrr"_get_mean_cell_probss    z#TapasTokenizer._get_mean_cell_probs?cCs|}|dk r|}dd|D}d||dk<ddt| |d}ddd d d d d g}|d}|ddddd|df}|ddddd|d f} |ddddd|df} |jd} g} t| D]} || }|| }| | }| | }|}|}|dkr0|dkr0q| ||||}g}t|D]H}t|D]8}| ||fd}|dk rd||krd| ||fqdqXt |}| |q| f}|dk r|j dd}| |f}|S)a Converts logits of [`TapasForQuestionAnswering`] to actual predicted answer coordinates and optional aggregation indices. The original implementation, on which this function is based, can be found [here](https://github.com/google-research/tapas/blob/4908213eb4df7aa988573350278b44c4dbe3f71b/tapas/experiments/prediction_utils.py#L288). Args: data (`dict`): Dictionary mapping features to actual values. Should be created using [`TapasTokenizer`]. logits (`torch.Tensor` or `tf.Tensor` of shape `(batch_size, sequence_length)`): Tensor containing the logits at the token level. logits_agg (`torch.Tensor` or `tf.Tensor` of shape `(batch_size, num_aggregation_labels)`, *optional*): Tensor containing the aggregation logits. cell_classification_threshold (`float`, *optional*, defaults to 0.5): Threshold to be used for cell selection. All table cells for which their probability is larger than this threshold will be selected. Returns: `tuple` comprising various elements depending on the inputs: - predicted_answer_coordinates (`List[List[[tuple]]` of length `batch_size`): Predicted answer coordinates as a list of lists of tuples. Each element in the list contains the predicted answer coordinates of a single example in the batch, as a list of tuples. Each tuple is a cell, i.e. (row index, column index). - predicted_aggregation_indices (`List[int]`of length `batch_size`, *optional*, returned when `logits_aggregation` is provided): Predicted aggregation operator indices of the aggregation head. NcSs"i|]\}}|dkr||qS)Ztraining)numpy)r rrrrr"r#sz@TapasTokenizer.convert_logits_to_predictions..g,Vrrr>r<r=rrrrrrrr])Zaxis)rnrzrjexprPr*r)tolistr#rlrrurZargmax)rdataZlogitsZ logits_aggZcell_classification_thresholdreZ token_typesrr>r=r<Z num_batchZpredicted_answer_coordinatesrfZprobabilities_exampleZsegment_ids_exampleZrow_ids_exampleZcolumn_ids_example max_width max_heightZcell_coords_to_probrrhrZ cell_proboutputZpredicted_aggregation_indicesrrr"convert_logits_to_predictionssb                z,TapasTokenizer.convert_logits_to_predictions)TTNrWrXrYrZr[r\TNr]NNFFNNrN)N)N)NF)NNNTFFNNNNNFFFFT)NNNTFFNNNNNFFFFT)NNTFFNNNTNFFFFT)NNNNTFFNNNTTFFFTF)NTFFNN)NNNTFFNNNNNFFFT)NNTFFNNNTTFFFT)NNNNTFFNNNTTFFFTF)N)N)Nrm)Zr'r(r)r*rZvocab_files_namesPRETRAINED_VOCAB_FILES_MAPZpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESZmax_model_input_sizesr5rrrrrpropertyr$rrrrrrr rr!rrrrrrr-TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRINGr rrrrr%rrrrrrrr7rrrrrrrrr rrrr(rrrr+r,r0r1r r8r r rrKrOrSrVrWr\r]rrrrrdrirlru __classcell__rrrr"rVsxO `            t   `   A   G   *     [     7     C  ; -   #) &  ^ rVc@sNeZdZdZdddZdddZdd Zdd d Zd d ZddZ ddZ dS)r|a Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.). Args: do_lower_case (`bool`, *optional*, defaults to `True`): Whether or not to lowercase the input when tokenizing. never_split (`Iterable`, *optional*): Collection of tokens which will never be split during tokenization. Only has an effect when `do_basic_tokenize=True` tokenize_chinese_chars (`bool`, *optional*, defaults to `True`): Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)). strip_accents (`bool`, *optional*): Whether or not to strip all accents. If this option is not specified, then it will be determined by the value for `lowercase` (as in the original BERT). do_split_on_punc (`bool`, *optional*, defaults to `True`): In some instances we want to skip the basic punctuation splitting so that later tokenization can capture the full context of the words, such as contractions. TNcCs2|dkr g}||_t||_||_||_||_dSr)r$r9rhrirjdo_split_on_punc)rr$rhrirjr{rrr"rs zBasicTokenizer.__init__cCs|r|jt|n|j}||}|jr4||}td|}t|}g}|D]R}||kr|j r| }|j dk r| |}n|j r| |}| |||qPtd|}|S)aj Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer. Args: never_split (`List[str]`, *optional*) Kept for backward compatibility purposes. Now implemented directly at the base class level (see [`PreTrainedTokenizer.tokenize`]) List of token not to split. NFCFr)rhunionr9 _clean_textri_tokenize_chinese_chars unicodedata normalizerUr$lowerrj_run_strip_accentsextend_run_split_on_puncr)rrTrhZunicode_normalized_textZ orig_tokensrr- output_tokensrrr"r,s$       zBasicTokenizer.tokenizecCsBtd|}g}|D]"}t|}|dkr,q||qd|S)z$Strips accents from a piece of text.ZNFDZMnr)rrcategoryrur)rrTrtcharcatrrr"rRs   z!BasicTokenizer._run_strip_accentscCs|jr|dk r||kr|gSt|}d}d}g}|t|kr||}t|r^||gd}n |rl|gd}|d||d7}q0dd|DS) z&Splits punctuation on a piece of text.NrTFr]rcSsg|]}d|qS)r)r)r xrrr"rgqsz5BasicTokenizer._run_split_on_punc..)r{rrrru)rrTrhcharsrfZstart_new_wordrtrrrr"r]s"    z!BasicTokenizer._run_split_on_punccCsTg}|D]@}t|}||r>|d|||dq||qd|S)z)Adds whitespace around any CJK character.rr)ord_is_chinese_charrurrrTrtrcprrr"rss     z&BasicTokenizer._tokenize_chinese_charscCs|dkr|dks|dkr |dks|dkr0|dks|dkr@|dks|d krP|d ks|d kr`|d ks|d krp|dks|dkr|dkrdSdS)z6Checks whether CP is the codepoint of a CJK character.iNii4iMiiߦii?i@ii iiiiiTFr)rrrrr"rsD  zBasicTokenizer._is_chinese_charcCsXg}|D]D}t|}|dks|dkst|r.qt|rB|dq||qd|S)zBPerforms invalid character removal and whitespace cleanup on text.rirr)rr rrurrrrr"r~s  zBasicTokenizer._clean_text)TNTNT)N)N) r'r(r)r*rrrrrrr~rrrr"r|s  &  r|c@s"eZdZdZdddZddZdS) r~zRuns WordPiece tokenization.dcCs||_||_||_dSr)rNrkmax_input_chars_per_word)rrNrkrrrr"rszWordpieceTokenizer.__init__c Csg}t|D]}t|}t||jkr4||jq d}d}g}|t|krt|}d} ||krd|||} |dkrd| } | |jkr| } q|d8}qX| dkrd}q|| |}q@|r||jq ||q |S)a Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform tokenization using the given vocabulary. For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`. Args: text: A single token or whitespace separated tokens. This should have already been passed through *BasicTokenizer*. Returns: A list of wordpiece tokens. FrNrr@rT) rUrrrrurkrrNr) rrTrr-rZis_badstartZ sub_tokensendZ cur_substrsubstrrrr"rs:       zWordpieceTokenizer.tokenizeN)r)r'r(r)r*rrrrrr"r~s r~c@s0eZdZdZdZdZdZdZdZdZ dZ d Z d S) r=rrr ra N) r'r(r)ZHEADER_TO_CELLZCELL_TO_HEADERZQUERY_TO_HEADERZ QUERY_TO_CELLZ ROW_TO_CELLZ CELL_TO_ROWr>LTGTrrrr"r=sr=c@s>eZdZUdZeeed<dZeeed<dZeeed<dS)DateNyearmonthday) r'r(r)rrr5r6rrrrrr"rs rc@s.eZdZUdZeeed<dZeeed<dS) NumericValueNrEdate) r'r(r)rErrCr6rrrrrr"rs rc@s6eZdZUdZeed<dZeed<dZee ed<dS)NumericValueSpanNrZr[r5) r'r(r)rZr5r6r[r5rrrrrr"r s   rc@s&eZdZUeed<dZeeed<dS)CellrTNr-)r'r(r)rr6r-rrrrrr"r s rc@s2eZdZUeed<eed<dZeeeed<dS)Question original_textrTNr:) r'r(r)rr6r:rrrrrrr"r s r _DateMaskrrrF%B%Yz%Ysz%b %Yz%B %Yz%B %dz%b %dz%d %bz%d %Bz %B %d, %Yz%d %B %Yz%m-%d-%Yz%Y-%m-%dz%Y-%mz%d %b %Yz %b %d, %Yz%d.%m.%Yz %A, %b %dz %A, %B %d))z%A\w+)rr)rz\d{4})z%bz\w{3})z%d\d{1,2})z%mrcCs~|\}}|}|dtd}|dtd}|dd}tD]\}}|||}q@d|ksft|||td|dfS)z.)r_DATE_PATTERNSrrrr"_process_date_patternsY srrzeroonetwothreefourfivesixseveneightninetenZelevenZtwelveZzerothfirstsecondthirdZfourthZfithZsixthZseventhZeighthZninthZtenthZeleventhZtwelfthstndrdthz.((^|\s)[+-])?((\.\d+)|(\d+(,\d\d\d)*(\.\d*)?))iiZINFcCs^|jtks|jtkr$td|jt}|jr8|j|_|jrF|j|_|jrT|j|_t|dS)zYConverts date (datetime Python object) to a NumericValue object with a Date object value.zInvalid year: )r)r _MIN_YEAR _MAX_YEARryrrrr)rrZnew_daterrr"_get_numeric_value_from_date srcCs|d|d|d fS)zHSorts span by decreasing length first and increasing first index second.rrr)spanrrr"_get_span_length_key srcCs t|dS)zDConverts float (Python) to a NumericValue object with a float value.)rE)rrrrr"_get_numeric_value_from_float src Cstdd|}tD]p\}}}||s(qztj||}Wntk rVYqYnXzt||WStk rYqYqXqdS)zAAttempts to format a text as a standard date string (yyyy-mm-dd).zSept\bSepN) rsub_PROCESSED_DATE_PATTERNSmatchdatetimestrptimerryr)rTZ in_patternrrrrrr" _parse_date s   rcCsxtD]$}||r|dt| }q*q|dd}z t|}Wntk rXYdSXt|rhdS|tkrtdS|S)z,Parses simple cardinal and ordinals numbers.N,r) _ORDINAL_SUFFIXESendswithrrrCrymathisnan_INF)rTsuffixrrrr" _parse_number s    rccsg}t|D]v\}}|sq |dks6||ds@|||dt|ks`||ds || dD]}||dfVqnq dS)z Split a text into all possible ngrams up to 'max_ngram_length'. Split points are white space and punctuation. Args: text: Text to split. max_ngram_length: maximal ngram length. Yields: Spans, tuples of begin-end index. rrN)rLisalnumrur)rTmax_ngram_lengthZ start_indexesrPr start_indexrrr" get_all_spans s   rcCsd|S)Nr)rrrSrTrrr"normalize_for_match srcCs`|}|dks$|dks$|dkr(t}tdd|dd}d|}|}|r\|StS)z"Lowercases and strips punctuation.zn/a?rAz[^\w\d]+rr)rrRrrrrrrSrrrr"r s rcCstt}t|D]>}|||}t|}|dk r|| t |qt |ddD]\}}||f|krvq`|||}t|}|dk r|||f t |t t D].\}}||kr|||f t t|qqt tD].\}}||kr|||f t t|q`qq`t |tdD]:\}}|||}t|}|dk r&|||f |q&t|dddd} g} | D]R\} } | D]4\} }| d| dkr| d| dkrqq| | | fq| jd dd g}| D]&\} }| t| d| d|d q|S) z Extracts longest number and date spans. Args: text: text to annotate Returns: List of longest numeric value spans. Nr)rcSs t|dSrH)rZ span_valuerrr"r rzparse_text..T)rreversercSs |ddSrHrrrrr"r$ rr)rZr[r5)rHr6r_NUMBER_PATTERNfinditerrrrrrurrrL _NUMBER_WORDSrC_ORDINAL_WORDS_MAX_DATE_NGRAM_SIZErrrzrXr)rTZ span_dictrZ span_textnumberrZr[wordrspansZselected_spansrrZ selected_spanrZnumeric_value_spansr5rrr" parse_text sL       $   rZEMPTYrrcCs.|jdk rtS|jdk rtStd|dS)NUnknown type: )rE NUMBER_TYPEr DATE_TYPEryr-rrr"_get_value_type= s   rcCs|jdk r|jS|jdk rz|j}dddg}|jdk rBt|j|d<|jdk rZt|j|d<|jdk rrt|j|d<t|Std|dS)z7Maps a NumericValue proto to a float or tuple of float.Nrrrr)rErrrCrrrry)r-rZ value_tuplerrr"_get_value_as_primitive_valueE s      rcCsdd|DS)NcSsh|] }t|qSr)r)r rrrr" X sz!_get_all_types..r)rrrr"_get_all_typesW srcst|}t|dkr"td|tt|}|tkr:tSttt |D]>}t|}t |t sdt t |D]\}}|dkrl|qlqJstd|fdd}|S)a$ Creates a function that can be used as a sort key or to compare the values. Maps to primitive types and finds the biggest common subset. Consider the values "05/05/2010" and "August 2007". With the corresponding primitive values (2010.,5.,5.) and (2007.,8., None). These values can be compared by year and date so we map to the sequence (2010., 5.), (2007., 8.). If we added a third value "2006" with primitive value (2006., None, None), we could only compare by the year so we would map to (2010.,), (2007.,) and (2006.,). Args: numeric_values: Values to compare Returns: A function that can be used as a sort key function (mapping numeric values to a comparable tuple) Raises: ValueError if values don't have a common type or are not comparable. rzNo common value type in NzNo common value in cst|tfddDS)Nc3s|]}|VqdSrr)r rPrrr"r& sz@get_numeric_sort_key_fn.._sort_key_fn..)rrrZ valid_indexesrr" _sort_key_fn sz-get_numeric_sort_key_fn.._sort_key_fn)rrrynextiterrrr9r)_DATE_TUPLE_SIZErrrrLdiscard)rZ value_types value_typer-rZ tuple_indexZ inner_valuerrrr"r4[ s"    r4cCst}|D]}|t|q|s,iSt|}|t||krLiSt}|D]\}}||krZ| |qZt|dkrt |kst t } n t t |} i} |D]*\} } | D]} t| | kr| | | <qqq| S)a( Finds the most common numeric values in a column and returns them Args: row_index_to_values: For each row index all the values in that cell. min_consolidation_fraction: Fraction of cells that need to have consolidated value. debug_info: Additional information only used for logging Returns: For each row index the first value that matches the most common value. Rows that don't have a matching value are dropped. Empty list if values can't be consolidated. r)rHCounterr5updaterr#rr9rzr<rrrrr)Zrow_index_to_valuesmin_consolidation_fraction debug_infoZ type_countsr max_countZ valid_typesrcountZmax_typeZnew_row_index_to_valuerPr5rrrr"_consolidate_numeric_values s.       r cCst|}tjdd|DS)z'Parses text and returns numeric values.css|] }|jVqdSr)r5)r rrrr"r& sz&_get_numeric_values..)r itertoolschain)rTr:rrr"r  sr cCs8i}|D]&\}}t||j}tt|||<q |S)a= Parses text in column and returns a dict mapping row_index to values. This is the _get_column_values function from number_annotation_utils.py of the original implementation Args: table: Pandas dataframe col_index: integer, indicating the index of the column to get the numeric values of )rrrTrr )rr.Zindex_to_valuesr3rrTrrr"r0 s r0cCs>||}||}||krtjS||kr,tjS||kr:tjSdS)z7Compares two values and returns their relation or None.N)r=r>rr)rZ other_valuer@rrr"r; sr;cCs"|}t|}t|}t|||dS)z'Adds numeric value spans to a question.)rrTr:)rrr)r?rr:rrr"r  sr cCst|trdS|dfS)z@Return an empty string and True if 'text' is in invalid unicode.)rTF)rbytesrrrr"filter_invalid_unicode sr c Cst|dsd|_|D]H\}}t|D]6\}}t|\}}|r(td|jd|d|q(qt|jD]0\}}t|\}}|rltd|jd|qldS)z Removes invalid unicode from table. Checks whether a table cell text contains an invalid unicode encoding. If yes, reset the table cell text to an empty str and log a warning for each invalid cell Args: table: table to clean. table_idrz(Scrub an invalid table body @ table_id: z , row_index: z , col_index: z*Scrub an invalid table header @ table_id: N)hasattrrrrLr rrr3)rr3rr.rZ is_invalidrrrr"!filter_invalid_unicode_from_table s   rffffff?c Cs|}t||D].\}}t|D]\}}t|d|j||f<q(qt|jD]B\}}tt|||||fd}| D]\}} | |j||f_ qzqR|S)a Parses text in table column-wise and adds the consolidated values. Consolidation refers to finding values with a common types (date or number) Args: table: Table to annotate. min_consolidation_fraction: Fraction of cells in a column that need to have consolidated value. debug_info: Additional information used for logging. r)rr) copyrrrLrrDr3r r0rzr-) rrrr3rr.rrZ column_valuesr-rrr"r s r)rN)yr*rHrenumr rrvrr dataclassesrtypingrrrrrrr r rnrjZtokenization_utilsr r rrZtokenization_utils_baserrrrrrutilsrrrrrrZpandasrZ get_loggerr'rrrvr7rwZPRETRAINED_INIT_CONFIGURATIONr% namedtupler!r1r7r:rBrQrUryrVobjectr|r~Enumr=rrrrrrZ_YEARZ _YEAR_MONTHZ_YEAR_MONTH_DAYZ_MONTHZ _MONTH_DAYrrrrrrrrrrrrrrCrrrrrrrrrrZ_PrimitiveNumericValueEllipsisZ _SortKeyFnrrrrrrrr4r r r0r;r r rrrrrr"sj (   S   *&#B         ?"//