U 9%eM@sddlZddlZddlZddlmZmZmZmZm Z ddl m Z m Z m Z mZmZernddlZddlmZmZererddlZddlZddlmZmZGdd d e Zee Gd d d eZdS) N)add_end_docstrings#is_tensorflow_probability_availableis_tf_availableis_torch_availablerequires_backends)PIPELINE_INIT_ARGSArgumentHandlerDatasetPipelinePipelineException),MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES0MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES)/TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES3TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMESc@seZdZdZdddZdS)%TableQuestionAnsweringArgumentHandlerzB Handles arguments for the TableQuestionAnsweringPipeline NcKstt|dddl}|dkr&tdn|dkrt|tr^|ddk r^|ddk r^|g}nt|trt|dkrtdd|Dstdd d|D|dddk r|dddk r|}ntd |d d n:t dk rt|t st|t j r|Std t |d n ||dg}|D]@}t|d|js.|ddkrZtd||d|d<q.|S)Npandasrz(Keyword argument `table` cannot be None.querytablecss|]}t|tVqdSN) isinstancedict.0drn/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/pipelines/table_question_answering.py 7szATableQuestionAnsweringArgumentHandler.__call__..z:Keyword argument `table` should be a list of dict, but is css|]}t|VqdSr)typerrrrr9szIf keyword argument `table` is a list of dictionaries, each dictionary should have a `table` and `query` key, but only dictionary has keys z `table` and `query` keys.zZInvalid input. Keyword argument `table` should be either of type `dict` or `list`, but is ))rrzTable cannot be None.)rr ValueErrorrrgetlistlenallkeysr types GeneratorTyperZ DataFrame)selfrrkwargspdZtqa_pipeline_inputsZtqa_pipeline_inputrrr__call__'s:   &$" z.TableQuestionAnsweringArgumentHandler.__call__)NN)__name__ __module__ __qualname____doc__r,rrrrr"srcsleZdZdZdZeffdd ZddZddZfd d Z dd d Z dddZ dddZ ddZ ZS)TableQuestionAnsweringPipelinea Table Question Answering pipeline using a `ModelForTableQuestionAnswering`. This pipeline is only available in PyTorch. Example: ```python >>> from transformers import pipeline >>> oracle = pipeline(model="google/tapas-base-finetuned-wtq") >>> table = { ... "Repository": ["Transformers", "Datasets", "Tokenizers"], ... "Stars": ["36542", "4512", "3934"], ... "Contributors": ["651", "77", "34"], ... "Programming language": ["Python", "Python", "Rust, Python and NodeJS"], ... } >>> oracle(query="How many stars does the transformers repository have?", table=table) {'answer': 'AVERAGE > 36542', 'coordinates': [(0, 1)], 'cells': ['36542'], 'aggregator': 'AVERAGE'} ``` Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) This tabular question answering pipeline can currently be loaded from [`pipeline`] using the following task identifier: `"table-question-answering"`. The models that this pipeline can use are models that have been fine-tuned on a tabular question answering task. See the up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=table-question-answering). z table,querycstj||||_|jdkr2t}|tnt}|t | |t t |j jddott t |j jdd|_t|j jdrdnd|_dS)Ntfaggregation_labelsZnum_aggregation_labelstapas)super__init__ _args_parser frameworkrcopyupdaterrrZcheck_model_typeboolgetattrmodelconfig aggregatehasattrr)r)Z args_parserargsr*mapping __class__rrr6ys    z'TableQuestionAnsweringPipeline.__init__cKs |jf|Sr)r=)r)inputsrrrbatch_inferencesz.TableQuestionAnsweringPipeline.batch_inferencec sr|jdkrg}g}d}|djd}|d|j}|d|j}|d|j}d} t|D].} |dk rR| dddf} t| } || } t| jdD]} | dddf | }| dddf | d}| ddd f | d}|dkr|dkr|dkrt |||f| | <qt |  t j|j| dddf<|| }|| }|| } |j|d|d| dd }|j}|jr||j||t jj|d }|j| t j|jj}ttt| D]\} }| dddf | }| dddf | d}| ddd f | d}|dkr|dkr|dkr||f|qfd d D}qbt t|d}|js|fS|t t|dfSg}g}d}|djd}|d}|d}|d}d} t|D]0} |dk r| dddf} tj| tj d} || } t| jdD]} | dddf | }| dddf | d}| ddd f | d}|dkrN|dkrN|dkrNt |||f| | <qN| | dddf<|| }|| }|| } |jtj!|ddtj!|ddtj!| ddd }|j}|jrH||j||t"jj|d }|#t$%|t$j}tt| } tt$| D]\} }| dddf | }| dddf | d}| ddd f | d}|dkr|dkr|dkr||f|qfdd D}qt$&t|d}|jsZ|fS|t$&t|dfSdS)z Inference used for models that need to process sequences in a sequential fashion, like the SQA models which handle conversational query related to a table. ptN input_idsrattention_masktoken_type_idsrr)rHrIrJ)logitscs$i|]}|t|dkqSg?nparrayZmeanrkeyZcoords_to_probsrr szGTableQuestionAnsweringPipeline.sequential_inference..)Zdtype)Zaxiscs$i|]}|t|dkqSrMrNrQrSrrrTs)'r8shapetoZdevicerangerOZ zeros_likecpunumpytolistinttorchZ from_numpyrlongr=Z unsqueezerLr?appendZlogits_aggregation distributionsZ BernoulliZprobsZfloat32 collections defaultdictr# enumerateZsqueezecattupleZint32Z expand_dimstfpZprobs_parameterr2castconcat)r)rEZ all_logitsZall_aggregationsZ prev_answersZ batch_sizerHrIrJZtoken_type_ids_exampleindexZprev_labels_exampleZ model_labelsiZ segment_idZcol_idZrow_idZinput_ids_exampleZattention_mask_exampleoutputsrLZdist_per_tokenZ probabilitiespcolrowZ logits_batchrrSrsequential_inferences  &   "        z3TableQuestionAnsweringPipeline.sequential_inferencecs4|j||}tj|f|}t|dkr0|dS|S)a Answers queries according to a table. The pipeline accepts several types of inputs which are detailed below: - `pipeline(table, query)` - `pipeline(table, [query])` - `pipeline(table=table, query=query)` - `pipeline(table=table, query=[query])` - `pipeline({"table": table, "query": query})` - `pipeline({"table": table, "query": [query]})` - `pipeline([{"table": table, "query": query}, {"table": table, "query": query}])` The `table` argument should be a dict or a DataFrame built from that dict, containing the whole table: Example: ```python data = { "actors": ["brad pitt", "leonardo di caprio", "george clooney"], "age": ["56", "45", "59"], "number of movies": ["87", "53", "69"], "date of birth": ["7 february 1967", "10 june 1996", "28 november 1967"], } ``` This dictionary can be passed in as such, or can be converted to a pandas DataFrame: Example: ```python import pandas as pd table = pd.DataFrame.from_dict(data) ``` Args: table (`pd.DataFrame` or `Dict`): Pandas DataFrame or dictionary that will be converted to a DataFrame containing all the table values. See above for an example of dictionary. query (`str` or `List[str]`): Query or list of queries that will be sent to the model alongside the table. sequential (`bool`, *optional*, defaults to `False`): Whether to do inference sequentially or as a batch. Batching is faster, but models like SQA require the inference to be done sequentially to extract relations within sequences, given their conversational nature. padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`): Activates and controls padding. Accepts the following values: - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence if provided). - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum acceptable input length for the model if that argument is not provided. - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different lengths). truncation (`bool`, `str` or [`TapasTruncationStrategy`], *optional*, defaults to `False`): Activates and controls truncation. Accepts the following values: - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum acceptable input length for the model if that argument is not provided. This will truncate row by row, removing rows from the table. - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths greater than the model maximum admissible input size). Return: A dictionary or a list of dictionaries containing results: Each result is a dictionary with the following keys: - **answer** (`str`) -- The answer of the query given the table. If there is an aggregator, the answer will be preceded by `AGGREGATOR >`. - **coordinates** (`List[Tuple[int, int]]`) -- Coordinates of the cells of the answers. - **cells** (`List[str]`) -- List of strings made up of the answer cell values. - **aggregator** (`str`) -- If the model has an aggregator, this returns the aggregator. rr)r7r5r,r$)r)rAr*Zpipeline_inputsresultsrCrrr,s K  z'TableQuestionAnsweringPipeline.__call__NcKsBi}|dk r||d<|dk r$||d<i}|dk r8||d<||ifS)Npadding truncation sequentialr)r)rrrprqr*Zpreprocess_paramsZforward_paramsrrr_sanitize_parameters`sz3TableQuestionAnsweringPipeline._sanitize_parametersTcCsv|dkr|jdkrd}nd}|d|d}}|jrr3rripred)r)rrrTsz>TableQuestionAnsweringPipeline.postprocess..cs&i|]\}}|kr||dqS)z > rr}) aggregatorsno_agg_label_indexrrrTsrcsg|]}j|qSrZiatrZ coordinaterrr sz>TableQuestionAnsweringPipeline.postprocess..rtz, csg|]}j|qSrrrrrrrs)answer coordinatescells aggregatorz Empty answercSsg|] }d|iqS)rr)rrrrrrsT)Zskip_special_tokensr)rr?rvZconvert_logits_to_predictionsrbr=r>Zno_aggregation_label_indexr"joinr^r$r Z batch_decode)r)r{rErjrLZ logits_aggZ predictionsZanswer_coordinates_batchZagg_predictionsZaggregators_prefixZanswersrhrrrZaggregator_prefixrr)rrr)rr postprocesssD        z*TableQuestionAnsweringPipeline.postprocess)NNN)NTN)F)r-r.r/r0Zdefault_input_namesrr6rFrnr,rsrwr|r __classcell__rrrCrr1Ws R  r1)r`r'rYrOutilsrrrrrbaser r r r r r\Zmodels.auto.modeling_autorrZ tensorflowr2Ztensorflow_probabilityreZmodels.auto.modeling_tf_autorrrr1rrrrs 5