U
    -ed                     @   s  d dl mZmZ d dlmZ d dlmZmZmZm	Z	m
Z
mZmZmZmZ d dlZd dlZd dlZd dlmZ ddlmZ ddlmZ dd	lmZmZmZmZ dd
l m!Z! ddl"m#Z# edZ$edZ%edZ&edZ'e(e)dddZ*edddZ+e!ee,e-e(e.e
f e!ej!dddZ/e!ee,e-e(e.e
f ej!dddZ0ej1e)dddZ2G dd  d e	e%e&e'f Z3ee.ee$ f ee.e$f d!d"d#Z4G d$d% d%e3ej!ej1ej!f Z5G d&d' d'e3e6e7e6f Z8G d(d) d)e3e6ej9e6f Z:G d*d+ d+e3ej;ej<ej;f Z=G d,d- d-Z>G d.d/ d/Z?G d0d1 d1eZ@G d2d3 d3e@ZAG d4d5 d5e@ZBG d6d7 d7e	e%e&e'f ZCG d8d9 d9eCe%e&e'f ZDG d:d; d;eCej!ej1ej!f ZEG d<d= d=eCee7ef ZFG d>d? d?eCej;ej<ej;f ZGG d@dA dAeCe6e&e6f ZHe.ee. ddBdCdDZIee,e-e(e
f e,ddEdFdGZJee,e-e(e.e
f e.ddHdIZKdPe!ee,e-e(e.e
f ee! ej!ddJdKZLdQe!ee,e-e(e.e
f eCee7 dMdNdOZMdS )R    )MappingMutableMapping)partial)	AnyCallableDictGenericIterableListOptionalTypeVarUnionN)version   )config)Features)_ArrayXDExtensionType_is_zero_copy_onlydecode_nested_examplepandas_types_mapper)Table)no_op_if_value_is_nullT	RowFormatColumnFormatBatchFormat)keyreturnc                 C   s   | j dko| j| jkS )N   )stepstopstartr    r#   _/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/formatting/formatting.py_is_range_contiguous(   s    r%   r"   c                 C   s   t d|  dt|  dd S )NzWrong key type: 'z' of type 'z6'. Expected one of int, slice, range, str or Iterable.)	TypeErrortyper"   r#   r#   r$   _raise_bad_key_type,   s    r(   )tabler   indicesr   c                    s   t |tr4 | j ddd  }t| |S t |trNt|	 j }t |trt
|r|jdkrt| dd  |j|j|j dD S t |tr| |g} t|  d S t |trt|  fdd|D S t| dS )aE  
    Query a pyarrow Table to extract the subtable that correspond to the given key.
    The :obj:`indices` parameter corresponds to the indices mapping in case we cant to take into
    account a shuffling or an indices selection for example.
    The indices table must contain one column named "indices" of type uint64.
    r   r   c                 S   s   g | ]}|  qS r#   )as_py.0ir#   r#   r$   
<listcomp>C   s     z5_query_table_with_indices_mapping.<locals>.<listcomp>c                    s&   g | ]}  |d dd  qS )r   r   )
fast_slicecolumnr+   r,   r*   r#   r$   r/   K   s     N)
isinstanceintr0   num_rowsr1   r+   _query_tablesliceranger*   r%   r!   r    strselect	to_pylistr	   r(   )r)   r   r*   r#   r2   r$   !_query_table_with_indices_mapping2   s"    	
 


 $

r<   )r)   r   r   c                    s   t  tr|  | j dS t  tr6t | j  t  trht rh jdkrh|  j j	 j S t  t
r| j fdd| jD S t  trt tj t dkr| jddS |  | j S t  dS )zY
    Query a pyarrow Table to extract the subtable that correspond to the given key.
    r   r   c                    s   g | ]}| kr|qS r#   r#   )r-   r1   r"   r#   r$   r/   ^   s      z _query_table.<locals>.<listcomp>N)r3   r4   r0   r5   r7   r8   r*   r%   r!   r    r9   r)   dropcolumn_namesr	   npZfromiterZint64lenZfast_gatherr(   )r)   r   r#   r"   r$   r6   P   s    




r6   pa_arrayr   c                 C   s
   | j dkS Nr   )Z
null_count)rB   r#   r#   r$   _is_array_with_nullsi   s    rD   c                   @   sF   e Zd ZdZejedddZejedddZ	eje
dddZd	S )
BaseArrowExtractorz
    Arrow extractor are used to extract data from pyarrow tables.
    It makes it possible to extract rows, columns and batches.
    These three extractions types have to be implemented.
    pa_tabler   c                 C   s   t d S NNotImplementedErrorselfrG   r#   r#   r$   extract_rowt   s    zBaseArrowExtractor.extract_rowc                 C   s   t d S rH   rI   rK   r#   r#   r$   extract_columnw   s    z!BaseArrowExtractor.extract_columnc                 C   s   t d S rH   rI   rK   r#   r#   r$   extract_batchz   s    z BaseArrowExtractor.extract_batchN)__name__
__module____qualname____doc__par   r   rM   r   rN   r   rO   r#   r#   r#   r$   rE   m   s   rE   )py_dictr   c                 C   s   dd |   D S )z:Return the first element of a batch (dict) as a row (dict)c                 S   s   i | ]\}}||d  qS )r   r#   )r-   r   arrayr#   r#   r$   
<dictcomp>   s      z_unnest.<locals>.<dictcomp>)items)rU   r#   r#   r$   _unnest~   s    rY   c                   @   sH   e Zd ZejejdddZejejdddZejejdddZdS )	SimpleArrowExtractorrF   c                 C   s   |S rH   r#   rK   r#   r#   r$   rM      s    z SimpleArrowExtractor.extract_rowc                 C   s
   | dS rC   )r1   rK   r#   r#   r$   rN      s    z#SimpleArrowExtractor.extract_columnc                 C   s   |S rH   r#   rK   r#   r#   r$   rO      s    z"SimpleArrowExtractor.extract_batchN)	rP   rQ   rR   rT   r   rM   ArrayrN   rO   r#   r#   r#   r$   rZ      s   rZ   c                   @   sB   e Zd ZejedddZejedddZejedddZ	dS )	PythonArrowExtractorrF   c                 C   s   t | S rH   )rY   	to_pydictrK   r#   r#   r$   rM      s    z PythonArrowExtractor.extract_rowc                 C   s   | d S rC   )r1   r;   rK   r#   r#   r$   rN      s    z#PythonArrowExtractor.extract_columnc                 C   s   |  S rH   )r]   rK   r#   r#   r$   rO      s    z"PythonArrowExtractor.extract_batchN)
rP   rQ   rR   rT   r   dictrM   listrN   rO   r#   r#   r#   r$   r\      s   r\   c                   @   s`   e Zd Zdd ZejedddZejej	dddZ
ejeddd	Zejej	d
ddZdS )NumpyArrowExtractorc                 K   s
   || _ d S rH   )np_array_kwargs)rL   ra   r#   r#   r$   __init__   s    zNumpyArrowExtractor.__init__rF   c                 C   s   t | |S rH   )rY   rO   rK   r#   r#   r$   rM      s    zNumpyArrowExtractor.extract_rowc                 C   s   |  ||jd  S rC   )_arrow_array_to_numpyr>   rK   r#   r#   r$   rN      s    z"NumpyArrowExtractor.extract_columnc                    s    fdd j D S )Nc                    s   i | ]}|  | qS r#   )rc   r-   colrG   rL   r#   r$   rW      s      z5NumpyArrowExtractor.extract_batch.<locals>.<dictcomp>)r>   rK   r#   rf   r$   rO      s    z!NumpyArrowExtractor.extract_batchrA   c                    s   t |tjrrt |jtr>t|jjddfdd|jD  qt|joZtdd |jD fdd|jD  nNt |jtrt|jjdd|j	d n$t|jot
| |j	d  t d	krt fd
d D rtj dtdS tj ddS )NT)Zunnestc                    s"   g | ]}|j  d D ]}|qqS zero_copy_onlyto_numpyr-   chunkrowrh   r#   r$   r/      s      z=NumpyArrowExtractor._arrow_array_to_numpy.<locals>.<listcomp>c                 s   s   | ]}t | V  qd S rH   )rD   )r-   rm   r#   r#   r$   	<genexpr>   s    z<NumpyArrowExtractor._arrow_array_to_numpy.<locals>.<genexpr>c                    s"   g | ]}|j  d D ]}|qqS rg   rj   rl   rh   r#   r$   r/      s      rh   r   c                 3   sH   | ]@}t |tjr,|jtkp>|j d  jkp>t |to>t|V  qdS )r   N)r3   r?   ndarraydtypeobjectshapefloatisnan)r-   x)rV   r#   r$   ro      s   &F)copyrq   rw   )r3   rT   ZChunkedArrayr'   r   r   Zstorage_dtypechunksallrk   rD   tolistr@   anyr?   rV   rr   )rL   rB   r#   )rV   ri   r$   rc      s.    

z)NumpyArrowExtractor._arrow_array_to_numpyN)rP   rQ   rR   rb   rT   r   r^   rM   r?   rp   rN   rO   r[   rc   r#   r#   r#   r$   r`      s
   r`   c                   @   sH   e Zd ZejejdddZejejdddZ	ejejdddZ
dS )	PandasArrowExtractorrF   c                 C   s   |j ddjtdS )Nr   )lengthZtypes_mapper)r7   	to_pandasr   rK   r#   r#   r$   rM      s    z PandasArrowExtractor.extract_rowc                 C   s   | dgjtd|jd  S )Nr   r   )r:   r   r   r>   rK   r#   r#   r$   rN      s    z#PandasArrowExtractor.extract_columnc                 C   s   |j tdS )Nr   )r   r   rK   r#   r#   r$   rO      s    z"PandasArrowExtractor.extract_batchN)rP   rQ   rR   rT   r   pd	DataFramerM   SeriesrN   rO   r#   r#   r#   r$   r}      s   r}   c                   @   sP   e Zd Zee dddZeedddZee	eddd	Z
eed
ddZdS )PythonFeaturesDecoderfeaturesc                 C   s
   || _ d S rH   r   rL   r   r#   r#   r$   rb      s    zPythonFeaturesDecoder.__init__rn   r   c                 C   s   | j r| j |S |S rH   )r   Zdecode_example)rL   rn   r#   r#   r$   
decode_row   s    z PythonFeaturesDecoder.decode_rowr1   column_namer   c                 C   s   | j r| j ||S |S rH   )r   decode_column)rL   r1   r   r#   r#   r$   r      s    z#PythonFeaturesDecoder.decode_columnbatchr   c                 C   s   | j r| j |S |S rH   )r   decode_batchrL   r   r#   r#   r$   r      s    z"PythonFeaturesDecoder.decode_batchN)rP   rQ   rR   r   r   rb   r^   r   r_   r9   r   r   r#   r#   r#   r$   r      s   r   c                   @   s\   e Zd Zee dddZejejdddZej	e
ej	ddd	Zejejd
ddZdS )PandasFeaturesDecoderr   c                 C   s
   || _ d S rH   r   r   r#   r#   r$   rb      s    zPandasFeaturesDecoder.__init__r   c                    s@    j r fdd j  D ni }|r<|||t| < |S )Nc                    s,   i | ]$\}} j j| r|ttt|qS r#   )r   _column_requires_decodingr   r   r   )r-   r   featurerL   r#   r$   rW      s    z4PandasFeaturesDecoder.decode_row.<locals>.<dictcomp>)r   rX   	transformr_   keys)rL   rn   decoder#   r   r$   r      s    	z PandasFeaturesDecoder.decode_rowr   c                 C   sF   | j r0|| j kr0| j j| r0ttt| j | nd }|rB||}|S rH   )r   r   r   r   r   r   )rL   r1   r   r   r#   r#   r$   r      s    

z#PandasFeaturesDecoder.decode_columnr   c                 C   s
   |  |S rH   )r   r   r#   r#   r$   r      s    z"PandasFeaturesDecoder.decode_batchN)rP   rQ   rR   r   r   rb   r   r   r   r   r9   r   r   r#   r#   r#   r$   r      s   
r   c                   @   s   e Zd ZdZejddddZdd Zdd	 Zd
d Z	ddddZ
dd Zdd Zdd Zejedkrdd Zdd Zdd Zdd Zdd  Zed'd!d"Zd#d$ Zd%d& ZdS )(LazyDictzeA dictionary backed by Arrow data. The values are formatted on-the-fly when accessing the dictionary.	Formatter)rG   	formatterc                 C   s2   || _ || _dd |jD | _t| j | _d S )Nc                 S   s   i | ]
}|d qS rH   r#   )r-   r   r#   r#   r$   rW     s      z%LazyDict.__init__.<locals>.<dictcomp>)rG   r   r>   datasetr   keys_to_format)rL   rG   r   r#   r#   r$   rb     s    zLazyDict.__init__c                 C   s
   t | jS rH   )r@   r   r   r#   r#   r$   __len__
  s    zLazyDict.__len__c                 C   s8   | j | }|| jkr4| |}|| j |< | j| |S rH   )r   r   formatremoverL   r   valuer#   r#   r$   __getitem__  s    



zLazyDict.__getitem__c                 C   s$   || j kr| j | || j|< d S rH   r   r   r   r   r#   r#   r$   __setitem__  s    
zLazyDict.__setitem__N)r   c                 C   s"   || j kr| j | | j|= d S rH   r   rL   r   r#   r#   r$   __delitem__  s    
zLazyDict.__delitem__c                 C   s
   t | jS rH   )iterr   r   r#   r#   r$   __iter__  s    zLazyDict.__iter__c                 C   s
   || j kS rH   )r   r   r#   r#   r$   __contains__"  s    zLazyDict.__contains__c                 C   s   |    t| jS rH   )_format_allreprr   r   r#   r#   r$   __repr__%  s    zLazyDict.__repr__z3.9c                 C   s   t |trH|  }| }|  | j|j 8  _|j|jB |_|S t |tr||  }| j| 8  _|j|B |_|S tS rH   	r3   r   rw   r   r   r   r   r^   NotImplementedrL   otherinstr#   r#   r$   __or__,  s    

zLazyDict.__or__c                 C   s   t |trH|  }| }|  | j|j 8  _|j|jB |_|S t |tr||  }| j| 8  _||jB |_|S tS rH   r   r   r#   r#   r$   __ror__;  s    

zLazyDict.__ror__c                 C   sd   t |tr@| }|  |  j|j 8  _|  j|jO  _n |  j| 8  _|  j|O  _| S rH   )r3   r   rw   r   r   r   r   )rL   r   r#   r#   r$   __ior__J  s    
zLazyDict.__ior__c                 C   sH   | j | j }|j| j | jd  |jd< | jd  |jd< |S )Nr   r   )	__class____new____dict__updaterw   )rL   r   r#   r#   r$   __copy__U  s
    zLazyDict.__copy__c                 C   s   dd l }| | S rC   rx   )rL   rw   r#   r#   r$   rw   ^  s    zLazyDict.copyc                 C   s   t d S rH   rI   )clsiterabler   r#   r#   r$   fromkeysc  s    zLazyDict.fromkeysc                 C   s   t d S rH   rI   r   r#   r#   r$   r   g  s    zLazyDict.formatc                 C   s*   | j D ]}| || j|< q| j   d S rH   )r   r   r   clearr   r#   r#   r$   r   j  s    
zLazyDict._format_all)N)rP   rQ   rR   rS   rT   r   rb   r   r   r   r   r   r   r   r   Z
PY_VERSIONr   parser   r   r   r   rw   classmethodr   r   r   r#   r#   r#   r$   r      s&   	r   c                   @   s   e Zd Zdd ZdS )LazyRowc                 C   s   | j | j|gd S rC   r   format_columnrG   r:   r   r#   r#   r$   r   q  s    zLazyRow.formatNrP   rQ   rR   r   r#   r#   r#   r$   r   p  s   r   c                   @   s   e Zd Zdd ZdS )	LazyBatchc                 C   s   | j | j|gS rH   r   r   r#   r#   r$   r   v  s    zLazyBatch.formatNr   r#   r#   r#   r$   r   u  s   r   c                   @   s   e Zd ZdZeZeZeZ	e
Zdee dddZejeeeeef dddZejed	d
dZejed	ddZejed	ddZdS )r   z
    A formatter is an object that extracts and formats data from pyarrow tables.
    It defines the formatting for rows, columns and batches.
    Nr   c                 C   s"   || _ t| j | _t| j | _d S rH   )r   r   python_features_decoderr   pandas_features_decoderr   r#   r#   r$   rb     s    zFormatter.__init__)rG   
query_typer   c                 C   s:   |dkr|  |S |dkr$| |S |dkr6| |S d S Nrn   r1   r   )
format_rowr   format_batch)rL   rG   r   r#   r#   r$   __call__  s    

zFormatter.__call__rF   c                 C   s   t d S rH   rI   rK   r#   r#   r$   r     s    zFormatter.format_rowc                 C   s   t d S rH   rI   rK   r#   r#   r$   r     s    zFormatter.format_columnc                 C   s   t d S rH   rI   rK   r#   r#   r$   r     s    zFormatter.format_batch)N)rP   rQ   rR   rS   rZ   simple_arrow_extractorr\   python_arrow_extractorr`   Znumpy_arrow_extractorr}   pandas_arrow_extractorr   r   rb   rT   r   r9   r   r   r   r   r   r   r   r   r#   r#   r#   r$   r   z  s   r   c                   @   s   e Zd ZedddZdS )TensorFormatter)data_structc                 C   s   t d S rH   rI   )rL   r   r#   r#   r$   recursive_tensorize  s    z#TensorFormatter.recursive_tensorizeN)rP   rQ   rR   r^   r   r#   r#   r#   r$   r     s   r   c                   @   sH   e Zd ZejejdddZejejdddZejejdddZdS )	ArrowFormatterrF   c                 C   s   |   |S rH   )r   rM   rK   r#   r#   r$   r     s    zArrowFormatter.format_rowc                 C   s   |   |S rH   )r   rN   rK   r#   r#   r$   r     s    zArrowFormatter.format_columnc                 C   s   |   |S rH   )r   rO   rK   r#   r#   r$   r     s    zArrowFormatter.format_batchN)	rP   rQ   rR   rT   r   r   r[   r   r   r#   r#   r#   r$   r     s   r   c                       sT   e Zd Zd fdd	ZejedddZejeddd	Z	ejedd
dZ
  ZS )PythonFormatterNFc                    s   t  | || _d S rH   )superrb   lazy)rL   r   r   r   r#   r$   rb     s    zPythonFormatter.__init__rF   c                 C   s.   | j rt|| S |  |}| j|}|S rH   )r   r   r   rM   r   r   rL   rG   rn   r#   r#   r$   r     s
    
zPythonFormatter.format_rowc                 C   s&   |   |}| j||jd }|S rC   )r   rN   r   r   r>   rL   rG   r1   r#   r#   r$   r     s    zPythonFormatter.format_columnc                 C   s.   | j rt|| S |  |}| j|}|S rH   )r   r   r   rO   r   r   rL   rG   r   r#   r#   r$   r     s
    
zPythonFormatter.format_batch)NF)rP   rQ   rR   rb   rT   r   r   r   r_   r   r   __classcell__r#   r#   r   r$   r     s   r   c                   @   sH   e Zd ZejejdddZejejdddZ	ejejdddZ
dS )	PandasFormatterrF   c                 C   s   |   |}| j|}|S rH   )r   rM   r   r   r   r#   r#   r$   r     s    zPandasFormatter.format_rowc                 C   s&   |   |}| j||jd }|S rC   )r   rN   r   r   r>   r   r#   r#   r$   r     s    zPandasFormatter.format_columnc                 C   s   |   |}| j|}|S rH   )r   rO   r   r   r   r#   r#   r$   r     s    zPandasFormatter.format_batchN)rP   rQ   rR   rT   r   r   r   r   r   r   r   r#   r#   r#   r$   r     s   r   c                       sh   e Zd ZdZdeegef d fddZejedddZ	eje
dd	d
ZejedddZ  ZS )CustomFormattera  
    A user-defined custom formatter function defined by a ``transform``.
    The transform must take as input a batch of data extracted for an arrow table using the python extractor,
    and return a batch.
    If the output batch is not a dict, then output_all_columns won't work.
    If the ouput batch has several fields, then querying a single column won't work since we don't know which field
    to return.
    N)r   c                    s   t  j|d || _d S )Nr   )r   rb   r   )rL   r   r   kwargsr   r#   r$   rb     s    zCustomFormatter.__init__rF   c              
   C   sL   |  |}z
t|W S  tk
rF } ztd| |W 5 d }~X Y nX d S )Nz]Custom formatting function must return a dict of sequences to be able to pick a row, but got )r   rY   	Exceptionr&   rL   rG   Zformatted_batchexcr#   r#   r$   r     s    

zCustomFormatter.format_rowc              
   C   s   |  |}t|dr>t| dkrLtdt|  dntd| z||jd  W S  tk
r } ztd| |W 5 d }~X Y nX d S )Nr   r   zTried to query a column but the custom formatting function returns too many columns. Only one column was expected but got columns .zPCustom formatting function must return a dict to be able to pick a row, but got r   )r   hasattrr@   r   r&   r_   r>   r   r   r#   r#   r$   r     s"    

zCustomFormatter.format_columnc                 C   s$   |   |}| j|}| |S rH   )r   rO   r   r   r   r   r#   r#   r$   r      s    zCustomFormatter.format_batch)N)rP   rQ   rR   rS   r   r^   rb   rT   r   r   r   r   r   r   r#   r#   r   r$   r     s
   		r   )r   columnsr   c                 C   s    | |krt d|  d| d S )NzColumn z5 not in the dataset. Current columns in the dataset: )KeyError)r   r   r#   r#   r$   _check_valid_column_key  s    r   )r   sizer   c                 C   s   t | tr>| dk r| | dk s&| |kr:td|  d| d S t | trJnt | trt| dkrtt| |d tt| |d nHt | t	rt| dkrttt| |d ttt| |d nt
|  d S )Nr   zInvalid key: z is out of bounds for size )r   )r3   r4   
IndexErrorr7   r8   r@   _check_valid_index_keymaxminr	   r(   )r   r   r#   r#   r$   r     s    



r   c                 C   s<   t | trdS t | trdS t | tttfr0dS t|  d S r   )r3   r4   r9   r7   r8   r	   r(   r"   r#   r#   r$   key_to_query_type  s    

r   c                 C   sx   t |tttttfst| t |tr4t|| j n|dk	rB|j	n| j	}t
|| |dkrft| |}nt| ||d}|S )a1  
    Query a Table to extract the subtable that correspond to the given key.

    Args:
        table (``datasets.table.Table``): The input Table to query from
        key (``Union[int, slice, range, str, Iterable]``): The key can be of different types:
            - an integer i: the subtable containing only the i-th row
            - a slice [i:j:k]: the subtable containing the rows that correspond to this slice
            - a range(i, j, k): the subtable containing the rows that correspond to this range
            - a string c: the subtable containing all the rows but only the column c
            - an iterable l: the subtable that is the concatenation of all the i-th rows for all i in the iterable
        indices (Optional ``datasets.table.Table``): If not None, it is used to re-map the given key to the table rows.
            The indices table must contain one column named "indices" of type uint64.
            This is used in case of shuffling or rows selection.


    Returns:
        ``pyarrow.Table``: the result of the query on the input table
    Nr2   )r3   r4   r7   r8   r9   r	   r(   r   r>   r5   r   r6   r<   )r)   r   r*   r   Zpa_subtabler#   r#   r$   query_table(  s    

r   F)r)   r   r   format_columnsc                    s   t | tr| j}n| }t|}tdd} dkr<|||dS |dkrd| krV|||S |||dS nx| fdd|jD }|||d}	|rt |	tr| fdd|jD }
||
|d}|	| nt	d|	 |	S dS )	a  
    Format a Table depending on the key that was used and a Formatter object.

    Args:
        table (``datasets.table.Table``): The input Table to format
        key (``Union[int, slice, range, str, Iterable]``): Depending on the key that was used, the formatter formats
            the table as either a row, a column or a batch.
        formatter (``datasets.formatting.formatting.Formatter``): Any subclass of a Formatter such as
            PythonFormatter, NumpyFormatter, etc.
        format_columns (:obj:`List[str]`, optional): if not None, it defines the columns that will be formatted using the
            given formatter. Other columns are discarded (unless ``output_all_columns`` is True)
        output_all_columns (:obj:`bool`, defaults to False). If True, the formatted output is completed using the columns
            that are not in the ``format_columns`` list. For these columns, the PythonFormatter is used.


    Returns:
        A row, column or batch formatted object defined by the Formatter:
        - the PythonFormatter returns a dictionary for a row or a batch, and a list for a column.
        - the NumpyFormatter returns a dictionary for a row or a batch, and a np.array for a column.
        - the PandasFormatter returns a pd.DataFrame for a row or a batch, and a pd.Series for a column.
        - the TorchFormatter returns a dictionary for a row or a batch, and a torch.Tensor for a column.
        - the TFFormatter returns a dictionary for a row or a batch, and a tf.Tensor for a column.
    Nr   )r   r1   c                 3   s   | ]}| kr|V  qd S rH   r#   rd   r   r#   r$   ro   |  s      zformat_table.<locals>.<genexpr>c                 3   s   | ]}| kr|V  qd S rH   r#   rd   r   r#   r$   ro     s     z\Custom formatting function must return a dict to work with output_all_columns=True, but got )
r3   r   r)   r   r   r=   r>   r   r   r&   )r)   r   r   r   Zoutput_all_columnsrG   r   Zpython_formatterZpa_table_to_formatZformatted_outputZpa_table_with_remaining_columnsZremaining_columns_dictr#   r   r$   format_tableP  s0    



r   )N)NF)Ncollections.abcr   r   	functoolsr   typingr   r   r   r   r	   r
   r   r   r   numpyr?   Zpandasr   ZpyarrowrT   	packagingr    r   r   r   Zfeatures.featuresr   r   r   r   r)   r   Zutils.py_utilsr   r   r   r   r   r8   boolr%   r(   r4   r7   r9   r<   r6   r[   rD   rE   rY   rZ   r^   r_   r\   rp   r`   r   r   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r#   r#   r#   r$   <module>   sv   ,  "$ .  p"  0 ,  