U
    -e1                     @   s~   d dl Z d dlmZ d dlmZ d dlZd dlZddl	m
Z
 ddlmZ ddlmZ er`d dlZG d	d
 d
eedef ZdS )    N)Mapping)TYPE_CHECKING   )config)
map_nested   )TensorFormatterc                       sz   e Zd Zd fdd	Zdd Zdd Zdd	 Zed
ddZe	j
edddZe	j
ddddZe	j
edddZ  ZS )TFFormatterNc                    s    t  j|d || _dd l}d S )N)featuresr   )super__init__tf_tensor_kwargs
tensorflow)selfr
   r   tf	__class__ a/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/formatting/tf_formatter.pyr   !   s    zTFFormatter.__init__c                    s`   dd l t tr\ r\t fdd D r8 S t fdd D r\j S  S )Nr   c                 3   s:   | ]2}t |jo0|j d  jko0|j d  jkV  qdS )r   N)
isinstanceTensorshapedtype.0xcolumnr   r   r   	<genexpr>*   s    z+TFFormatter._consolidate.<locals>.<genexpr>c                 3   s:   | ]2}t |jjfo0|jd ko0|j d jkV  qdS )r   r   N)r   r   ZRaggedTensorndimr   r   r   r   r   r   .   s   )r   r   listallstackZragged)r   r   r   r   r   _consolidate&   s    
zTFFormatter._consolidatec                 C   s   dd l }|d kr|S i }t|tjtjfrFt|jtjrFd|ji}n^t|tjtjfrtt|jtj	rtd|j
i}n0tjrdtjkrdd l}t||jjrt|}|j|f|| jS )Nr   r   PIL)r   r   npnumberndarrayZ
issubdtyper   integerZint64ZfloatingZfloat32r   ZPIL_AVAILABLEsysmodulesZ	PIL.ImageZImageZasarrayZconvert_to_tensorr   )r   valuer   Zdefault_dtyper$   r   r   r   
_tensorize7   s    ""
zTFFormatter._tensorizec                    s   dd l }tjrFdtjkrFdd l}t||jrF |	 
  d S t|drdt||jsd| }t|tjr|jtkr  fdd|D S n&t|ttfr  fdd|D S  |S )Nr   torchr   	__array__c                    s   g | ]}  |qS r   recursive_tensorizer   Z	substructr   r   r   
<listcomp>Y   s     z4TFFormatter._recursive_tensorize.<locals>.<listcomp>c                    s   g | ]}  |qS r   r/   r1   r2   r   r   r3   [   s     )r   r   ZTORCH_AVAILABLEr)   r*   r-   r   r   r,   detachcpunumpyhasattrr.   r%   r'   r   objectr#   r    tuple)r   data_structr   r-   r   r2   r   _recursive_tensorizeK   s    
z TFFormatter._recursive_tensorize)r:   c                 C   s   t | j|ddS )NF)Zmap_list)r   r;   )r   r:   r   r   r   r0   ^   s    zTFFormatter.recursive_tensorize)pa_tablereturnc                 C   s$   |   |}| j|}| |S N)numpy_arrow_extractorZextract_rowpython_features_decoderZ
decode_rowr0   )r   r<   rowr   r   r   
format_rowa   s    zTFFormatter.format_row	tf.Tensorc                 C   s:   |   |}| j||jd }| |}| |}|S )Nr   )r?   Zextract_columnr@   Zdecode_columnZcolumn_namesr0   r#   )r   r<   r   r   r   r   format_columnf   s
    

zTFFormatter.format_columnc                 C   sD   |   |}| j|}| |}|D ]}| || ||< q(|S r>   )r?   Zextract_batchr@   Zdecode_batchr0   r#   )r   r<   batchZcolumn_namer   r   r   format_batchm   s    
zTFFormatter.format_batch)N)__name__
__module____qualname__r   r#   r,   r;   dictr0   paTabler   rB   rD   rF   __classcell__r   r   r   r   r	       s   r	   rC   )r)   collections.abcr   typingr   r6   r%   ZpyarrowrK    r   Zutils.py_utilsr   Z
formattingr   r   r   r	   r   r   r   r   <module>   s   