U
    -e                     @   sv   d dl Z d dlmZ d dlmZ d dlZd dlZd dlm	Z	 ej
jeZeG dd dejZG dd dejZdS )	    N)	dataclass)Optional)
table_castc                   @   s$   e Zd ZU dZdZeej ed< dS )ArrowConfigzBuilderConfig for Arrow.Nfeatures)	__name__
__module____qualname____doc__r   r   datasetsFeatures__annotations__ r   r   f/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/packaged_modules/arrow/arrow.pyr      s   
r   c                   @   s<   e Zd ZeZdd Zdd ZejejdddZ	dd	 Z
d
S )Arrowc                 C   s   t j| jjdS )N)r   )r   ZDatasetInfoconfigr   )selfr   r   r   _info   s    zArrow._infoc                    s  | j jstd| j j  | j j}t|tttfrv|}t|trL|g} fdd|D }tj	tj
jd|idgS g }| D ]\}}t|tr|g} fdd|D }| jjdkrtj|D ]8}t|d }tjtj|j| j_W 5 Q R X  qq|tj	|d|id q|S )	z-We handle string, list and dicts in datafilesz=At least one data file must be specified, but got data_files=c                    s   g | ]}  |qS r   Z
iter_files.0file
dl_managerr   r   
<listcomp>%   s     z+Arrow._split_generators.<locals>.<listcomp>files)nameZ
gen_kwargsc                    s   g | ]}  |qS r   r   r   r   r   r   r   ,   s     Nrb)r   
data_files
ValueErrorZdownload_and_extract
isinstancestrlisttupler   ZSplitGeneratorZSplitZTRAINitemsinfor   	itertoolschainfrom_iterableopenr   Zfrom_arrow_schemapaipcopen_streamZschemaappend)r   r   r   r   ZsplitsZ
split_namer   fr   r   r   _split_generators   s*    

$zArrow._split_generators)pa_tablereturnc                 C   s    | j jd k	rt|| j jj}|S )N)r%   r   r   Zarrow_schema)r   r0   r   r   r   _cast_table6   s    zArrow._cast_tablec           	      c   s   t tj|D ]\}}t|d}zFt tj|D ]0\}}tj	|g}| d| | 
|fV  q6W nD tk
r } z&td| dt| d|   W 5 d }~X Y nX W 5 Q R X qd S )Nr   _zFailed to read file 'z' with error z: )	enumerater&   r'   r(   r)   r*   r+   r,   TableZfrom_batchesr2   r   loggererrortype)	r   r   Zfile_idxr   r.   Z	batch_idxZrecord_batchr0   er   r   r   _generate_tables=   s      zArrow._generate_tablesN)r   r   r	   r   ZBUILDER_CONFIG_CLASSr   r/   r*   r5   r2   r:   r   r   r   r   r      s
   r   )r&   dataclassesr   typingr   Zpyarrowr*   r   Zdatasets.tabler   utilsloggingZ
get_loggerr   r6   ZBuilderConfigr   ZArrowBasedBuilderr   r   r   r   r   <module>   s   