U
    -eP                     @   s   d dl Z d dlZd dlmZmZmZ d dlZddlmZm	Z	m
Z
mZ ddlmZ ddlmZ ddlmZ ddlmZmZ d	d
lmZ G dd deZG dd dZdS )    N)BinaryIOOptionalUnion   )DatasetFeatures
NamedSplitconfig)query_table)Json)logging)NestedDataStructureLikePathLike   )AbstractDatasetReaderc                
       sN   e Zd Zdee ee ee ee	e	ee ee
 d fddZdd Z  ZS )	JsonDatasetReaderNF)path_or_pathssplitfeatures	cache_dirkeep_in_memory	streamingfieldnum_procc	           
   	      s^   t  j|f||||||d|	 || _t|tr6|n| j|i}tf ||||d|	| _d S )N)r   r   r   r   r   r   )r   
data_filesr   r   )super__init__r   
isinstancedictr   r   builder)
selfr   r   r   r   r   r   r   r   kwargs	__class__ Q/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/io/json.pyr      s,    
zJsonDatasetReader.__init__c                 C   sZ   | j r| jj| jd}n>d }d }d }d }| jj||||| jd | jj| j|| jd}|S )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r(   Z	in_memory)r   r   Zas_streaming_datasetr   Zdownload_and_preparer   Z
as_datasetr   )r    datasetr&   r'   r(   r)   r$   r$   r%   read0   s&      zJsonDatasetReader.read)NNNFFNN)__name__
__module____qualname__r   r   r   r   r   strboolintr   r+   __classcell__r$   r$   r"   r%   r      s$           r   c                   @   sX   e Zd Zdeeeef ee ee dddZ	edddZ
dd	 Zeed
ddZdS )JsonDatasetWriterN)r*   path_or_buf
batch_sizer   c                 K   sR   |d k	r |dkr t d| d|| _|| _|r4|ntj| _|| _d| _|| _d S )Nr   z	num_proc z must be an integer > 0.zutf-8)	
ValueErrorr*   r4   r	   ZDEFAULT_MAX_BATCH_SIZEr5   r   encodingto_json_kwargs)r    r*   r4   r5   r   r8   r$   r$   r%   r   J   s    zJsonDatasetWriter.__init__)returnc              	   C   s   | j dd }| j dd}| j d|dkr0dnd}d| j krR|dkrRd| j d< | j d	d }|d
krxtd| dt| jtttjfrt	j
| jd|d }| jf |||d| j }W 5 Q R X n0|rtd| d| jf | j||d| j }|S )Nr4   orientrecordslinesTFindex)r   tablecompression)NZinfergzipbz2xzz&`datasets` currently does not support z compressionwb)r?   )file_objr:   r<   zUThe compression parameter is not supported when writing to a buffer, but compression=z1 was passed. Please provide a local path instead.)r8   popNotImplementedErrorr   r4   r/   bytesosr   fsspecopen_write)r    _r:   r<   r?   bufferwrittenr$   r$   r%   write\   s"    
&
zJsonDatasetWriter.writec                 C   sh   |\}}}}t | jjt||| j | jjd}| jf d ||d|}|ds\|d7 }|	| j
S )N)r>   keyindices)r4   r:   r<   
)r
   r*   dataslicer5   Z_indicesZ	to_pandasto_jsonendswithencoder7   )r    argsoffsetr:   r<   r8   batchjson_strr$   r$   r%   _batch_jsons   s    
zJsonDatasetWriter._batch_json)rD   r9   c              
      s   d}| j dks| j dkrhtjtdt| j| jdt  ddD ]$}| | f}||	|7 }q@nt| j| j }}	t
| j p}
tj|
| j fddtd||	D ||	 r||	 d n||	 dt  dd	D ]}||	|7 }qW 5 Q R X |S )
zWrites the pyarrow table as JSON lines to a binary file handle.

        Caller is responsible for opening and closing the handle.
        r   Nr   bazCreating json from Arrow format)unitdisabledescc                    s   g | ]}| fqS r$   r$   ).0rY   r<   r:   r8   r$   r%   
<listcomp>   s     z,JsonDatasetWriter._write.<locals>.<listcomp>)totalr^   r_   r`   )r   r   Ztqdmrangelenr*   r5   Zis_progress_bar_enabledr\   rO   multiprocessingZPoolimap)r    rD   r:   r<   r8   rN   rY   r[   Znum_rowsr5   poolr$   rb   r%   rK      s0    


zJsonDatasetWriter._write)NN)r,   r-   r.   r   r   r   r   r   r1   r   rO   r\   rK   r$   r$   r$   r%   r3   I   s     
r3   )rg   rH   typingr   r   r   rI    r   r   r   r	   Z
formattingr
   Zpackaged_modules.json.jsonr   utilsr   Zutils.typingr   r   abcr   r   r3   r$   r$   r$   r%   <module>   s   :