U
    -e=                     @   s   d dl Z d dlmZmZmZ ddlmZmZmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ ertd dlZd dlZG d
d deZG dd dZdS )    N)TYPE_CHECKINGOptionalUnion   )DatasetFeaturesconfig)query_table)Sql)logging   )AbstractDatasetInputStreamc                       sL   e Zd Zdeedf eedddf ee eed fdd	Zd
d Z	  Z
S )SqlDatasetReaderNFzsqlalchemy.sql.Selectablesqlalchemy.engine.Connectionsqlalchemy.engine.Enginesqlite3.Connection)sqlconfeatures	cache_dirkeep_in_memoryc                    s8   t  jf |||d| tf ||||d|| _d S )N)r   r   r   )r   r   r   r   )super__init__r
   builder)selfr   r   r   r   r   kwargs	__class__ P/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/io/sql.pyr      s    	zSqlDatasetReader.__init__c                 C   s<   d }d }d }d }| j j||||d | j jd|| jd}|S )N)download_configdownload_modeverification_mode	base_pathtrain)splitr"   Z	in_memory)r   Zdownload_and_prepareZ
as_datasetr   )r   r    r!   r"   r#   datasetr   r   r   read$   s     	  zSqlDatasetReader.read)NNF)__name__
__module____qualname__r   strr   r   boolr   r'   __classcell__r   r   r   r   r      s      
r   c                   @   s\   e Zd Zdeeeedddf ee ee dddZedd	d
Z	dd Z
edddZdS )SqlDatasetWriterNr   r   r   )r&   namer   
batch_sizenum_procc                 K   sR   |d k	r |dkr t d| d|| _|| _|| _|r:|ntj| _|| _|| _d S )Nr   z	num_proc z must be an integer > 0.)	
ValueErrorr&   r/   r   r   ZDEFAULT_MAX_BATCH_SIZEr0   r1   to_sql_kwargs)r   r&   r/   r   r0   r1   r3   r   r   r   r   :   s    	zSqlDatasetWriter.__init__)returnc                 C   sD   | j dd }| j dd }| j dd}| jf d|i| j }|S )Nr   r   indexF)r3   pop_write)r   _r5   writtenr   r   r   writeM   s
    zSqlDatasetWriter.writec                 C   st   |\}}}|dkr|ddin|}t | jjt||| j | jjd}| }|j| j| j	fd|i|}|prt
|S )Nr   Z	if_existsappend)tablekeyindicesr5   )r	   r&   dataslicer0   Z_indicesZ	to_pandasZto_sqlr/   r   len)r   argsoffsetr5   r3   batchZdfnum_rowsr   r   r   
_batch_sqlU   s    
zSqlDatasetWriter._batch_sqlc              
      s   d}| j dks| j dkr\tjtdt| j| jdt  ddD ]}|| | f7 }q@nt| j| j }}t	
| j h}tj|| j fddtd||D || r|| d n|| dt  dd	D ]}||7 }qW 5 Q R X |S )
zWrites the pyarrow table as SQL to a database.

        Caller is responsible for opening and closing the SQL connection.
        r   Nr   bazCreating SQL from Arrow format)unitdisabledescc                    s   g | ]}| fqS r   r   ).0rC   r5   r3   r   r   
<listcomp>v   s     z+SqlDatasetWriter._write.<locals>.<listcomp>)totalrH   rI   rJ   )r1   r   ZtqdmrangerA   r&   r0   Zis_progress_bar_enabledrF   multiprocessingZPoolimap)r   r5   r3   r9   rC   rE   r0   poolr   rL   r   r7   a   s.    


zSqlDatasetWriter._write)NN)r(   r)   r*   r   r+   r   r   intr   r:   rF   r7   r   r   r   r   r.   9   s     r.   )rP   typingr   r   r    r   r   r   Z
formattingr	   Zpackaged_modules.sql.sqlr
   utilsr   abcr   sqlite3Z
sqlalchemyr   r.   r   r   r   r   <module>   s   (