U
    -e                     @   s\   d dl mZ d dlZddlmZmZ ddlmZ ddlm	Z	 ddl
mZ G d	d
 d
eZdS )    )OptionalN   )Features
NamedSplit)DownloadMode)Spark   )AbstractDatasetReaderc                       sL   e Zd ZdZdejjee ee	 e
ee
ee
ed	 fddZd	d
 Z  ZS )SparkDatasetReaderzA dataset reader that reads from a Spark DataFrame.

    When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
    provided. Streaming is not currently supported.
    NTFarrow)	dfsplitfeatures	streaming	cache_dirkeep_in_memoryworking_dirload_from_cache_filefile_formatc
                    sH   t  jf |||||d|
 || _|	| _tf ||||d|
| _d S )N)r   r   r   r   r   )r   r   r   r   )super__init___load_from_cache_file_file_formatr   builder)selfr   r   r   r   r   r   r   r   r   kwargs	__class__ R/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/io/spark.pyr      s&    zSparkDatasetReader.__init__c                 C   sH   | j r| jj| jdS | jr d ntj}| jj|| jd | jj	| jdS )N)r   )download_moder   )
r   r   Zas_streaming_datasetr   r   r   ZFORCE_REDOWNLOADZdownload_and_preparer   Z
as_dataset)r   r    r   r   r   read1   s    zSparkDatasetReader.read)NNTNFNTr   )__name__
__module____qualname____doc__pysparksqlZ	DataFramer   r   r   boolstrr   r!   __classcell__r   r   r   r   r
      s*   	        r
   )typingr   r&    r   r   downloadr   Zpackaged_modules.spark.sparkr   abcr	   r
   r   r   r   r   <module>   s   