U
    -e                     @   s   d dl Z d dlZd dlmZ d dlmZmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddl m!Z! ddl"m#Z# dd	l$m%Z% e!e&Z'erdd
l(m)Z) dee dddZ*ddddZ+dS )    N)wraps)TYPE_CHECKINGOptional   )DownloadConfig)	xbasenamexdirname	xet_parsexexistsxgetsizexglob
xgzip_openxisdirxisfilexjoinxlistdirxnumpy_loadxopenxpandas_read_csvxpandas_read_excelxPathxrelpathxsio_loadmatxsplit	xsplitextxwalkxxml_dom_minidom_parse)
get_logger)patch_submodule)get_importsDatasetBuilderdownload_configc                    s  t | }t|dr>|jr>t|jtr: j|j_ j|j_dS  fdd}t|d|t	
  t|d|t
  t|d|t
  t|d|t
  t|d	t
  t|d
t
  t|dt
  t|dt
  t|dt
  t|dt
  t|d|t
  t|d|t
  t|d|t
  t|d|t
  t|dt
  t|d|t
  t|d|t
  t|d|tdgd
  t|d|tdgd
  t|d|tdgd
  t|d|t
  t|d|t
   |_dS )aE  Extend the module to support streaming.

    We patch some functions in the module to use `fsspec` to support data streaming:
    - We use `fsspec.open` to open and read remote files. We patch the module function:
      - `open`
    - We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module
      functions:
      - `os.path.join`
      - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator)

    The patched functions are replaced with custom functions defined to work with the
    :class:`~download.streaming_download_manager.StreamingDownloadManager`.

    Args:
        module_path: Path to the module to be extended.
        download_config : mainly use use_auth_token or storage_options to support different platforms and auth types.
    _patched_for_streamingNc                    s    t   fdd}d|_|S )Nc                     s   | d i|S )Nr#    )argskwargs)r#   functionr%   S/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/streaming.pywrapperH   s    z?extend_module_for_streaming.<locals>.wrap_auth.<locals>.wrapper	wrap_auth)r   Z_decorator_name_)r(   r*   r"   )r(   r)   r+   G   s    z.extend_module_for_streaming.<locals>.wrap_authopenz
os.listdirzos.walkz	glob.globzos.path.joinzos.path.dirnamezos.path.basenamezos.path.relpathzos.path.splitzos.path.splitextzos.path.existszos.path.isdirzos.path.isfilezos.path.getsizezpathlib.Pathz	gzip.openz
numpy.loadzpandas.read_csv__version__)attrszpandas.read_excelzscipy.io.loadmatzxml.etree.ElementTree.parsezxml.dom.minidom.parse) 	importlibimport_modulehasattrr$   
isinstancer   tokenstorage_optionsr   r   startr   r   r   r   r   r   r   r   r   r
   r   r   r   r   r   r   r   r   r   r	   r   )module_pathr#   moduler+   r%   r"   r)   extend_module_for_streaming+   s<    


	r8   r!   )builderc                    s   t | j| jd}t| j|d | jdstt| j	D ]D}|d dkr:|d }d
| jddd	 |g }t||d q:dd
lm   fddt| jdd D }|D ]}t||d qdS )zExtend the dataset builder module and the modules imported by it to support streaming.

    Args:
        builder (:class:`DatasetBuilder`): Dataset builder instance.
    )r4   r3   r"   z	datasets.r   Zinternalr   .Nr    c                    s(   g | ] }t | r|j jkr|jqS r%   )
issubclass
__module__).0clsr    r%   r)   
<listcomp>   s   
 z8extend_dataset_builder_for_streaming.<locals>.<listcomp>)r   r4   r3   r8   r=   
startswithr   inspectgetfile	__class__joinsplitr9   r!   type__mro__)r9   r#   ZimportsZinternal_import_nameZinternal_module_nameZparent_builder_modulesr7   r%   r    r)   $extend_dataset_builder_for_streamingl   s     
rI   )N),r/   rB   	functoolsr   typingr   r   Zdownload.download_configr   Z#download.streaming_download_managerr   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Zutils.loggingr   Zutils.patchingr   Zutils.py_utilsr   __name__loggerr9   r!   r8   rI   r%   r%   r%   r)   <module>   s   `A