U
    -e9                     @   sb  d Z ddlmZmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZ eZg ZzddlmZ W n ek
r~   ed Y nX zddlmZ W n ek
r   ed Y nX z0ddlmZmZmZmZmZmZmZm Z m!Z! W n ek
r   ed	 Y nX e  dd
l"Z"e"#e dd Z$dd Z%dddZ&dddZ'ddddddZ(G dd deZ)d
S )zO
FileSystem abstraction to interact with various local and remote filesystems.
    )_is_path_like_stringify_path)FileSelectorFileTypeFileInfo
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystem_copy_files_copy_files_selector)HadoopFileSystemr   )GcsFileSystemr   )	AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyS3FileSystem
S3LogLevelS3RetryStrategyensure_s3_initializedfinalize_s3initialize_s3resolve_s3_regionr   Nc                 C   s(   | t krtd| td| d S )Nz<The pyarrow installation is not built with support for '{0}'z*module 'pyarrow.fs' has no attribute '{0}')_not_importedImportErrorformatAttributeError)name r   K/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/fs.py__getattr__B   s    r!   c                 C   s^   t | \}}||}|rZ||gd }|jtjkrPtd|jj	|j
| t||}|S )Nr   zThe path component of the filesystem URI must point to a directory but it has a type: `{}`. The path component is `{}` and the given filesystem URI is `{}`)r   from_urinormalize_pathget_file_infotyper   	Directory
ValueErrorr   r   pathr	   )uri
filesystemprefixZprefix_infor   r   r    _filesystem_from_strN   s    
  
r,   Fc                 C   s   t | tr| S t | tr,|r$tdt| S zdd l}W n tk
rL   Y n2X t | |jr~t| j	dkrrt
|dS tt| S dd lm} t | |j
rt
|dS |rt | |jr| S tdt| d S )NzXSpecifying to use memory mapping not supported for filesystem specified as an URI stringr   r   use_mmapzlUnrecognized filesystem: {}. `filesystem` argument must be a FileSystem instance or a valid file system URI')
isinstancer   strr'   r,   fsspecr   ZAbstractFileSystemr%   __name__r   r   FSSpecHandlerZpyarrow.filesystemr*   	TypeErrorr   )r*   r.   allow_legacy_filesystemr1   Zlegacyfsr   r   r    _ensure_filesystemb   s4    



r6   c              
   C   s,  t | s |dk	rtd|| fS |dk	rrt|||d}t|trJt| } nt| ts\td|sj|| } || fS t| } t|d}z|	| }W n tk
r   d}d}Y nX |j
tjk}|szt| \}} W nB tk
r } z"dt|krdt|kr W 5 d}~X Y nX n
|| } || fS )	z`
    Return filesystem/path from path which could be an URI or a plain
    filesystem path.
    Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.)r.   r5   zPExpected string path; path-like objects are only allowed with a local filesystemr-   Fzempty schemezCannot parse URI)r   r'   r6   r/   r   r   r0   r4   r#   r$   r%   r   NotFoundr   r"   )r(   r*   r5   Z
memory_map	file_infoZexists_locallyer   r   r    _resolve_filesystem_and_path   sN     







r:   i   T)
chunk_sizeuse_threadsc                C   sh   t | |\}}t ||\}}	||}
|
jtjkrRt|dd}t||||	|| nt||||	|| dS )a  
    Copy files between FileSystems.

    This functions allows you to recursively copy directories of files from
    one file system to another, such as from S3 to your local machine.

    Parameters
    ----------
    source : string
        Source file path or URI to a single file or directory.
        If a directory, files will be copied recursively from this path.
    destination : string
        Destination file path or URI. If `source` is a file, `destination`
        is also interpreted as the destination file (not directory).
        Directories will be created as necessary.
    source_filesystem : FileSystem, optional
        Source filesystem, needs to be specified if `source` is not a URI,
        otherwise inferred.
    destination_filesystem : FileSystem, optional
        Destination filesystem, needs to be specified if `destination` is not
        a URI, otherwise inferred.
    chunk_size : int, default 1MB
        The maximum size of block to read before flushing to the
        destination file. A larger chunk_size will use more memory while
        copying but may help accommodate high latency FileSystems.
    use_threads : bool, default True
        Whether to use multiple threads to accelerate copying.

    Examples
    --------
    Inspect an S3 bucket's files:

    >>> s3, path = fs.FileSystem.from_uri(
    ...            "s3://registry.opendata.aws/roda/ndjson/")
    >>> selector = fs.FileSelector(path)
    >>> s3.get_file_info(selector)
    [<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]

    Copy one file from S3 bucket to a local directory:

    >>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path))

    >>> fs.LocalFileSystem().get_file_info(str(local_path)+
    ...                                    '/index_copy.ndjson')
    <FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...>

    Copy file using a FileSystem object:

    >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path),
    ...               source_filesystem=fs.S3FileSystem())
    T	recursiveN)r:   r$   r%   r   r&   r   r   r   )sourcedestinationZsource_filesystemZdestination_filesystemr;   r<   Z	source_fssource_pathZdestination_fsZdestination_pathr8   Z
source_selr   r   r    
copy_files   s.    8  
    rB   c                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Ze	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*S )+r3   a  
    Handler for fsspec-based Python filesystems.

    https://filesystem-spec.readthedocs.io/en/latest/index.html

    Parameters
    ----------
    fs : FSSpec-compliant filesystem instance

    Examples
    --------
    >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP
    c                 C   s
   || _ d S N)fs)selfrD   r   r   r    __init__%  s    zFSSpecHandler.__init__c                 C   s   t |tr| j|jkS tS rC   r/   r3   rD   NotImplementedrE   otherr   r   r    __eq__(  s    
zFSSpecHandler.__eq__c                 C   s   t |tr| j|jkS tS rC   rG   rI   r   r   r    __ne__-  s    
zFSSpecHandler.__ne__c                 C   s$   | j j}t|tr|d }d|S )Nr   z
fsspec+{0})rD   protocolr/   listr   )rE   rM   r   r   r    get_type_name2  s    
zFSSpecHandler.get_type_namec                 C   s   |S rC   r   rE   r(   r   r   r    r#   8  s    zFSSpecHandler.normalize_pathc                 C   sR   |d }|d dkrt j}n|d dkr4t j}d }nt j}t| |||dd dS )Nsizer%   file	directorymtime)rQ   rT   )r   Filer&   Unknownr   get)r(   inforQ   Zftyper   r   r    _create_file_info;  s    zFSSpecHandler._create_file_infoc              	   C   s\   g }|D ]N}z| j |}W n& tk
rB   |t|tj Y qX || || q|S rC   )rD   rX   FileNotFoundErrorappendr   r   r7   rY   )rE   pathsinfosr(   rX   r   r   r    r$   H  s    zFSSpecHandler.get_file_infoc                 C   s   | j |js<| j |jr(t|jn|jr2g S t|j|jrHd }nd}g }| j j|j|ddd}|	 D ]\}}|
| || qn|S )N   T)maxdepthZwithdirsdetail)rD   isdirbase_direxistsNotADirectoryErrorZallow_not_foundrZ   r>   finditemsr[   rY   )rE   selectorr_   r]   Zselected_filesr(   rX   r   r   r    get_file_info_selectorS  s&    
   z$FSSpecHandler.get_file_info_selectorc                 C   s.   z| j j||d W n tk
r(   Y nX d S )N)Zcreate_parents)rD   mkdirFileExistsError)rE   r(   r>   r   r   r    
create_dirk  s    zFSSpecHandler.create_dirc                 C   s   | j j|dd d S NTr=   )rD   rmrP   r   r   r    
delete_dirr  s    zFSSpecHandler.delete_dirc                 C   sz   z| j j|dd}W n  tk
r4   |r.Y d S  Y nX |D ]:}| j |r\| j j|dd q:| j |r:| j | q:d S )NF)r`   Tr=   )rD   listdirrZ   ra   rm   isfile)rE   r(   missing_dir_okZsubpathssubpathr   r   r    _delete_dir_contentsu  s    z"FSSpecHandler._delete_dir_contentsc                 C   s*   | ddkrtd|d| || d S )N/ z$delete_dir_contents called on path '')stripr'   rs   )rE   r(   rq   r   r   r    delete_dir_contents  s      z!FSSpecHandler.delete_dir_contentsc                 C   s   |  d d S )Nrt   )rs   )rE   r   r   r    delete_root_dir_contents  s    z&FSSpecHandler.delete_root_dir_contentsc                 C   s$   | j |st|| j | d S rC   )rD   rc   rZ   rm   rP   r   r   r    delete_file  s    zFSSpecHandler.delete_filec                 C   s   | j j||dd d S rl   )rD   mvrE   srcdestr   r   r    move  s    zFSSpecHandler.movec                 C   s   | j || d S rC   )rD   copyr|   r   r   r    	copy_file  s    zFSSpecHandler.copy_filec                 C   s8   ddl m} | j|s t||| jj|ddddS Nr   
PythonFilerbmoderpyarrowr   rD   rp   rZ   openrE   r(   r   r   r   r    open_input_stream  s    zFSSpecHandler.open_input_streamc                 C   s8   ddl m} | j|s t||| jj|ddddS r   r   r   r   r   r    open_input_file  s    zFSSpecHandler.open_input_filec                 C   s$   ddl m} || jj|ddddS )Nr   r   wbr   wr   r   rD   r   rE   r(   metadatar   r   r   r    open_output_stream  s    z FSSpecHandler.open_output_streamc                 C   s$   ddl m} || jj|ddddS )Nr   r   abr   r   r   r   r   r   r    open_append_stream  s    z FSSpecHandler.open_append_streamN)r2   
__module____qualname____doc__rF   rK   rL   rO   r#   staticmethodrY   r$   rh   rk   rn   rs   rx   ry   rz   r   r   r   r   r   r   r   r   r   r    r3     s,   
r3   )FF)NFF)NN)*r   Zpyarrow.utilr   r   Zpyarrow._fsr   r   r   r   r   r	   r
   r   r   r   r   Z	FileStatsr   Zpyarrow._hdfsr   r   r[   Zpyarrow._gcsfsr   Zpyarrow._s3fsr   r   r   r   r   r   r   r   r   atexitregisterr!   r,   r6   r:   rB   r3   r   r   r   r    <module>   sF   40
   
*     
A    K