U
    饡c8                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dl	Z
d dlmZmZmZmZ edddZG dd dZG d	d
 d
eZG dd deZG dd deZdd Zdd ZdddZdS )    N)join)
implements_stringify_path_is_path_like	_DEPR_MSGzfilesystem.LocalFileSystemz2.0.0zfs.LocalFileSystemc                   @   s   e Zd ZdZdd Zdd Zd(ddZd	d
 Zdd Zdd Z	d)ddZ
dd Zdd Zd*ddZdd Zdd Zdd Zdd Zd+d!d"Zd,d$d%Zed&d' Zd S )-
FileSystemz(
    Abstract filesystem interface.
    c              
   C   s,   |  |d}| W  5 Q R  S Q R X dS )z
        Return contents of file as a bytes object.

        Parameters
        ----------
        path : str
            File path to read content from.

        Returns
        -------
        contents : bytes
        rbN)openread)selfpathf r   6/tmp/pip-unpacked-wheel-qoi2rb4q/pyarrow/filesystem.pycat)   s    zFileSystem.catc                 C   s   t dS )z
        Return list of file paths.

        Parameters
        ----------
        path : str
            Directory to list contents from.
        NNotImplementedErrorr   r   r   r   r   ls9   s    	zFileSystem.lsFc                 C   s   t dS )z
        Delete the indicated file or directory.

        Parameters
        ----------
        path : str
            Path to delete.
        recursive : bool, default False
            If True, also delete child paths for directories.
        Nr   r   r   	recursiver   r   r   deleteD   s    zFileSystem.deletec           	      C   sl   t |}| |}|d dkr&|d S d}| |D ]2\}}}|D ]"}| ||}|| |d 7 }qBq4|S )z
        Compute bytes used by all contents under indicated path in file tree.

        Parameters
        ----------
        path : str
            Can be a file path or directory.

        Returns
        -------
        usage : int
        kindfilesizer   )r   statwalk
_path_join)	r   r   Z	path_infototalrootdirectoriesfilesZ
child_pathabspathr   r   r   
disk_usageQ   s    
zFileSystem.disk_usagec                 G   s   | j |S N)pathsepr   )r   argsr   r   r   r   k   s    zFileSystem._path_joinc                 C   s   t ddS )zl
        Information about a filesystem entry.

        Returns
        -------
        stat : dict
        FileSystem.statNr   r   r   r   r   r   n   s    r'   c                 C   s   | j ||dS )z.
        Alias for FileSystem.delete.
        r   )r   r   r   r   r   rmx   s    zFileSystem.rmc                 C   s   |  ||S )z.
        Alias for FileSystem.rename.
        )renamer   r   new_pathr   r   r   mv~   s    zFileSystem.mvc                 C   s   t ddS )z
        Rename file, like UNIX mv command.

        Parameters
        ----------
        path : str
            Path to alter.
        new_path : str
            Path to move to.
        FileSystem.renameNr   r+   r   r   r   r*      s    r.   Tc                 C   s   t dS )z
        Create a directory.

        Parameters
        ----------
        path : str
            Path to the directory.
        create_parents : bool, default True
            If the parent directories don't exists create them as well.
        Nr   r   r   Zcreate_parentsr   r   r   mkdir   s    zFileSystem.mkdirc                 C   s   t dS )z
        Return True if path exists.

        Parameters
        ----------
        path : str
            Path to check.
        Nr   r   r   r   r   exists   s    	zFileSystem.existsc                 C   s   t dS )z
        Return True if path is a directory.

        Parameters
        ----------
        path : str
            Path to check.
        Nr   r   r   r   r   isdir   s    	zFileSystem.isdirc                 C   s   t dS )z
        Return True if path is a file.

        Parameters
        ----------
        path : str
            Path to check.
        Nr   r   r   r   r   isfile   s    	zFileSystem.isfilec                 C   s   t dS )zf
        Returns True if this FileSystem is a unix-style file store with
        directories.
        Nr   r   r   r   r   _isfilestore   s    zFileSystem._isfilestoreNc           	      C   s,   ddl m} ||||| d}|j|||dS )a]  
        Read Parquet data from path in file system. Can read from a single file
        or a directory of files.

        Parameters
        ----------
        path : str
            Single file path or directory
        columns : List[str], optional
            Subset of columns to read.
        metadata : pyarrow.parquet.FileMetaData
            Known metadata to validate files against.
        schema : pyarrow.parquet.Schema
            Known schema to validate files against. Alternative to metadata
            argument.
        use_threads : bool, default True
            Perform multi-threaded column reads.
        use_pandas_metadata : bool, default False
            If True and file has custom pandas schema metadata, ensure that
            index columns are also loaded.

        Returns
        -------
        table : pyarrow.Table
        r   )ParquetDataset)schemametadata
filesystem)columnsuse_threadsuse_pandas_metadata)Zpyarrow.parquetr6   r
   )	r   r   r:   r8   r7   r;   r<   r6   Zdatasetr   r   r   read_parquet   s    zFileSystem.read_parquetr   c                 C   s   t dS )3
        Open file for reading or writing.
        Nr   r   r   moder   r   r   r	      s    zFileSystem.openc                 C   s   dS )N/r   r4   r   r   r   r%      s    zFileSystem.pathsep)F)F)T)NNNTF)r   )__name__
__module____qualname____doc__r   r   r   r#   r   r   r)   r-   r*   r0   r1   r2   r3   r5   r=   r	   propertyr%   r   r   r   r   r   $   s,   



    
!
r   c                       s   e Zd ZdZ fddZedd Zedd Zee	j
dd	 Z
ee	jdddZee	jdd Zee	jdd Zee	jdd Zee	jdd Zee	jdddZedd Zdd Z  ZS )LocalFileSystemNc                    s   t jttdd t   d S N   
stacklevel)warningswarn_FS_DEPR_MSGFutureWarningsuper__init__r4   	__class__r   r   rQ      s    zLocalFileSystem.__init__c              	   C   s6   | j d kr0t  td t | _ W 5 Q R X | j S )Nignore)	_instancerL   catch_warningssimplefilterrG   clsr   r   r   _get_instance   s
    


zLocalFileSystem._get_instancec                 C   s   t jttdd |  S rH   )rL   rM   rN   rO   rZ   rX   r   r   r   get_instance  s    zLocalFileSystem.get_instancec                    s$   t   t fddt D S )Nc                 3   s   | ]}t  |V  qd S r$   )pjoin.0xr   r   r   	<genexpr>
  s     z%LocalFileSystem.ls.<locals>.<genexpr>)r   sortedoslistdirr   r   r`   r   r     s    zLocalFileSystem.lsTc                 C   s&   t |}|rt| n
t| d S r$   )r   rc   makedirsr0   r/   r   r   r   r0     s    zLocalFileSystem.mkdirc                 C   s   t |}tj|S r$   )r   rc   r   r2   r   r   r   r   r2     s    zLocalFileSystem.isdirc                 C   s   t |}tj|S r$   )r   rc   r   r3   r   r   r   r   r3     s    zLocalFileSystem.isfilec                 C   s   dS )NTr   r4   r   r   r   r5     s    zLocalFileSystem._isfilestorec                 C   s   t |}tj|S r$   )r   rc   r   r1   r   r   r   r   r1   "  s    zLocalFileSystem.existsr   c                 C   s   t |}t||dS r>   )r@   )r   r	   r?   r   r   r   r	   '  s    zLocalFileSystem.openc                 C   s   t jjS r$   )rc   r   sepr4   r   r   r   r%   /  s    zLocalFileSystem.pathsepc                 C   s   t |}t|S )z8
        Directory tree generator, see os.walk.
        )r   rc   r   r   r   r   r   r   3  s    zLocalFileSystem.walk)T)r   )rB   rC   rD   rU   rQ   classmethodrZ   r[   r   r   r   r0   r2   r3   r5   r1   r	   rF   r%   r   __classcell__r   r   rR   r   rG      s.   







rG   c                   @   s   e Zd ZdZdd Zeejdd Zeejdd Zeej	dd	 Z	eej
dddZ
eejdd ZeejdddZeejdddZdddZdd ZdS )DaskFileSystemzJ
    Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.
    c                 C   s   t jdtdd || _d S )Nz~The pyarrow.filesystem.DaskFileSystem/S3FSWrapper are deprecated as of pyarrow 3.0.0, and will be removed in a future version.rI   rJ   )rL   rM   rO   fs)r   rk   r   r   r   rQ   @  s     zDaskFileSystem.__init__c                 C   s   t dd S NzUnsupported file system APIr   r   r   r   r   r2   G  s    zDaskFileSystem.isdirc                 C   s   t dd S rl   r   r   r   r   r   r3   K  s    zDaskFileSystem.isfilec                 C   s   dS )zh
        Object Stores like S3 and GCSFS are based on key lookups, not true
        file-paths.
        Fr   r4   r   r   r   r5   O  s    zDaskFileSystem._isfilestoreFc                 C   s   t |}| jj||dS )Nr(   )r   rk   r)   r   r   r   r   r   W  s    zDaskFileSystem.deletec                 C   s   t |}| j|S r$   )r   rk   r1   r   r   r   r   r1   \  s    zDaskFileSystem.existsTc                 C   s(   t |}|r| j|S | j|S d S r$   )r   rk   Zmkdirsr0   r/   r   r   r   r0   a  s    zDaskFileSystem.mkdirr   c                 C   s   t |}| jj||dS rf   )r   rk   r	   r?   r   r   r   r	   i  s    zDaskFileSystem.openc                 C   s   t |}| jj||dS )N)detail)r   rk   r   )r   r   rm   r   r   r   r   q  s    zDaskFileSystem.lsc                 C   s   t |}| j|S )z9
        Directory tree generator, like os.walk.
        )r   rk   r   r   r   r   r   r   u  s    zDaskFileSystem.walkN)F)T)r   )F)rB   rC   rD   rE   rQ   r   r   r2   r3   r5   r   r1   r0   r	   r   r   r   r   r   r   rj   ;  s$   




rj   c                   @   s:   e Zd Zeejdd Zeejdd Zd	ddZdS )
S3FSWrapperc                 C   s\   t t|}z4| j|}t|dkr8|d |kr8W dS W dS W n tk
rV   Y dS X d S )N   r   FT_sanitize_s3r   rk   r   lenOSErrorr   r   contentsr   r   r   r2     s    
zS3FSWrapper.isdirc                 C   sN   t t|}z&| j|}t|dko0|d |kW S  tk
rH   Y dS X d S )Nro   r   Frp   rt   r   r   r   r3     s    zS3FSWrapper.isfileFc                 #   s   t t|}t  t }t| jj||dD ]<}|d }|d dkrP | q,|d dkr^q,|| q,t fdd|D }tdd  D  | |fV   D ]}| j||dE d	H  qd	S )
z
        Directory tree generator, like os.walk.

        Generator version of what is in s3fs, which yields a flattened list of
        files.
        )refreshZKeyZStorageClassZ	DIRECTORYZBUCKETc                    s"   g | ]}| krt |d  qS ro   	posixpathsplit)r^   r   r    r   r   
<listcomp>  s    z$S3FSWrapper.walk.<locals>.<listcomp>c                 S   s   g | ]}t |d  qS rw   rx   r]   r   r   r   r|     s   N)	rq   r   setlistrk   Z_lsaddrb   r   )r   r   rv   r!   key	directoryr   r{   r   r     s"    zS3FSWrapper.walkN)F)rB   rC   rD   r   r   r2   r3   r   r   r   r   r   rn   }  s
   

rn   c                 C   s   |  dr| ddS | S d S )Nzs3:// )
startswithreplacer`   r   r   r   rq     s    
rq   c                 C   sN   t | }t|tsFdtjkr6tjd }t| |jr6| S td|n| S d S )NfsspeczUnrecognized filesystem: {})	type
issubclassr   sysmodules
isinstanceZAbstractFileSystemrs   format)rk   Zfs_typer   r   r   r   _ensure_filesystem  s    


r   c           	      C   s,  t | s |dk	rtd|| fS |dk	rdt|}t|trDt| }nt| tsXtdn| }||fS t| }tj	
|}|jdks|jdkr|jd}|d }|dkrd	}n|jd
 | }d}t|dkr|d  rt|d }tjj||d}|j}n(|jdkrt }|j}nt }|}||fS )zn
    Return filesystem from path which could be an HDFS URI, a local URI,
    or a plain filesystem path.
    NzVfilesystem passed but where is file-like, so there is nothing to open with filesystem.zPExpected string path; path-like objects are only allowed with a local filesystemhdfsZviewfs:r   r   defaultz://rI   ro   )hostportr   )r   
ValueErrorr   r   rG   r   str	TypeErrorurllibparseurlparseschemenetlocrz   rr   	isnumericintpar   _connectr   rZ   )	wherer9   r   Z
parsed_uriZnetloc_splitr   r   rk   Zfs_pathr   r   r   resolve_filesystem_and_path  sB    


r   )N)rc   ry   r   urllib.parser   rL   os.pathr   r\   Zpyarrowr   Zpyarrow.utilr   r   r   r   r   rN   r   rG   rj   rn   rq   r   r   r   r   r   r   <module>   s(      OIB7