U
    9%e0                      @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZmZmZmZ dd ZdaG dd	 d	eZed
dddddddddg	G dd dejZG dd deZdS )    N)suppress)cached_propertywraps)AbstractFileSystem)"get_package_version_without_importinfer_storage_optionsmirror_fromtokenizec                    s   t   fdd}|S )Nc               
      sj   z | |W S  t k
rd } z:|js& |j^}} t|trRd|krRttj||n W 5 d }~X Y nX d S )Nzdoes not exist)OSErrorargs
isinstancestrFileNotFoundErrorerrnoENOENT)r   kwargs	exceptionmessagefunc [/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/fsspec/implementations/arrow.pywrapper   s    
z wrap_exceptions.<locals>.wrapper)r   )r   r   r   r   r   wrap_exceptions   s    r   c                       s   e Zd ZdZdZ fddZedd Zedd Z	e
d	d
 Zd-ddZdd Zdd Zdd Zedd Zedd ZeZedd Zed.ddZed/dd Zed0d!d"Zed1d#d$Zed%d& Zed'd( Zd2 fd)d*	Z fd+d,Z  ZS )3ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem.

    Parameters
    ----------
    fs : pyarrow.fs.FileSystem

    /c                    s    t da|| _t jf | d S )NZpyarrow)r   PYARROW_VERSIONfssuper__init__)selfr   r   	__class__r   r   r   2   s    zArrowFSWrapper.__init__c                 C   s   | j jS N)r   	type_namer    r   r   r   protocol8   s    zArrowFSWrapper.protocolc                 C   s   dt | jj| jj S )NZhdfs_)r	   r   hostportr%   r   r   r   fsid<   s    zArrowFSWrapper.fsidc                 C   s*   t |}|d }|dr&|dd  }|S )Npathz//   )r   
startswith)clsr*   opsr   r   r   _strip_protocol@   s
    
zArrowFSWrapper._strip_protocolFc                    sN     |}ddlm}  fdd j||D }|r<|S dd |D S d S )Nr   )FileSelectorc                    s   g | ]}  |qS r   )_make_entry.0entryr%   r   r   
<listcomp>M   s   z%ArrowFSWrapper.ls.<locals>.<listcomp>c                 S   s   g | ]}|d  qS )namer   r2   r   r   r   r5   T   s     )r/   
pyarrow.fsr0   r   get_file_info)r    r*   detailr   r0   entriesr   r%   r   lsI   s    

zArrowFSWrapper.lsc                 K   s$   |  |}| j|g\}| |S r#   )r/   r   r8   r1   )r    r*   r   infor   r   r   r<   V   s    
zArrowFSWrapper.infoc                 C   s8   |  |}z| | W n tk
r.   Y dS X dS d S )NFT)r/   r<   r   r    r*   r   r   r   exists[   s    
zArrowFSWrapper.existsc                 C   sn   ddl m} |j|jkrd}n<|j|jkr0d}n*|j|jkrVttjt	
tj|jnd}|j|j||jdS )Nr   )FileType	directoryfileother)r6   sizetypemtime)r7   r?   rD   Z	DirectoryFileZNotFoundr   r   r   osstrerrorr*   rC   rE   )r    r<   r?   kindr   r   r   r1   d   s    zArrowFSWrapper._make_entryc                 K   s   |  |d}|  |d}| |d}| dtd }z6| |d}t|| W 5 Q R X | j	|| W n6 t
k
r   tt | j| W 5 Q R X  Y nX W 5 Q R X d S )Nr   rbz.tmp.   wb)r/   rstrip_opensecretsZ	token_hexopenshutilcopyfileobjr   moveBaseExceptionr   r   delete_file)r    path1path2r   ZlstreamZ	tmp_fnameZrstreamr   r   r   cp_filew   s    
zArrowFSWrapper.cp_filec                 K   s2   |  |d}|  |d}| j|| d S )Nr   )r/   rM   r   rS   )r    rV   rW   r   r   r   r   mv   s    zArrowFSWrapper.mvc                 C   s   |  |}| j| d S r#   )r/   r   rU   r=   r   r   r   rm_file   s    
zArrowFSWrapper.rm_fileNc                 C   sF   |  |d}| |r6|r,| j| qBtdn| j| d S )Nr   z0Can't delete directories without recursive=False)r/   rM   isdirr   
delete_dir
ValueErrorrU   )r    r*   	recursiveZmaxdepthr   r   r   rm   s    

zArrowFSWrapper.rmrJ   Tc           	      K   s   |dkr |r| j j}qR| j j}n2|dkr2| j j}n |dkrD| j j}ntd|i }|dksb|sttdd dkrd |d< ||f|}t	| ||||f|S )	NrJ   rL   abz'unsupported mode for Arrow filesystem: .r      compression)
r   Zopen_input_fileZopen_input_streamZopen_output_streamZopen_append_streamr]   intr   split	ArrowFile)	r    r*   mode
block_sizeseekabler   method_kwargsstreamr   r   r   rN      s    



zArrowFSWrapper._openc                 K   s2   |  |}|r| j|dd n| jj|dd d S )NT)exist_okFr^   )r/   makedirsr   
create_dir)r    r*   Zcreate_parentsr   r   r   r   mkdir   s    
zArrowFSWrapper.mkdirc                 C   s   |  |}| jj|dd d S )NTrn   )r/   r   rp   )r    r*   rm   r   r   r   ro      s    
zArrowFSWrapper.makedirsc                 C   s   |  |}| j| d S r#   )r/   r   r\   r=   r   r   r   rmdir   s    
zArrowFSWrapper.rmdirc                 C   s   |  |}| j|jS r#   )r/   r   r8   rE   r=   r   r   r   modified   s    
zArrowFSWrapper.modifiedc                    s&   |dk|d< t  j|fd d d|S )N)Nr   ri   )startend)r   cat_file)r    r*   rt   ru   r   r!   r   r   rv      s    zArrowFSWrapper.cat_filec                    s   d|d< t  j||f| d S )NFri   )r   get_file)r    ZrpathZlpathr   r!   r   r   rw      s    zArrowFSWrapper.get_file)F)FN)rJ   NT)T)F)NN)__name__
__module____qualname____doc__Zroot_markerr   propertyr&   r   r)   classmethodr/   r;   r<   r>   r1   r   rX   rY   Zmv_filerZ   r_   rN   rq   ro   rr   rs   rv   rw   __classcell__r   r   r!   r   r   '   sD   



	





r   rl   readseektellwritereadablewritablecloserC   ri   c                   @   s&   e Zd ZdddZdd Zdd ZdS )	rf   Nc                 K   s.   || _ || _|| _|| _| | _| _|| _d S r#   )r*   rg   r   rl   	blocksizerh   r   )r    r   rl   r*   rg   rh   r   r   r   r   r      s    zArrowFile.__init__c                 C   s   | S r#   r   r%   r   r   r   	__enter__   s    zArrowFile.__enter__c                 G   s   |   S r#   )r   )r    r   r   r   r   __exit__   s    zArrowFile.__exit__)N)rx   ry   rz   r   r   r   r   r   r   r   rf      s   

rf   c                       s2   e Zd ZdZdZd
 fdd	Zedd	 Z  ZS )HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem
    to connect it's interface with fsspecZhdfsdefaultr   Nc           	         s8   ddl m} ||||||d}t jf d|i| dS )a  

        Parameters
        ----------
        host: str
            Hostname, IP or "default" to try to read from Hadoop config
        port: int
            Port to connect on, or default from Hadoop config if 0
        user: str or None
            If given, connect as this username
        kerb_ticket: str or None
            If given, use this ticket for authentication
        extra_conf: None or dict
            Passed on to HadoopFileSystem
        r   )r   )r'   r(   userkerb_ticket
extra_confr   N)r7   r   r   r   )	r    r'   r(   r   r   r   r   r   r   r!   r   r   r      s    zHadoopFileSystem.__init__c                 C   sX   t | }i }|dd r$|d |d< |dd r<|d |d< |dd rT|d |d< |S )Nr'   usernamer   r(   )r   get)r*   r.   outr   r   r   _get_kwargs_from_urls  s    z&HadoopFileSystem._get_kwargs_from_urls)r   r   NNN)	rx   ry   rz   r{   r&   r   staticmethodr   r~   r   r   r!   r   r      s        #r   )r   iorG   rO   rQ   
contextlibr   	functoolsr   r   Zfsspec.specr   Zfsspec.utilsr   r   r   r	   r   r   r   IOBaserf   r   r   r   r   r   <module>   s6    0