U
    9%eV                  
   @   s  d dl Z d dlZd dlZd dlZd dlmZ ddlmZmZm	Z	m
Z
mZmZ ddlmZ ddlmZmZ ddlmZmZmZmZ edZG d	d
 d
ZG dd deZd(ddZdd Zdd Zd)ddZd*ddZ dd Z!dd Z"dd Z#d d! Z$d+d"d#Z%d$d% Z&G d&d' d'e j'Z(dS ),    N)	has_magic   )	BaseCache
BlockCache
BytesCache	MMapCacheReadAheadCachecaches)compr)
filesystemget_filesystem_class)_unstrip_protocolbuild_name_functioninfer_compressionstringify_pathZfsspecc                   @   sV   e Zd ZdZdddZdd Zdd	 Zd
d Zdd Ze	dd Z
dd Zdd ZdS )OpenFileaF  
    File-like object to be used in a context

    Can layer (buffered) text-mode and compression over any file-system, which
    are typically binary-only.

    These instances are safe to serialize, as the low-level file object
    is not created until invoked using ``with``.

    Parameters
    ----------
    fs: FileSystem
        The file system to use for opening the file. Should be a subclass or duck-type
        with ``fsspec.spec.AbstractFileSystem``
    path: str
        Location to open
    mode: str like 'rb', optional
        Mode of the opened file
    compression: str or None, optional
        Compression to apply
    encoding: str or None, optional
        The encoding to use if opened in text mode.
    errors: str or None, optional
        How to handle encoding errors if opened in text mode.
    newline: None or str
        Passed to TextIOWrapper in text mode, how to handle line endings.
    autoopen: bool
        If True, calls open() immediately. Mostly used by pickle
    pos: int
        If given and autoopen is True, seek to this location immediately
    rbNc                 C   s:   || _ || _|| _t||| _|| _|| _|| _g | _d S N)	fspathmodeget_compressioncompressionencodingerrorsnewlinefobjects)selfr   r   r   r   r   r   r    r   J/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/fsspec/core.py__init__=   s    
zOpenFile.__init__c                 C   s$   t | j| j| j| j| j| j| jffS r   )r   r   r   r   r   r   r   r   r   r   r   r   
__reduce__P   s    zOpenFile.__reduce__c                 C   s   d | jS )Nz<OpenFile '{}'>)formatr   r!   r   r   r   __repr__^   s    zOpenFile.__repr__c                 C   s   | j ddddd }| jj| j|d}|g| _| jd k	rdt| j }|||d d}| j| d| j krt	|| j
| j| jd}| j| | jd S )Nt b)r   r   )r   r   r   )r   replacer   openr   r   r   r
   appendPickleableTextIOWrapperr   r   r   )r   r   fcompressr   r   r   	__enter__a   s     


   zOpenFile.__enter__c                 G   s   |    d S r   )close)r   argsr   r   r   __exit__v   s    zOpenFile.__exit__c                 C   s   t | j| jS r   )r   r   r   r!   r   r   r   	full_namey   s    zOpenFile.full_namec                 C   s   |   S )a  Materialise this as a real open file without context

        The OpenFile object should be explicitly closed to avoid enclosed file
        instances persisting. You must, therefore, keep a reference to the OpenFile
        during the life of the file-like it generates.
        r/   r!   r   r   r   r*   }   s    zOpenFile.openc                 C   s>   t | jD ]$}d| jkr&|js&|  |  q
| j  dS )z#Close all encapsulated file objectsrN)reversedr   r   closedflushr0   clear)r   r-   r   r   r   r0      s
    
zOpenFile.close)r   NNNN)__name__
__module____qualname____doc__r    r"   r$   r/   r2   propertyr3   r*   r0   r   r   r   r   r      s   $     

	r   c                       sL   e Zd ZdZddd fdd
Zdd Zd	d
 Z fddZdd Z  Z	S )	OpenFilesa  List of OpenFile instances

    Can be used in a single context, which opens and closes all of the
    contained files. Normal list access to get the elements works as
    normal.

    A special case is made for caching filesystems - the files will
    be down/uploaded together at the start or end of the context, and
    this may happen concurrently, if the target filesystem supports it.
    r   Nr   r   c                   s"   || _ || _g | _t j|  d S r   )r   r   filessuperr    )r   r   r   r1   	__class__r   r   r       s    zOpenFiles.__init__c                 C   sb   | j d krtd| j }t|dr4|| | _| jS t|drT|j d k	rT|j }qqTqdd | D S )NzContext has already been used	open_manyr   c                 S   s   g | ]}|  qS r   r4   .0sr   r   r   
<listcomp>   s     z'OpenFiles.__enter__.<locals>.<listcomp>)r   
ValueErrorhasattrrE   rA   )r   r   r   r   r   r/      s    

zOpenFiles.__enter__c                    s`   | j } fdd| D  d| jkr\t|dr<|| j d S t|dr\|j d k	r\|j }q"q\q"d S )Nc                    s   g | ]}|j   qS r   )r2   rF   r1   r   r   rI      s     z&OpenFiles.__exit__.<locals>.<listcomp>r5   rE   r   )r   r   rK   Zcommit_manyrA   )r   r1   r   r   rL   r   r2      s    

zOpenFiles.__exit__c                    s,   t  |}t|tr(t|| j| jdS |S )Nr@   )rB   __getitem__
isinstanceslicer?   r   r   )r   itemoutrC   r   r   rM      s    
zOpenFiles.__getitem__c                 C   s   dt |  S )Nz<List of %s OpenFile instances>)lenr!   r   r   r   r$      s    zOpenFiles.__repr__)
r:   r;   r<   r=   r    r/   r2   rM   r$   __classcell__r   r   rC   r   r?      s   r?   r   utf8Tc              	      s   t | |||||
d\}}jdkr.|	_n0dkr^|	r^fdd|D }fdd|D  t fdd|D d	S )
av
  Given a path or paths, return a list of ``OpenFile`` objects.

    For writing, a str path must contain the "*" character, which will be filled
    in by increasing numbers, e.g., "part*" ->  "part1", "part2" if num=2.

    For either reading or writing, can instead provide explicit list of paths.

    Parameters
    ----------
    urlpath: string or list
        Absolute or relative filepath(s). Prefix with a protocol like ``s3://``
        to read from alternative filesystems. To read from multiple files you
        can pass a globstring or a list of paths, with the caveat that they
        must all have the same protocol.
    mode: 'rb', 'wt', etc.
    compression: string or None
        If given, open file using compression codec. Can either be a compression
        name (a key in ``fsspec.compression.compr``) or "infer" to guess the
        compression from the filename suffix.
    encoding: str
        For text mode only
    errors: None or str
        Passed to TextIOWrapper in text mode
    name_function: function or None
        if opening a set of files for writing, those files do not yet exist,
        so we need to generate their names by formatting the urlpath for
        each sequence number
    num: int [1]
        if writing mode, number of files we expect to create (passed to
        name+function)
    protocol: str or None
        If given, overrides the protocol found in the URL.
    newline: bytes or None
        Used for line terminator in text mode. If None, uses system default;
        if blank, uses no translation.
    auto_mkdir: bool (True)
        If in write mode, this will ensure the target directory exists before
        writing, by calling ``fs.mkdirs(exist_ok=True)``.
    expand: bool
    **kwargs: dict
        Extra options that make sense to a particular storage connection, e.g.
        host, port, username, password, etc.

    Examples
    --------
    >>> files = open_files('2015-*-*.csv')  # doctest: +SKIP
    >>> files = open_files(
    ...     's3://bucket/2015-*-*.csv.gz', compression='gzip'
    ... )  # doctest: +SKIP

    Returns
    -------
    An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can
    be used as a single context

    Notes
    -----
    For a full list of the available protocols and the implementations that
    they map across to see the latest online documentation:

    - For implementations built into ``fsspec`` see
      https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations
    - For implementations in separate packages see
      https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations
    )numname_functionstorage_optionsprotocolexpandfiler5   c                    s   h | ]}  |qS r   )_parentrG   r   r   r   r   	<setcomp>$  s     zopen_files.<locals>.<setcomp>c                    s   g | ]} j |d dqS )T)exist_ok)makedirs)rG   parentr]   r   r   rI   %  s     zopen_files.<locals>.<listcomp>c                    s"   g | ]}t | d qS ))r   r   r   r   r   )r   r\   r   r   r   r   r   r   r   r   rI   '  s   
r@   )get_fs_token_pathsrX   
auto_mkdirr?   )urlpathr   r   r   r   rV   rU   rX   r   rd   rY   kwargsZfs_tokenpathsparentsr   rb   r   
open_files   s*    O	

ri   c                    s   t d d| kr* fdd| dD n| g}g }d }| }t|D ]}|dd pft|d pfd}t|}||}||i }	||d kr|		| t
f ||	}
||}|dkrd	|
kr|}||||
f |}qHtt|}|S )
Nz.*[^a-z]+.*z::c                    s*   g | ]"}d |ks  |r|n|d  qS )://)matchrG   pxr   r   rI   ;  s     z_un_chain.<locals>.<listcomp>rX   r   rZ   >   	filecachesimplecache
blockcachetarget_protocol)recompilesplitcopyr6   popsplit_protocolr   Z_get_kwargs_from_urlsupdatedict_strip_protocolr+   list)r   rf   bitsrQ   Zprevious_bitbitrX   clsextra_kwargskwskwr   rn   r   	_un_chain8  s4    



r   c                    s   ddddddddh  fd	d
|  D }t| |}i }tt|D ]T\}}|\}}}|t|d krvtf ||}qDtf |||d< ||d< ||d< qD|d \}	}}
t|f|}||	fS )a  
    Turn fully-qualified and potentially chained URL into filesystem instance

    Parameters
    ----------
    url : str
        The fsspec-compatible URL
    **kwargs: dict
        Extra options that make sense to a particular storage connection, e.g.
        host, port, username, password, etc.

    Returns
    -------
    filesystem : FileSystem
        The new filesystem discovered from ``url`` and created with
        ``**kwargs``.
    urlpath : str
        The file-systems-specific URL for ``url``.
    r   r   r   rY   r   rV   r   rU   c                    s   i | ]\}}| kr||qS r   r   )rG   kvZknown_kwargsr   r   
<dictcomp>w  s       zurl_to_fs.<locals>.<dictcomp>r   target_optionsrs   for   )itemsr   	enumerater6   rR   r{   r   )urlrf   chaininkwargsichurlsrX   r   re   _r   r   r   r   	url_to_fsW  s.    



r   c           	      K   s6   t f | g||||||dd|}|s.t| |d S )a  Given a path or paths, return one ``OpenFile`` object.

    Parameters
    ----------
    urlpath: string or list
        Absolute or relative filepath. Prefix with a protocol like ``s3://``
        to read from alternative filesystems. Should not include glob
        character(s).
    mode: 'rb', 'wt', etc.
    compression: string or None
        If given, open file using compression codec. Can either be a compression
        name (a key in ``fsspec.compression.compr``) or "infer" to guess the
        compression from the filename suffix.
    encoding: str
        For text mode only
    errors: None or str
        Passed to TextIOWrapper in text mode
    protocol: str or None
        If given, overrides the protocol found in the URL.
    newline: bytes or None
        Used for line terminator in text mode. If None, uses system default;
        if blank, uses no translation.
    **kwargs: dict
        Extra options that make sense to a particular storage connection, e.g.
        host, port, username, password, etc.

    Examples
    --------
    >>> openfile = open('2015-01-01.csv')  # doctest: +SKIP
    >>> openfile = open(
    ...     's3://bucket/2015-01-01.csv.gz', compression='gzip'
    ... )  # doctest: +SKIP
    >>> with openfile as f:
    ...     df = pd.read_csv(f)  # doctest: +SKIP
    ...

    Returns
    -------
    ``OpenFile`` object.

    Notes
    -----
    For a full list of the available protocols and the implementations that
    they map across to see the latest online documentation:

    - For implementations built into ``fsspec`` see
      https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations
    - For implementations in separate packages see
      https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations
    F)re   r   r   r   r   rX   r   rY   r   )ri   FileNotFoundError)	re   r   r   r   r   rX   r   rf   rQ   r   r   r   r*     s    <	r*   c              	   K   sz   d|krt dt| fd|i|}t|d jdds>t d|}dd	 |D }W 5 Q R X t| trvt| sv|d S |S )
aK  Open file(s) which can be resolved to local

    For files which either are local, or get downloaded upon open
    (e.g., by file caching)

    Parameters
    ----------
    url: str or list(str)
    mode: str
        Must be read mode
    storage_options:
        passed on to FS for or used by open_files (e.g., compression)
    r5   z(Can only ensure local files when readingr   r   
local_fileFzOopen_local can only be used on a filesystem which has attribute local_file=Truec                 S   s   g | ]
}|j qS r   )namerG   r-   r   r   r   rI     s     zopen_local.<locals>.<listcomp>)rJ   ri   getattrr   rN   strr   )r   r   rW   ZofrA   rg   r   r   r   
open_local  s    r   c                 C   s0   |dkrt | }|d k	r,|tkr,td| |S )NZinferz!Compression type %s not supported)r   r
   rJ   )re   r   r   r   r   r     s
    r   c                 C   s<   t | } d| kr4| dd\}}t|dkr4||fS d| fS )zReturn protocol, path pairrj   r   N)r   rv   rR   )re   rX   r   r   r   r   ry     s    ry   c                 C   s   t | \}}t|}|| S )zCReturn only path part of full URL, according to appropriate backend)ry   r   r|   )re   rX   r   r   r   r   r   strip_protocol  s    r   c                 C   s   g }t | } d|krtdd | D dkr2tdt|t| }| D ]*}d|krd|t||| qD|| qDt||kr|d| }n.| D ](}t|r||	| q|| q|S )a(  Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]``
    in them (read mode).

    :param paths: list of paths
    mode: str
        Mode in which to open files.
    num: int
        If opening in writing mode, number of files we expect to create.
    fs: filesystem object
    name_function: callable
        If opening in writing mode, this callable is used to generate path
        names. Names are generated for each partition by
        ``urlpath.replace('*', name_function(partition_index))``.
    :return: list of paths
    wc                 S   s   g | ]}d |krdqS )*r   r   rl   r   r   r   rI     s      z*expand_paths_if_needed.<locals>.<listcomp>r   z;When writing data, only one filename mask can be specified.r   N)
r}   sumrJ   maxrR   extend_expand_pathsr+   r   glob)rg   r   rU   r   rV   Zexpanded_pathsZ	curr_pathr   r   r   expand_paths_if_needed  s&    r   c                    s  t | tttfr.| stdtt| d }nt| }p<i |rJ|d< t|pTi }i }	tt|D ]T\}
}|\}}}|
t	|d krt
f ||	}	qht
f ||	|	d< ||	d< ||	d< qh|d \}}}t|f|	 t | tttfr.fdd	| D }t	d
d |D dkrtd| dd	 |D }n
 |}t |tttfr\t||| |}njd|krz|rzt|||}nLd|kr|rt|||}n.d|kr fdd	t |D }n|g}  j|fS )a  Filesystem, deterministic token, and paths from a urlpath and options.

    Parameters
    ----------
    urlpath: string or iterable
        Absolute or relative filepath, URL (may include protocols like
        ``s3://``), or globstring pointing to data.
    mode: str, optional
        Mode in which to open files.
    num: int, optional
        If opening in writing mode, number of files we expect to create.
    name_function: callable, optional
        If opening in writing mode, this callable is used to generate path
        names. Names are generated for each partition by
        ``urlpath.replace('*', name_function(partition_index))``.
    storage_options: dict, optional
        Additional keywords to pass to the filesystem class.
    protocol: str or None
        To override the protocol specifier in the URL
    expand: bool
        Expand string paths for writing, assuming the path is a directory
    zempty urlpath sequencer   rX   r   r   rs   r   c                    s"   g | ]}t t| pi d  qS r   )r   r   )rG   u)rW   r   r   rI   p  s    z&get_fs_token_paths.<locals>.<listcomp>c                 S   s   h | ]}|d  qS )r   r   rG   Zpcr   r   r   r^   s  s     z%get_fs_token_paths.<locals>.<setcomp>z$Protocol mismatch getting fs from %sc                 S   s   g | ]}|d  qS r   r   r   r   r   r   rI   u  s     r   ro   r   c                    s   g | ]}  |s|qS r   )isdirr   r]   r   r   rI     s     
 )rN   r}   tuplesetrJ   r   r   r   r6   rR   r{   r   r|   r   r   sortedr   Z	_fs_token)re   r   rU   rV   rW   rX   rY   Zurlpath0r   r   r   r   r   Znested_protocolr   rg   r   Zpchainsr   )r   rW   r   rc   :  sL    





rc   c                    s   t tr|ddkr"tdndkr8tjd d krLt|d   fddt|D }|t	|krt
d n0t ttfrt|kstt}ntd|S )	Nr   r   z.Output path spec must contain exactly one '*'.z*.partc                    s   g | ]} d  |qS )r   )r)   )rG   r   rV   r   r   r   rI     s     z!_expand_paths.<locals>.<listcomp>zqIn order to preserve order between partitions paths created with ``name_function`` should sort to partition orderzPath should be either
1. A list of paths: ['foo.json', 'bar.json', ...]
2. A directory: 'foo/
3. A path with a '*' in it: 'foo.*.json')rN   r   countrJ   osr   joinr   ranger   loggerwarningr   r}   rR   AssertionError)r   rV   rU   rg   r   r   r   r     s&    


r   c                       s*   e Zd ZdZd fdd	Zdd Z  ZS )	r,   zTextIOWrapper cannot be pickled. This solves it.

    Requires that ``buffer`` be pickleable, which all instances of
    AbstractBufferedFile are.
    NFc                    s$   ||||||f| _ t j| j   d S r   )r1   rB   r    )r   bufferr   r   r   line_bufferingwrite_throughrC   r   r   r      s    	z PickleableTextIOWrapper.__init__c                 C   s
   t | jfS r   )r,   r1   r!   r   r   r   r"     s    z"PickleableTextIOWrapper.__reduce__)NNNFF)r:   r;   r<   r=   r    r"   rS   r   r   rC   r   r,     s   	     r,   )
r   NrT   NNr   NNTT)r   NrT   NNN)r   )r   r   NNNT))iologgingr   rt   r   r   Zcachingr   r   r   r   r   r	   r   r
   registryr   r   utilsr   r   r   r   	getLoggerr   r   r}   r?   ri   r   r   r*   r   r   ry   r   r   rc   r   TextIOWrapperr,   r   r   r   r   <module>   sZ    
s<          
o3      
L
1      
M