U -eJT @s2ddlmZmZmZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZddlmZddlmZmZddlmZmZmZmZed ZGd d d ZGd d d eZd)ddZ ddZ!ddZ"d*ddZ#d+ddZ$ddZ%ddZ&dd Z'd!d"Z(d,d#d$Z)d%d&Z*Gd'd(d(ej+Z,dS)-)absolute_importdivisionprint_functionN) has_magic) BaseCache BlockCache BytesCache MMapCacheReadAheadCachecaches)compr) filesystemget_filesystem_class)_unstrip_protocolbuild_name_functioninfer_compressionstringify_pathZfsspecc@sVeZdZdZdddZddZdd Zd d Zd d Ze ddZ ddZ ddZ dS)OpenFileaF File-like object to be used in a context Can layer (buffered) text-mode and compression over any file-system, which are typically binary-only. These instances are safe to serialize, as the low-level file object is not created until invoked using ``with``. Parameters ---------- fs: FileSystem The file system to use for opening the file. Should be a subclass or duck-type with ``fsspec.spec.AbstractFileSystem`` path: str Location to open mode: str like 'rb', optional Mode of the opened file compression: str or None, optional Compression to apply encoding: str or None, optional The encoding to use if opened in text mode. errors: str or None, optional How to handle encoding errors if opened in text mode. newline: None or str Passed to TextIOWrapper in text mode, how to handle line endings. autoopen: bool If True, calls open() immediately. Mostly used by pickle pos: int If given and autoopen is True, seek to this location immediately rbNcCs:||_||_||_t|||_||_||_||_g|_dSN) fspathmodeget_compression compressionencodingerrorsnewlinefobjects)selfrrrrrrrr!L/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/fsspec/core.py__init__?s  zOpenFile.__init__cCs$t|j|j|j|j|j|j|jffSr)rrrrrrrrr r!r!r" __reduce__RszOpenFile.__reduce__cCs d|jS)Nz)formatrr$r!r!r"__repr__`szOpenFile.__repr__cCs|jddddd}|jj|j|d}|g|_|jdk rdt|j}|||dd}|j|d|jkrt ||j |j |j d}|j||jdS)Ntb)rr)rrr) rreplaceropenrrrr appendPickleableTextIOWrapperrrr)r rfcompressr!r!r" __enter__cs      zOpenFile.__enter__cGs |dSr)close)r argsr!r!r"__exit__xszOpenFile.__exit__cCst|j|jSr)rrrr$r!r!r" full_name{szOpenFile.full_namecCs|S)aMaterialise this as a real open file without context The OpenFile object should be explicitly closed to avoid enclosed file instances persisting. You must, therefore, keep a reference to the OpenFile during the life of the file-like it generates. r2r$r!r!r"r-sz OpenFile.opencCs>t|jD]$}d|jkr&|js&||q |jdS)z#Close all encapsulated file objectsrN)reversedrrclosedflushr3clear)r r0r!r!r"r3s  zOpenFile.close)rNNNN) __name__ __module__ __qualname____doc__r#r%r'r2r5propertyr6r-r3r!r!r!r"rs$   rcsLeZdZdZdddfdd ZddZd d Zfd d Zd dZZ S) OpenFilesaList of OpenFile instances Can be used in a single context, which opens and closes all of the contained files. Normal list access to get the elements works as normal. A special case is made for caching filesystems - the files will be down/uploaded together at the start or end of the context, and this may happen concurrently, if the target filesystem supports it. rNrrcs"||_||_g|_tj|dSr)rrfilessuperr#)r rrr4 __class__r!r"r#szOpenFiles.__init__cCsb|jdkrtd|j}t|dr4|||_|jSt|drT|jdk rT|j}qqTqdd|DS)NzContext has already been used open_manyrcSsg|] }|qSr!r7.0sr!r!r" sz'OpenFiles.__enter__..)r ValueErrorhasattrrHrD)r rr!r!r"r2s   zOpenFiles.__enter__cs`|j}fdd|Dd|jkr\t|dr<||jdSt|dr\|jdk r\|j}q"q\q"dS)Ncsg|]}|jqSr!)r5rIr4r!r"rLsz&OpenFiles.__exit__..r8rHr)rrrNZ commit_manyrD)r r4rr!rOr"r5s   zOpenFiles.__exit__cs,t|}t|tr(t||j|jdS|S)NrC)rE __getitem__ isinstanceslicerBrr)r itemoutrFr!r"rPs  zOpenFiles.__getitem__cCs dt|S)Nz)lenr$r!r!r"r'szOpenFiles.__repr__) r=r>r?r@r#r2r5rPr' __classcell__r!r!rFr"rBs   rBrutf8Tc  srt|||| || d\} } dkrL| rLfdd| D}fdd|Dtfdd| DdS) av Given a path or paths, return a list of ``OpenFile`` objects. For writing, a str path must contain the "*" character, which will be filled in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. For either reading or writing, can instead provide explicit list of paths. Parameters ---------- urlpath: string or list Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol. mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode name_function: function or None if opening a set of files for writing, those files do not yet exist, so we need to generate their names by formatting the urlpath for each sequence number num: int [1] if writing mode, number of files we expect to create (passed to name+function) protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. auto_mkdir: bool (True) If in write mode, this will ensure the target directory exists before writing, by calling ``fs.mkdirs(exist_ok=True)``. expand: bool **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> files = open_files('2015-*-*.csv') # doctest: +SKIP >>> files = open_files( ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' ... ) # doctest: +SKIP Returns ------- An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can be used as a single context Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations )num name_functionstorage_optionsprotocolexpandr8csh|]}|qSr!)_parentrJrrr!r" $szopen_files..csg|]}j|ddqS)T)exist_ok)makedirs)rJparentr_r!r"rL%szopen_files..c s"g|]}t|dqS))rrrrr)rr^rrrrrrr!r"rL's rC)get_fs_token_pathsrB)urlpathrrrrrYrXr[rZ auto_mkdirr\kwargsZfs_tokenpathsparentsr!rdr" open_filess&O   rjc stdd|kr*fdd|dDn|g}g}d}|}t|D]}|ddpft|dpfd}t|}||}||i} ||dkr| |t f|| } | |}|dkrd | kr|}| ||| f|}qHt t|}|S) Nz .*[^a-z]+.*z::cs*g|]"}d|ks|r|n|dqS)://)matchrJpxr!r"rL;sz_un_chain..r[rfile> filecache blockcache simplecachetarget_protocol)recompilesplitcopyr9popsplit_protocolrZ_get_kwargs_from_urlsupdatedict_strip_protocolr.list) rrgbitsrTZ previous_bitbitr[cls extra_kwargskwskwr!ror" _un_chain8s4        rc Kst||}i}tt|D]T\}}|\}}}|t|dkrLtf||}qtf|||d<||d<||d<q|d\} }} t|f|} | | fS)a Turn fully-qualified and potentially chained URL into filesystem instance Parameters ---------- url : str The fsspec-compatible URL **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Returns ------- filesystem : FileSystem The new filesystem discovered from ``url`` and created with ``**kwargs``. urlpath : str The file-systems-specific URL for ``url``. rtarget_optionsrufor)r enumerater9rUr}r) urlrgchaininkwargsichurlsr[rrf_rr!r!r" url_to_fsWs    rc Ks6tf|g||||||dd|}|s.t||dS)aGiven a path or paths, return one ``OpenFile`` object. Parameters ---------- urlpath: string or list Absolute or relative filepath. Prefix with a protocol like ``s3://`` to read from alternative filesystems. Should not include glob character(s). mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> openfile = open('2015-01-01.csv') # doctest: +SKIP >>> openfile = open( ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' ... ) # doctest: +SKIP >>> with openfile as f: ... df = pd.read_csv(f) # doctest: +SKIP ... Returns ------- ``OpenFile`` object. Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations F)rfrrrrr[rr\r)rjFileNotFoundError) rfrrrrr[rrgrTr!r!r"r-{s<  r-c Kszd|krtdt|fd|i|}t|djdds>td|}dd |D}W5QRXt|trvt|sv|dS|S) aKOpen file(s) which can be resolved to local For files which either are local, or get downloaded upon open (e.g., by file caching) Parameters ---------- url: str or list(str) mode: str Must be read mode storage_options: passed on to FS for or used by open_files (e.g., compression) r8z(Can only ensure local files when readingrr local_fileFzOopen_local can only be used on a filesystem which has attribute local_file=TruecSsg|] }|jqSr!)namerJr0r!r!r"rLszopen_local..)rMrjgetattrrrQstrr)rrrZZofrDrhr!r!r" open_localsrcCs0|dkrt|}|dk r,|tkr,td||S)NZinferz!Compression type %s not supported)rr rM)rfrr!r!r"rs  rcCs<t|}d|kr4|dd\}}t|dkr4||fSd|fS)zReturn protocol, path pairrkrN)rrxrU)rfr[rr!r!r"r{s  r{cCst|\}}t|}||S)zCReturn only path part of full URL, according to appropriate backend)r{rr~)rfr[rrr!r!r"strip_protocols rcCsg}t|}d|krtdd|Ddkr2tdt|t|}|D]*}d|krd|t|||qD||qDt||kr|d|}n.|D](}t|r|| |q||q|S)a(Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` in them (read mode). :param paths: list of paths mode: str Mode in which to open files. num: int If opening in writing mode, number of files we expect to create. fs: filesystem object name_function: callable If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. :return: list of paths wcSsg|]}d|krdqS)*rr!rmr!r!r"rLsz*expand_paths_if_needed..rz;When writing data, only one filename mask can be specified.rN) rsumrMmaxrUextend _expand_pathsr.rglob)rhrrXrrYZexpanded_pathsZ curr_pathr!r!r"expand_paths_if_neededs&   rcst|tttfr.|stdtt|d}nt|}p.css|]}|dVqdS)rNr!rJZpcr!r!r" fsz%get_fs_token_paths..z$Protocol mismatch getting fs from %scSsg|] }|dqSrr!rr!r!r"rLhsrrcsg|]}|s|qSr!)isdirrr_r!r"rLqs )rQrtuplesetrMrrrr9rUr}rr~rrsortedrZ _fs_token)rfrrXrYrZr[r\Zurlpath0rrrrrZnested_protocolrrhrZpchainsr!)rrZr"re-sH       recsttr|ddkr"tdndkr8tjddkrLt|dfddt|D}|t |krt dn0tt t frt|kstt }ntd|S) Nrrz.Output path spec must contain exactly one '*'.z*.partcsg|]}d|qS)r)r,)rJrrYrr!r"rLsz!_expand_paths..zqIn order to preserve order between partitions paths created with ``name_function`` should sort to partition orderzPath should be either 1. A list of paths: ['foo.json', 'bar.json', ...] 2. A directory: 'foo/ 3. A path with a '*' in it: 'foo.*.json')rQrcountrMosrjoinrrangerloggerwarningrrrUAssertionError)rrYrXrhr!rr"rxs&     rcs*eZdZdZdfdd ZddZZS) r/zTextIOWrapper cannot be pickled. This solves it. Requires that ``buffer`` be pickleable, which all instances of AbstractBufferedFile are. NFcs$||||||f|_tj|jdSr)r4rEr#)r bufferrrrline_buffering write_throughrFr!r"r#s z PickleableTextIOWrapper.__init__cCs t|jfSr)r/r4r$r!r!r"r%sz"PickleableTextIOWrapper.__reduce__)NNNFF)r=r>r?r@r#r%rVr!r!rFr"r/s  r/) rNrWNNrNNTT)rNrWNNN)r)rrNNNT)- __future__rrriologgingrrvrrZcachingrrr r r r rr registryrrutilsrrrr getLoggerrrrrBrjrrr-rrr{rrrer TextIOWrapperr/r!r!r!r"s\    s< m& L  1 K