U -eFw@s$ddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z ddlmZmZddlmZddlmZddlmZmZdd lmZdd lmZdd lmZed ZGd ddeZ Gddde Z!Gddde!Z"GdddZ#ddZ$ej%dddZ&dS)) annotationsN)rmtree)ClassVar)AbstractFileSystem filesystem)_DEFAULT_CALLBACK)compr) BaseCache MMapCache)BlocksizeMismatchError)AbstractBufferedFile)infer_compressionz fsspec.cachedc seZdZUdZdZded<d,fd d Zd d ZddZddZ ddZ ddZ ddZ d-ddZ ddZd.ddZd d!Zd"d#Zfd$d%Zd&d'Zd(d)Zd*d+ZZS)/CachingFileSystemaSLocally caching filesystem, layer over any other FS This class implements chunk-wise local storage of remote files, for quick access after the initial download. The files are stored in a given directory with hashes of URLs for the filenames. If no directory is given, a temporary one is used, which should be cleaned up by the OS after the process ends. The files themselves are sparse (as implemented in :class:`~fsspec.caching.MMapCache`), so only the data which is accessed takes up space. Restrictions: - the block-size must be the same for each access of a given file, unless all blocks of the file have already been read - caching can only be applied to file-systems which produce files derived from fsspec.spec.AbstractBufferedFile ; LocalFileSystem is also allowed, for testing )Z blockcachecachedzClassVar[str | tuple[str, ...]]protocolNTMP F: c  stjf| |dkr&|dkr&td|dk|dkAs>td|dkrRtg} nt|trd|g} n|} tj| ddd| _ |pi_ |_ |_ |_ | _|_t|tr|nt|jtr|jn|jd_|dk r|n t|fj _fd d } | _dS) a Parameters ---------- target_protocol: str (optional) Target filesystem protocol. Provide either this or ``fs``. cache_storage: str or list(str) Location to store files. If "TMP", this is a temporary directory, and will be cleaned up by the OS when this process ends (or later). If a list, each location will be tried in the order given, but only the last will be considered writable. cache_check: int Number of seconds between reload of cache metadata check_files: bool Whether to explicitly see if the UID of the remote file matches the stored one before using. Warning: some file systems such as HTTP cannot reliably give a unique hash of the contents of some path, so be sure to set this option to False. expiry_time: int The time in seconds after which a local copy is considered useless. Set to falsy to prevent expiry. The default is equivalent to one week. target_options: dict or None Passed to the instantiation of the FS, if fs is None. fs: filesystem instance The target filesystem to run against. Provide this or ``protocol``. same_names: bool (optional) By default, target URLs are hashed, so that files from different backends with the same basename do not conflict. If this is true, the original basename is used. compression: str (optional) To decompress on download. Can be 'infer' (guess from the URL name), one of the entries in ``fsspec.compression.compr``, or None for no decompression. Nz9Please provide filesystem instance(fs) or target_protocolz@Both filesystems (fs) and target_protocol may not be both given.rTexist_okrcsjt|SN)fs_strip_protocoltypepathself^/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/fsspec/implementations/cached.pyrsz3CachingFileSystem.__init__.._strip_protocol)super__init__ ValueErrortempfilemkdtemp isinstancestrosmakedirsstoragekwargs cache_check check_filesexpiry compression same_namesrtarget_protocol load_cacherrr) rr1Z cache_storager,r- expiry_timeZtarget_optionsrr0r/r+r*r __class__rr r"/s<0    zCachingFileSystem.__init__cCstj|jddddS)NrTr)r(r)r*rrrr _mkcacheszCachingFileSystem._mkcachec Csg}|jD]}tj|d}tj|rt|dF}t|}|D]"}t |dt rFt |d|d<qF| |W5QRXq | iq | |pig|_t|_dS)z#Read set of stored blocks from filecacherbblocksN)r*r(rjoinexistsopenpickleloadvaluesr&listsetappendr6 cached_filestime last_cache)rrCr*fnfZloaded_cached_filescrrr r2s       zCachingFileSystem.load_cachec Cstj|jdd}|jd}tj|rt|d}t|}W5QRX| D]\}}||krR|ddks~||ddkrd|d<n"||d}| |d||d<t |d||d|d<||d|d<qR| D]\}}||kr|||<qn|}dd | D}| D]&}t |dtrt|d|d<q|t|}t||W5QRX||jd<t|_d S) z#Save set of stored blocks from filerr7r8r9TrDuidcSsi|]\}}||qSr)copy).0kvrrr sz0CachingFileSystem.save_cache..N)r(rr:r*rCr;r<r=r>itemsupdatemaxr?r&rAr@r6 atomic_writedumprDrE)rrFr7rGrCrLrHr9rrr save_caches6         zCachingFileSystem.save_cachecCsN||jsdSt|j|jk}tdd|jD}|sB|sJ|dS)z0Reload caches if time elapsed or any disappearedNcss|]}tj|VqdSr)r(rr;)rKr*rrr sz1CachingFileSystem._check_cache..)r6r,rDrEallr*r2)rZtimecondZ existcondrrr _check_cacheszCachingFileSystem._check_cachecCs||}|t|j|jD]\}}||kr2q ||}|jrZ|d|j|krZq |j rxt |d|j krxq t j ||d}t j |r ||fSq dS)z Is path in cache and still validrIrDrFF)rrWzipr*rCrJr-rukeyr.rDr(rr:r;)rrr*r7detailrFrrr _check_files    zCachingFileSystem._check_filecCst|jd|dS)zRemove all files and metadat from the cache In the case of multiple cache locations, this clears only the last one, which is assumed to be the read/write one. rN)rr*r2rrrr clear_cacheszCachingFileSystem.clear_cachec Cs |s |j}||jdD]\}}t|d|kr$|jrltj |d}tj |j d|}ntj |j d|d}tj |r$t ||jd|q$|jdrtj |j dd}t|}t|jd|W5QRXnt|j d|dS)aRemove all expired files and metadata from the cache In the case of multiple cache locations, this clears only the last one, which is assumed to be the read/write one. Parameters ---------- expiry_time: int The time in seconds after which a local copy is considered useless. If not defined the default is equivalent to the attribute from the file caching instantiation. rrDoriginalrFr7N)r.rWrCrJrOrDr0r(rbasenamer:r*r;removepoprRr=rSrr2)rr3rrZr^rF cache_pathfcrrr clear_expired_caches$    z%CachingFileSystem.clear_expired_cachecCsd||}||}|sdS|\}}||jdrXt||jd||nt ddS)zRemove cached version of given file Deletes local copy of the given (remote) path. If it is found in a cache location which is not the last, it is assumed to be read-only, and raises PermissionError Nrzz)CachingFileSystem._open..)rr_openr[loggerdebugr< hash_namer0r(rr:r*rArDrYrCr6r/r rrsr r Z _fetch_rangesizer7rvrT) rrrjrkrlrmr+rZrFhashr9comprrur ry)s~          zCachingFileSystem._opencCs t||dS)N) same_name)r|)rrrrrr r|szCachingFileSystem.hash_namecCs|jr dS||j}|jd|}|ddk rPt|d|j|jkrPd|d<z td| tdWn:t k rtdYnt k rtdYnX|d|_dS) z9Called when a file is closing, so store the set of blocksNrr9Tz going to saveZsavedz&Cache saving failed while closing filez-Cache save failed due to interpreter shutdown) closedrrrClenrsr}rzr{rTOSError NameError)rrGrvrrHrrr rts  $ z"CachingFileSystem.close_and_updatecsdkrfddSdkr"tdkr8ttSdkrHtStd}|dd}|krp|S|dk rֈ|jkr|jSt|}t|}t|st |rt |d r|j dkr| ||S|St SdS) N)r2ryrTrtr"__getattribute__ __reduce___make_local_detailsr<catZcat_filegetZ read_blocktailheadr[rWr6r\rcrhr6 local_fileZ_paths_from_pathZ get_mapper open_many commit_manyr|__hash____eq__to_jsoncstt||Sr)getattrr__get__)argskwitemrrr rwsz4CachingFileSystem.__getattribute__..) __reduce_ex__)_cacher5__dict__r__self__)AttributeErrorrrobjectrrrinspect isfunctionisdatadescriptorhasattrrrr!)rrdrclsmr4rr rs2#      z"CachingFileSystem.__getattribute__cCs~||kr dSt|t|sdS|j|jko||j|jko||j|jko||j|jko||j|jko||j|jko||j|jko||j |j kS)zTest for equality.TF) r&rr*r+r,r-r.r/r0r1)rotherrrr rs&        zCachingFileSystem.__eq__cCsXtt|jtt|jAt|jAt|jAt|jAt|jAt|j At|j AS)zCalculate hash.) r~tupler*r'r+r,r-r.r/r0r1rrrr rs   zCachingFileSystem.__hash__cCs tddS)z[Calculate JSON representation. Not implemented yet for CachingFileSystem. z5CachingFileSystem JSON representation not implementedN)NotImplementedErrorrrrr rszCachingFileSystem.to_json) NrrFrNNFN)N)r8NTN)__name__ __module__ __qualname____doc__r__annotations__r"r6r2rTrWr[r\rcrhryr|rtrrrr __classcell__rrr4r rs>  X(  & [ C rc@sJeZdZdZdZdZddZddZdd Zd d e fd d Z dddZ dS)WholeFileCacheFileSystema Caches whole remote files on first access This class is intended as a layer over any other file system, and will make a local copy of each file accessed, so that all subsequent reads are local. This is similar to ``CachingFileSystem``, but without the block-wise functionality and so can work even when sparse files are not allowed. See its docstring for definition of the init arguments. The class still needs access to the remote store for listing files, and may refresh cached files. filecacheTcsddD}djkr"nfdd|DSjr@tfdd|D}ddt||D}fddt||D}ddt||D}|rވj||fd d|D}jd d d t||D d dfddt||DS)NcSsg|] }|jqSrr)rKZofrrr sz6WholeFileCacheFileSystem.open_many..ricsg|]}tj|jdqSrr) LocalTempFilerrjrKr) open_filesrrr rscsg|]}|qSr)r[)rKsprrr r"scSsg|]\}}|s|qSrrrKprrrr r#sc s.g|]&\}}tjjd|jqS)r)r(rr:r*r|r0rrrr r$scSsg|]\}}|s|qSrr)rKrFrrrr r(sc s2g|]*}||jdtj|dqS)Tro)r|r0rDrrYrrrr r.s  rcSsi|]\}}||qSrr)rKrrZrrr rN9sz6WholeFileCacheFileSystem.open_many..cSst|tr|dS|S)N)r&rrFrrr firstpart=sz5WholeFileCacheFileSystem.open_many..firstpartcs*g|]"\}}t|r|n|jdqSr)r<rj)rKZfn0Zfn1)rrrr rAs) rjr6r/rrXrrrCrPrT)rrpathsrfZdownpathZdownfn0ZdownfnZ newdetailr)rrrr rs6       z"WholeFileCacheFileSystem.open_manyc Csd|jdd|Ddd|Ddd|D|D]*}zt|jWq4tk r\Yq4Xq4dS)NcSsg|] }|jqSrrrKrGrrr rGsz8WholeFileCacheFileSystem.commit_many..cSsg|] }|jqSrrrrrr rGscSsg|] }|qSrrvrrrr rHs)rputr(r_nameFileNotFoundError)rrrGrrr rFs"z$WholeFileCacheFileSystem.commit_manycCs^|||j}tj|jd|}||dt|j|d}||j d|<t d||S)NrTroCopying %s to local cache) r|r0r(rr:r*rDrrYrCrzr{)rrr~rFrZrrr rPs z,WholeFileCacheFileSystem._make_local_detailsFraisec Ksr|j|||ddd}g}g}g} i} |D]} zV|| } | sd|| } || || nt| trr| nd| f\} } | | Wq0tk r}z(|dkr|dkr|| | <| | W5d}~XYq0Xq0|r|j ||| | t |t|| D]6\} } t| d}|| | <W5QRX|dqt|trnt |dkrn|dkrn| |d} | S) Nmaxdepth) recursiverrreturnr8rFr)Z expand_pathrrJr[rrBr&r Exceptionr_rrTZset_sizerrXr<readZrelative_updater')rrrZon_errorcallbackr+rZgetpathsZ storepathsfnsoutrrZrFerGrrr r^sF       $ zWholeFileCacheFileSystem.catr8c Ksr||}d|kr t|||dS||}|r|\}}|d|d}}|dkrxtd|t||}|d|_|Std|n | |}||d <| |j rP|j j |f|}t|d z} t|trtd |jj|j|_|j d krt|n|j } t| |d d}d} | r:t|dd} || } | | qW5QRXW5QRXn|j |||| ||S)NrirrrFr9Trnr]z?Attempt to open partially cached file %sas a wholly cached filerjwbrrqr8rsP)rrr[rzr{r<rr]r#rr6r/rryr&r r r7fetcherr}r rrrwriterT) rrrjr+rZrFrgr9rGf2rdatablockrrr rysJ         $zWholeFileCacheFileSystem._openN)r8) rrrrrrrrrrrryrrrr rs /  -rcsJeZdZdZdZdZfddZddZdd Zd d Z dd dZ Z S)SimpleCacheFileSystemaCaches whole remote files on first access This class is intended as a layer over any other file system, and will make a local copy of each file accessed, so that all subsequent reads are local. This implementation only copies whole files, and does not keep any metadata about the download time or file details. It is therefore safer to use in multi-threaded/concurrent situations. This is the only of the caching filesystems that supports write: you will be given a real local open file, and upon close and commit, it will be uploaded to the target filesystem; the writability or the target URL is not checked until that time. Z simplecacheTc sZ|}dD] }d||<q tjf||jD]}tj|s.tj|ddq.ig|_dS)N)r,r3r-FTr) rJr!r"r*r(rr;r)rC)rr+rkeyr*r4rr r"s   zSimpleCacheFileSystem.__init__cCsH||||j}|jD]&}tj||}tj|r|SqdSr)rWr|r0r*r(rr:r;)rrshar*rFrrr r[s   z!SimpleCacheFileSystem._check_filecCsdSrrrrrr rTsz SimpleCacheFileSystem.save_cachecCsdSrrrrrr r2sz SimpleCacheFileSystem.load_cacher8c Ks@||}d|kr t|||dS||}|r8t||S|||j}tj|j d|}t d|||d<| |j r&|jj|f|}t|dx}t|trtd|jj|j|_|j dkrt|n|j }t||d d}d } | rt|d d } || } || qW5QRXW5QRXn|j|||||S) Nrirrrrrjrrrqr8Trsr)rrr[r<r|r0r(rr:r*rzr{r6r/rryr&r r r7rr}r rrrrr) rrrjr+rFrrGrrrrrrr rys6       "zSimpleCacheFileSystem._open)r8) rrrrrrr"r[rTr2ryrrrr4r rs rc@s^eZdZdZdddZdd Zd d Zd d ZddZddZ ddZ e ddZ ddZ dS)rz8A temporary local file, which will be uploaded on commitNrTrcCsd|r||_t|||_nt\}|_t|||_||_|rH|j|||_||_d|_ ||_ dS)NF) rFr<fhr$mkstemprjseekrrrrl)rrrrFrjrlrfdrrr r"s  zLocalTempFile.__init__cCs t|j|j|jd|j|ffS)Nzrb+)rrrrFrltellrrrr r szLocalTempFile.__reduce__cCs|jSr)rrrrr __enter__'szLocalTempFile.__enter__cCs |dSrr)rexc_typeexc_valexc_tbrrr __exit__*szLocalTempFile.__exit__cCs,|jr dS|jd|_|jr(|dS)NT)rrrvrlcommitrrrr rv-s  zLocalTempFile.closecCs|jt|jdSr)rrvr(r_rFrrrr discard5s zLocalTempFile.discardc Cs@|j|j|jzt|jWnttfk r:YnXdSr)rrrFrr(r_rerrrrr r9s zLocalTempFile.commitcCs|jSrrrrrr rAszLocalTempFile.namecCs t|j|Sr)rr)rrrrr __getattr__EszLocalTempFile.__getattr__)NrTr)rrrrr"rrrrvrrpropertyrrrrrr rs  rcCs(|rtj|}nt|}|Sr)r(rr^hashlibsha256encode hexdigest)rrr~rrr r|Isr|rc cstjtj|tj|dd\}}z t|| }|VW5QRXWn6tk r|t t t |W5QRXYnXt ||dS)z A context manager that opens a temporary file next to `path` and, on exit, replaces `path` with the temporary file, thereby updating `path` atomically. -)dirprefixN) r$rr(rdirnamer^r< BaseException contextlibsuppressrunlinkreplace)rrjrrFfprrr rRQs    rR)r)' __future__rrrrloggingr(r=r$rDshutilrtypingrZfsspecrrZfsspec.callbacksrZfsspec.compressionrZ fsspec.corer r Zfsspec.exceptionsr Z fsspec.specr Z fsspec.utilsr getLoggerrzrrrrr|contextmanagerrRrrrr s:         p:O;