U 9%ef @s&ddlZddlZddlZddlZddlZddlZddlZddlmZe dZ GdddZ Gddde Z Gdd d e Z Gd d d e ZGd d d e ZGddde ZGddde ZGddde ZGdddZGddde Zde iZdddZe e ee eeeeef D]ZeeqdS)N)ThreadPoolExecutorZfsspecc@s$eZdZdZdZddZddZdS) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is nonecCs||_||_||_dSN) blocksizefetchersizeselfrrrr M/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/fsspec/caching.py__init__szBaseCache.__init__cCs<|dkr d}|dkr|j}||jks,||kr0dS|||SNr)rr)r startstopr r r _fetch$szBaseCache._fetchN)__name__ __module__ __qualname____doc__namer rr r r r r srcsFeZdZdZdZdfdd ZddZdd Zd d Zd d Z Z S) MMapCachezmemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix mmapNcs8t||||dkrtn||_||_||_dSr)superr setblockslocation _makefilecache)r rrrrr __class__r r r 9szMMapCache.__init__cCsddl}ddl}|jdkr tS|jdks8tj|js|jdkrT|}t |_ n t |jd}| |jd| d|n t |jd}|||jS)Nrzwb+1zrb+)rtempfiler bytearrayrospathexists TemporaryFilerropenseekwriteflushfileno)r rr$fdr r r r?s       zMMapCache._makefilec std|d||dkr"d}|dkr0j}|jksB||krFdS|j}|j}fddt||dD}|r|d}|j}t|jj}td|d |d|||j||<j |qvj||S) NzMMap cache fetching -rrcsg|]}|jkr|qSr )r).0ir r r _s z$MMapCache._fetch..r"zMMap get block #z () loggerdebugrrrangepopminrrradd) r rendZ start_blockZ end_blockZneedr2Zsstartsendr r3r rUs$    zMMapCache._fetchcCs|j}|d=|S)Nr)__dict__copyr stater r r __getstate__ls zMMapCache.__getstate__cCs|j|||_dSr)r=updaterrr?r r r __setstate__rs zMMapCache.__setstate__)NN) rrrrrr rrrArC __classcell__r r r r r.srcs,eZdZdZdZfddZddZZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). Z readaheadcs&t|||d|_d|_d|_dS)Nrr)rr rrr;r r r r r szReadAheadCache.__init__cCs|dkr d}|dks||jkr$|j}||jks6||kr:dS||}||jkrp||jkrp|j||j||jS|j|kr|jkrnn(|j||jd}|t|8}|j}nd}t|j||j}||||_||_|jt|j|_||jd|Sr)rrr;rlenr9rr)r rr;lpartr r r rs& zReadAheadCache._fetchrrrrrr rrDr r r r rExs rEcs,eZdZdZdZfddZddZZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstcst|||d|_dSr)rr rr r r r r szFirstChunkCache.__init__cCs|pd}|p|j}||jkr|jdkrh||jkrX|d|}|d|j|_||dS|d|j|_|j||}||jkr|||j|7}|S|||SdSNr)rrrr)r rr;datarHr r r rs       zFirstChunkCache._fetchrIr r r r rJs rJcsbeZdZdZdZdfdd ZddZdd Zd d Zd d Z ddZ fddZ ddZ Z S) BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. Z blockcache cs<t|||t|||_||_t||j|_ dSr) rr mathceilnblocks maxblocks functools lru_cache _fetch_block_fetch_block_cachedr rrrrSr r r r szBlockCache.__init__cCsd|j|j|jS)Nz.formatrrrRr3r r r __repr__s zBlockCache.__repr__cCs |jSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. rW cache_infor3r r r r^s zBlockCache.cache_infocCs|j}|d=|S)NrWr=r?r r r rAszBlockCache.__getstate__cCs&|j|t|d|j|_dS)NrS)r=rBrTrUrVrWr?r r r rCs  zBlockCache.__setstate__cCst|dkr d}|dkr|j}||jks,||kr0dS||j}||j}t||dD]}||qR|j||||dS)Nrrr"start_block_numberend_block_number)rrr7rW _read_cache)r rr;rarb block_numberr r r rs    zBlockCache._fetchcsN||jkrtd||j||j}||j}td|t||}|S)= Fetch the block of data for `block_number`. ;'block_number={}' is greater than the number of blocks ({})zBlockCache fetching block %drR ValueErrorrZrr5inforr)r rdrr;block_contentsr r r rVs    zBlockCache._fetch_blockc Cs||j}||j}||kr2||}|||Sg}||||dt|d|D]} ||| q\|||d|d|SdSz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. Nr"rrrWappendr7join r rr;rarb start_posend_posblockoutrdr r r rc#s    zBlockCache._read_cache)rOrrrrrr r[r^rArCrrVrcrDr r r r rNs  rNcs6eZdZdZdZd fdd ZddZdd ZZS) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. bytesTcs,t|||d|_d|_d|_||_dS)Nr)rr rrr;trim)r rrrrwr r r r Us zBytesCache.__init__cCs&|dkr d}|dkr|j}||jks,||kr0dS|jdk rx||jkrx|jdk rx||jkrx||j}|j||||S|jrt|j||j}n|}||ks||jkrdS|jdks||jkr|jdks||jkr||||_||_n||jkr@|j||jkr||||_||_n |||j}||_||j|_n^||jkr|j|jkr\nB||j|jkr||||_||_n||j|}|j||_|jt|j|_||j}|j||||}|jr"|j|j|jd}|dkr"|j|j|7_|j|j|d|_|S)Nrrr") rrr;rrr9rrFrw)r rr;offsetZbendnewrsnumr r r r\sh      zBytesCache._fetchcCs t|jSr)rFrr3r r r __len__szBytesCache.__len__)T) rrrrrr rr{rDr r r r ruFs  =rucs.eZdZdZdZdfdd ZddZZS) AllBytesz!Cache entire contents of the fileallNcs0t||||dkr&|d|j}||_dSrL)rr rrrM)r rrrrMr r r r szAllBytes.__init__cCs|j||Sr)rM)r rr;r r r rszAllBytes._fetch)NNNNrIr r r r r|sr|cs6eZdZdZdZidffdd ZfddZZS)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsTc stt||||||_|rt|}|dg}||dg} |ddD]f\} } |d\} } | | kr| | f|d<| d|| | f7<qP|| | f| || | fqPtt || |_ n||_ dS)Nrr") rr~r strictsortedkeysr8rmdictziprM)r rrrrMr_Z old_offsetsoffsetsrrrZstart0Zstop0r r r r s    zKnownPartsOfAFile.__init__csd}|jD]\\}}}||kr.|krnq||}|||||}|jrl||krh|krnn |d||t|7}|S|}qq|jdkrtd||fdtd||fdt d|d||t ||S)Nrz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching r0) rMitemsrrFrrhwarningswarnr5r6rr)r rrrsZloc0loc1rMoffr r r rs" zKnownPartsOfAFile._fetchrIr r r r r~sr~c@sNeZdZdZedddddgZddd Zd d Zd d Z ddZ ddZ dS) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache CacheInfohitsmissesmaxsizecurrsizecCs0t|_||_||_d|_d|_t|_ dSrL) collections OrderedDict_cache_func _max_size_hits_misses threadingLock_lock)r funcmax_sizer r r r s  zUpdatableLRU.__init__c Gs|j@||jkrB|j||jd7_|j|W5QRSW5QRX|j|}|j<||j|<|jd7_t|j|jkr|jjddW5QRX|S)Nr"Flast) rr move_to_endrrrrFrpopitem)r argsresultr r r __call__s     zUpdatableLRU.__call__c Gs(|j||jkW5QRSQRXdSr)rr)r rr r r is_key_cached%szUpdatableLRU.is_key_cachedc Gs>|j.||j|<t|j|jkr0|jjddW5QRXdS)NFr)rrrFrr)r rrr r r add_key)s zUpdatableLRU.add_keyc Cs<|j,|j|jt|j|j|jdW5QRSQRXdS)N)rrrr)rrrrFrrrr3r r r r^/szUpdatableLRU.cache_infoN)r) rrrrr namedtuplerr rrrr^r r r r rs  rcsdeZdZdZdZdfdd ZddZdd Zd d Zd d Z ddZ dfdd Z ddZ Z S)BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. backgroundrOcsZt|||t|||_||_t|j||_t dd|_ d|_ d|_ t |_dS)Nr" max_workers)rr rPrQrRrSrrVrWr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrXr r r r Us zBackgroundBlockCache.__init__cCsd|j|j|jS)Nz8rYr3r r r r[`s zBackgroundBlockCache.__repr__cCs |jSr\r]r3r r r r^es zBackgroundBlockCache.cache_infocCs(|j}|d=|d=|d=|d=|d=|S)NrWrrrrr_r?r r r rApsz!BackgroundBlockCache.__getstate__cCsD|j|t|j|d|_tdd|_d|_d|_t |_ dS)NrSr"r) r=rBrrVrWrrrrrrrr?r r r rCys   z!BackgroundBlockCache.__setstate__c Cs|dkr d}|dkr|j}||jks,||kr0dS||j}||j}d}d}|j|jdk r|jrtd|j|j |j d|_ d|_n:t ||j ko|kn}|r|j }|j}d|_ d|_W5QRX|dk rtd|j| |t ||dD]}||q |d} |jF|jdkrn| |j krn|j| sn| |_ |j|j| d|_W5QRX|j||||dS)Nrrz3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.r"asyncr`)rrrrdoner5rirWrrrboolr7rRrrsubmitrVrc) r rr;rarbZfetch_future_block_numberZ fetch_futureZ must_joinrdZend_block_plus_1r r r rsx       zBackgroundBlockCache._fetchsynccsP||jkrtd||j||j}||j}td||t||}|S)rerfz!BlockCache fetching block (%s) %drg)r rdZlog_inforr;rjr r r rVs   z!BackgroundBlockCache._fetch_blockc Cs||j}||j}||kr2||}|||Sg}||||dt|d|D]} ||| q\|||d|d|SdSrkrlror r r rcs    z BackgroundBlockCache._read_cache)rO)rrtr r r r r9s   KrFcCs6|j}|s*|tkr*td|dt||t|<dS)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rcachesrh)clsclobberrr r r register_caches  r)F)rrTloggingrPr&rrconcurrent.futuresr getLoggerr5rrrErJrNrur|r~rrrrcr r r r sF  !J*WU8J