U 楡cxk@s0ddlmZddlZddlZddlZddlZddlmZmZm Z m Z m Z ddl m Z mZmZmZmZddlmZmZddlmZdZzddlmZd ZWnek rYnXd ZGd d d eZGd ddeZGdddZGdddeZ GdddeZ!GdddeZ"GdddZ#GdddZ$dS))BytesION)msb_size stream_copyapply_delta_dataconnect_deltas delta_types)allocate_memory LazyMixinmake_shawriteclose) NULL_BYTE BYTE_SPACE) force_bytesF) apply_deltaT) DecompressMemMapReaderFDCompressedSha1WriterDeltaApplyReader Sha1WriterFlexibleSha1WriterZippedStoreShaWriterrFDStream NullStreamc@s~eZdZdZdZdZdddZddZd d Zd d Z e dddZ ddZ ddZ ddZeeddfddZdddZdS) raReads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly A constraint on the total size of bytes is activated, simulating a logical file within a possibly larger physical memory area To read efficiently, you clearly don't want to read individual bytes, instead, read a few kilobytes at least. **Note:** The chunk-size should be carefully selected as it will involve quite a bit of string copying due to the way the zlib is implemented. Its very wasteful, hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, that's all ... ) _m_zip_buf_buflen_br_cws_cwe_s_close_cbr_phiiNcCsR||_t|_d|_d|_|dk r*||_d|_d|_d|_ d|_ d|_ ||_ dS)z|Initialize with mmap for stream reading :param m: must be content data - use new if you have object data and no sizeNrF) rzlib decompressobjrrrr rrrr"r#r!)selfmclose_on_deletionsizer*0/tmp/pip-unpacked-wheel-jjg5dubb/gitdb/stream.py__init__Es zDecompressMemMapReader.__init__cCs|dks t|dS)Nr )AssertionError_parse_header_infor&attrr*r*r+ _set_cache_Us z"DecompressMemMapReader._set_cache_cCs |dSN)r r&r*r*r+__del__[szDecompressMemMapReader.__del__cCs~d}||_||}|t}|d|t\}}t|}||_d|_|d7}t||d|_ t |||_ d|_ ||fS)zIf this stream contains object data, parse the header info and skip the stream to a point where each read will yield object content :return: parsed type_string, sizei NrT) r readfindr splitrintrrrlenrr#)r&ZmaxbhdrZhdrendtypr)r*r*r+r.^s  z)DecompressMemMapReader._parse_header_infoFcCs"t||d}|\}}|||fS)aCreate a new DecompressMemMapReader instance for acting as a read-only stream This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. :param m: memory map on which to operate. It must be object data ( header + contents ) :param close_on_deletion: if True, the memory map will be closed once we are being deletedr)rr.)r&r'r(instr<r)r*r*r+new{s  zDecompressMemMapReader.newcCs|jS)z8:return: random access compatible data we are working on)rr3r*r*r+dataszDecompressMemMapReader.datacCs&|jr"t|jdr|jd|_dS)zClose our underlying stream of compressed bytes if this was allowed during initialization :return: True if we closed the underlying stream :note: can be called safely r FN)r!hasattrrr r3r*r*r+r s  zDecompressMemMapReader.closecCsx|j|jkrr|jjsrd|_t|jdrD|jjtjkrj|t j q&n&|jjsj|j t |j krj|t j qD|j|_|j S)z :return: number of compressed bytes read. This includes the bytes it took to decompress the header ( if there was one )rstatus)rr r unused_datar@rAr$ZZ_OKr6mmapPAGESIZEr"r:rr3r*r*r+compressed_bytes_reads z,DecompressMemMapReader.compressed_bytes_readSEEK_SETrcCsV|dks|ttddkr tdt|_d|_|_|_|_ |j rRd|_ |` dS)zgAllows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0rrFCan only seek to position 0FN) getattros ValueErrorr$r%rrrrr"r#r r&offsetwhencer*r*r+seeks zDecompressMemMapReader.seekcCs|dkr|j|j}nt||j|j}|dkr4dSd}|jr|j|krt|j|}|j|8_|j|7_|S|j}||j8}|j|j7_d|_d|_|jj}|r|jt ||_ |j ||_n|j }|j|_ |||_|j|j dkr|j d|_|j |j |j}|j t ||_|j ||}t jdkrZtjdksZt |jj}nt |jjt |jj}|jt ||7_|jt |7_|r||}|rt |t ||kr|j|jkr|||t |7}|S)Nr5r)z1.2.7z1.2.5darwin)r rminrrr6runconsumed_tailrr:rr decompressr$ ZLIB_VERSIONsysplatformrBr")r&r)ZdattailZcwsZindataZdcompdatZunused_datalenr*r*r+r6sN       *zDecompressMemMapReader.read)N)F)rO)__name__ __module__ __qualname____doc__ __slots__Z max_read_sizer,r1r4r. classmethodr>r?r rErHrIrNr6r*r*r*r+r.s   1rc@seZdZdZdZdZddZddZdd Ze s6eZ neZ dd d Z e e d d fddZeddZeddZeddZeddZdS)raA reader which dynamically applies pack deltas to a base object, keeping the memory demands to a minimum. The size of the final object is only obtainable once all deltas have been applied, unless it is retrieved from a pack index. The uncompressed Delta has the following layout (MSB being a most significant bit encoded dynamic size): * MSB Source Size - the size of the base against which the delta was created * MSB Target Size - the size of the resulting data after the delta was applied * A list of one byte commands (cmd) which are followed by a specific protocol: * cmd & 0x80 - copy delta_data[offset:offset+size] * Followed by an encoded offset into the delta data * Followed by an encoded size of the chunk to copy * cmd & 0x7f - insert * insert cmd bytes from the delta buffer into the output stream * cmd == 0 - invalid operation ( or error in delta stream ) )_bstream _dstreams _mm_target_sizericCs:t|dkstd|d|_t|dd|_d|_dS)zInitialize this instance with a list of streams, the first stream being the delta to apply on top of all following deltas, the last stream being the base object onto which to apply the deltasr5z+Need at least one delta and one base streamrONr)r:r-r`tuplerar)r& stream_listr*r*r+r,hs zDeltaApplyReader.__init__cCst|jdkr||St|j}|dkrBd|_td|_dS||_t|j|_t|jj }t |jj |j |jj dt j|jj }||||jddS)Nr5r)r:ra_set_cache_brute_rZrboundrcrrbr`r)rr6r rCrDapplyrN)r&r0Zdclbbufr r*r*r+_set_cache_too_slow_without_crs        z.DeltaApplyReader._set_cache_too_slow_without_ccCsnt}d}|jD]L}|d}t|\}}t||\}}|||d|||ft||}q|jj} |}t|jdkrt| |} }t | } t |jj| j | dt j t |} d} tt|t|jD]\\} }}}}t |j|}| | t |j|j |jdt j dtkr$t| || nt| ||t|| j | | } } | d| d|} q| |_| |_dS)z*If we are here, we apply the actual deltasriNr5rf c_apply_delta)listrar6rappendmaxr`r)r:rrr rCrDzipreversedglobalsrkrrNrbrc)r&r0Zbuffer_info_listZmax_target_sizeZdstreambufrLZsrc_sizeZ target_sizeZ base_sizeriZtbufZfinal_target_sizedbufZddatar*r*r+rgs:    $     z"DeltaApplyReader._set_cache_brute_rcCsB|j|j}|dks||kr |}|j|}|jt|7_|S)Nr5)rcrrbr6r:)r&countblr?r*r*r+r6s   zDeltaApplyReader.readrFcCs6|dks|ttddkr tdd|_|jddS)zhAllows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0rrFrGN)rHrIrJrrbrNrKr*r*r+rNszDeltaApplyReader.seekcCs<t|dkrtd|djtkr4td|dj||S)a Convert the given list of streams into a stream which resolves deltas when reading from it. :param stream_list: two or more stream objects, first stream is a Delta to the object that you want to resolve, followed by N additional delta streams. The list's last stream must be a non-delta stream. :return: Non-Delta OPackStream object whose stream can be used to obtain the decompressed resolved data :raise ValueError: if the stream list cannot be handledzNeed at least two streamsrOzNCannot resolve deltas if there is no base object stream, last one was type: %s)r:rJtype_idrtype)clsrer*r*r+r>s  zDeltaApplyReader.newcCs|jjSr2)r`rxr3r*r*r+rxszDeltaApplyReader.typecCs|jjSr2)r`rwr3r*r*r+rwszDeltaApplyReader.type_idcCs|jS)z3:return: number of uncompressed bytes in the stream)rcr3r*r*r+r)szDeltaApplyReader.sizeN)r)rZr[r\r]r^Zk_max_memory_mover,rjrg has_perf_modr1r6rHrIrNr_r>propertyrxrwr)r*r*r*r+rBs&  "K     rc@s.eZdZdZdZddZddZd dd Zd S) rzpSimple stream writer which produces a sha whenever you like as it degests everything it is supposed to writesha1cCs t|_dSr2)r r|r3r*r*r+r,2szSha1Writer.__init__cCs|j|t|S)z{:raise IOError: If not all bytes could be written :param data: byte object :return: length of incoming data)r|updater:r&r?r*r*r+r 7s zSha1Writer.writeFcCs|r|jS|jS)z]:return: sha so far :param as_hex: if True, sha will be hex-encoded, binary otherwise)r| hexdigestdigest)r&Zas_hexr*r*r+shaDs zSha1Writer.shaN)F)rZr[r\r]r^r,r rr*r*r*r+r,s  rc@s$eZdZdZdZddZddZdS)rzZWriter producing a sha1 while passing on the written bytes to the given write functionwritercCst|||_dSr2)rr,r)r&rr*r*r+r,Ts zFlexibleSha1Writer.__init__cCst||||dSr2)rr rr~r*r*r+r Xs zFlexibleSha1Writer.writeN)rZr[r\r]r^r,r r*r*r*r+rNsrc@sPeZdZdZdZddZddZddZd d Ze e d d fd dZ ddZ dS)rz=Remembers everything someone writes to it and generates a sha)rrrocCs$t|t|_ttj|_dSr2)rr,rrrr$ compressobj Z_BEST_SPEEDror3r*r*r+r,bs zZippedStoreShaWriter.__init__cCs t|j|Sr2)rHrrr/r*r*r+ __getattr__gsz ZippedStoreShaWriter.__getattr__cCs$t||}|j|j||Sr2)rr rrrocompress)r&r?alenr*r*r+r js zZippedStoreShaWriter.writecCs|j|jdSr2)rrr roflushr3r*r*r+r pszZippedStoreShaWriter.closerFrcCs0|dks|ttddkr td|jddS)z`Seeking currently only supports to rewind written data Multiple writes are not supportedrrFrGN)rHrIrJrrrNrKr*r*r+rNsszZippedStoreShaWriter.seekcCs |jS)zA:return: string value from the current stream position to the end)rrgetvaluer3r*r*r+r{szZippedStoreShaWriter.getvalueN) rZr[r\r]r^r,rr r rHrIrNrr*r*r*r+r]srcs<eZdZdZdZedZfddZddZdd Z Z S) rzDigests data written to it, making the sha available, then compress the data and write it to the file descriptor **Note:** operates on raw file descriptors **Note:** for this to work, you have to use the close-method of this instance)fdr|roz+Failed to write all bytes to filedescriptorcs"t||_ttj|_dSr2)superr,rr$rrror&r __class__r*r+r,s zFDCompressedSha1Writer.__init__cCs>|j||j|}t|j|}|t|kr6|jt|S)zZ:raise IOError: If not all bytes could be written :return: length of incoming data)r|r}rorr rr:exc)r&r?cdataZ bytes_writtenr*r*r+r s     zFDCompressedSha1Writer.writecCs.|j}t|j|t|kr$|jt|jSr2)rorr rr:rr )r& remainderr*r*r+r s zFDCompressedSha1Writer.close) rZr[r\r]r^IOErrorrr,r r __classcell__r*r*rr+rs   rc@sFeZdZdZdZddZddZddd Zd d Zd d Z ddZ dS)rzA simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream takes ownership_fd_poscCs||_d|_dSNrrrr*r*r+r,szFDStream.__init__cCs$|jt|7_t|j|dSr2)rr:rIr rr~r*r*r+r szFDStream.writercCs:|dkrtj|j}t|j|}|jt|7_|Sr)rIpathgetsizeZ _filepathr6rrr:)r&rtbytesr*r*r+r6s z FDStream.readcCs|jSr2)rr3r*r*r+filenoszFDStream.filenocCs|jSr2)rr3r*r*r+tellsz FDStream.tellcCst|jdSr2)r rr3r*r*r+r szFDStream.closeN)r) rZr[r\r]r^r,r r6rrr r*r*r*r+rs rc@s0eZdZdZeZd ddZddZddZd S) rzVA stream that does nothing but providing a stream interface. Use it like /dev/nullrcCsdS)Nr*)r&r)r*r*r+r6szNullStream.readcCsdSr2r*r3r*r*r+r szNullStream.closecCst|Sr2)r:r~r*r*r+r szNullStream.writeN)r) rZr[r\r]rdr^r6r r r*r*r*r+rs  r)%iorrCrIrWr$Z gitdb.funrrrrrZ gitdb.utilrr r r r Z gitdb.constr rZgitdb.utils.encodingrrzZgitdb_speedups._perfrrk ImportError__all__rrrrrrrrr*r*r*r+s4   k"#("