U
    9%eE                     @   sb  d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( erddl)m*Z* e$+e,Z-e.dZ/dZ0dddZ1eG dd dZ2e%d=ee2 ee3 e3e3ee3 eee4 ee4 f dddZ5G dd deZ6G dd deZ7deee3 dddd Z8e4d!d"d#Z9e4d$d%d&Z:e4d'd(d)Z;de3dd*d+d,Z<dee=e3dd-d.d/Z>ee2e=ee3 d0d1d2Z?ee e3e7d3d4d5Z@dee3 e=ee d6d7d8ZAdee3 e=ee d6d9d:ZBG d;d< d<eZCdS )>z.Git LFS related type definitions and utilities    N)AbstractContextManager)	dataclass)ceil)getsize)Path)TYPE_CHECKINGBinaryIODictIterableListOptionalTuple	TypedDict)HTTPBasicAuth)ENDPOINTHF_HUB_ENABLE_HF_TRANSFERREPO_TYPES_URL_PREFIXES)get_session   )get_token_to_sendhf_raise_for_statushttp_backoffloggingvalidate_hf_hub_args)sha256sha_fileobj)CommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)AcceptzContent-Typec                   @   s`   e Zd ZU dZeed< eed< eed< eedddZ	eedd	d
Z
eedddZdS )
UploadInfoad  
    Dataclass holding required information to determine whether a blob
    should be uploaded to the hub using the LFS protocol or the regular protocol

    Args:
        sha256 (`bytes`):
            SHA256 hash of the blob
        size (`int`):
            Size in bytes of the blob
        sample (`bytes`):
            First 512 bytes of the blob
    r   sizesample)pathc              	   C   sH   t |}t|d }|dd d }t|}W 5 Q R X | |||dS )Nrb   r   r   r    )r   ioopenpeekr   )clsr!   r   filer    sha r+   R/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/huggingface_hub/lfs.py	from_pathF   s
    zUploadInfo.from_pathdatac                 C   s&   t | }| t||d d |dS )Nr#   )r   r    r   )r   digestlen)r(   r/   r*   r+   r+   r,   
from_bytesN   s    zUploadInfo.from_bytes)fileobjc                 C   sD   | d}|dtj t|}| }|dtj | |||dS )Nr#   r   r$   )readseekr%   SEEK_SETr   tell)r(   r3   r    r*   r   r+   r+   r,   from_fileobjS   s    
zUploadInfo.from_fileobjN)__name__
__module____qualname____doc__bytes__annotations__intclassmethodstrr-   r2   r   r8   r+   r+   r+   r,   r   3   s   
r   )upload_infostoken	repo_typerepo_idendpointreturnc           
      C   s   |dk	r|nt }d}|tkr$t| }| d| | d}t j|tdddgdd	 | D d
dtdt|phdd}t| | }|	dd}	t
|	tstddd	 |	D dd	 |	D fS )a  
    Requests the LFS batch endpoint to retrieve upload instructions

    Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        upload_infos (`Iterable` of `UploadInfo`):
            `UploadInfo` for the files that are being uploaded, typically obtained
            from `CommitOperationAdd.upload_info`
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        token (`str`, *optional*):
            An authentication token ( See https://huggingface.co/settings/tokens )

    Returns:
        `LfsBatchInfo`: 2-tuple:
            - First element is the list of upload instructions from the server
            - Second element is an list of errors, if any

    Raises:
        `ValueError`: If an argument is invalid or the server response is malformed

        `HTTPError`: If the server returned an error
    N /z.git/info/lfs/objects/batchuploadbasic	multipartc                 S   s   g | ]}|j  |jd qS )oidr   )r   hexr   ).0rJ   r+   r+   r,   
<listcomp>   s   z'post_lfs_batch_info.<locals>.<listcomp>r   )	operationZ	transfersobjectsZ	hash_algoZaccess_tokenT)headersjsonauthrS   zMalformed response from serverc                 S   s   g | ]}d |krt |qS error)_validate_batch_actionsrP   objr+   r+   r,   rQ      s      c                 S   s   g | ]}d |krt |qS rW   )_validate_batch_errorrZ   r+   r+   r,   rQ      s      )r   r   r   postLFS_HEADERSr   r   r   rU   get
isinstancelist
ValueError)
rB   rC   rD   rE   rF   Z
url_prefixZ	batch_urlrespZ
batch_inforS   r+   r+   r,   post_lfs_batch_info]   s8    #

rd   c                   @   s   e Zd ZU eed< eed< dS )PayloadPartT
partNumberetagN)r9   r:   r;   r?   r>   rA   r+   r+   r+   r,   re      s   
re   c                   @   s&   e Zd ZU dZeed< ee ed< dS )CompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rN   partsN)r9   r:   r;   r<   rA   r>   r   re   r+   r+   r+   r,   rh      s   
rh   r   )rR   lfs_batch_actionrC   rG   c           	   	   C   sH  t | |d}|dkr2td| j d dS |d d }t| |d d}|dk	rdt| |di }|d}|dk	rzt|}W n( ttfk
r   td	| d
Y nX t	| |||d d nt
| |d d |dk	r2t| t j|d tdt|p
dd| jj | jjdd}t| t| j d dS )a  
    Handles uploading a given object to the Hub with the LFS protocol.

    Can be a No-op if the content of the file is already present on the hub large file storage.

    Args:
        operation (`CommitOperationAdd`):
            The add operation triggering this upload.
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
            more details.
        token (`str`, *optional*):
            A [user access token](https://hf.co/settings/tokens) to authenticate requests against the Hub

    Raises:
        - `ValueError` if `lfs_batch_action` is improperly formatted
        - `HTTPError` if the upload resulted in an error
    actionsNzContent of file z. is already present upstream - skipping uploadrJ   verifyheader
chunk_sizezTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.href)rR   rm   rn   
upload_url)rR   rp   USERT)usernamepasswordrM   )rV   rU   z: Upload successful)rY   r_   loggerdebugZpath_in_repo_validate_lfs_actionr?   rb   	TypeError_upload_multi_part_upload_single_partr   r]   r   r   upload_infor   rO   r   r   )	rR   rj   rC   rk   upload_actionverify_actionrm   rn   Zverify_respr+   r+   r,   
lfs_upload   s<    




r}   Z
lfs_actionc                 C   s:   t | dtr.| ddks6t | dts6td| S ).validates response from the LFS batch endpointro   rm   Nz"lfs_action is improperly formatted)r`   r_   rA   dictrb   r~   r+   r+   r,   rv      s    rv   )lfs_batch_actionsc                 C   sp   t | dtr t | dts(td| di d}| di d}|dk	r\t| |dk	rlt| | S )r   rN   r   z)lfs_batch_actions is improperly formattedrk   rJ   rl   N)r`   r_   rA   r?   rb   rv   )r   r{   r|   r+   r+   r,   rY      s     rY   )lfs_batch_errorc                 C   sh   t | dtr t | dts(td| d}t |tr\t |dtr\t |dtsdtd| S )r   rN   r   z'lfs_batch_error is improperly formattedrX   messagecode)r`   r_   rA   r?   rb   r   )r   
error_infor+   r+   r,   r\     s     
r\   )rR   rp   rG   c              	   C   s2   | j dd}td||d}t| W 5 Q R X dS )aZ  
    Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)

    Args:
        upload_url (`str`):
            The URL to PUT the file to.
        fileobj:
            The file-like object holding the data to upload.

    Returns: `requests.Response`

    Raises: `requests.HTTPError` if the upload resulted in an error
    TZ	with_tqdmPUTr.   N)as_filer   r   )rR   rp   r3   responser+   r+   r,   ry     s    ry   )rR   rm   rn   rp   rG   c                 C   s   t || j|d}t}tr>t| jts>t| jts>td d}|rPt	| ||dnt
| ||d}t j|t|| jj td}t| dS )z@
    Uploads file using HF multipart LFS transfer protocol.
    )rm   rz   rn   zlhf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular uploadF)rR   sorted_parts_urlsrn   )rU   rT   N)_get_sorted_parts_urlsrz   r   r`   path_or_fileobjrA   r   warningswarn_upload_parts_hf_transfer_upload_parts_iterativelyr   r]   _get_completion_payloadr   rO   r^   r   )rR   rm   rn   rp   r   Zuse_hf_transferresponse_headersZcompletion_resr+   r+   r,   rx   '  s,    

rx   )rm   rz   rn   rG   c                 C   sN   dd t dd |  D dd dD }t|}|t|j| krJtd|S )Nc                 S   s   g | ]\}}|qS r+   r+   )rP   _rp   r+   r+   r,   rQ   K  s   z*_get_sorted_parts_urls.<locals>.<listcomp>c                 S   s2   g | ]*\}}|  rt|d krt|d|fqS )r   
   )isdigitr1   r?   )rP   Zpart_numrp   r+   r+   r,   rQ   N  s    c                 S   s   | d S Nr   r+   )tr+   r+   r,   <lambda>S      z(_get_sorted_parts_urls.<locals>.<lambda>)keyz0Invalid server response to upload large LFS file)sorteditemsr1   r   r   rb   )rm   rz   rn   Zsorted_part_upload_urlsZ	num_partsr+   r+   r,   r   J  s    r   )r   rN   rG   c                 C   sf   g }t | D ]N\}}|d}|d ks.|dkrFtd| d|d  ||d |d q||dS )Nrg   rH   zInvalid etag (`z`) returned for part r   )rf   rg   )rN   ri   )	enumerater_   rb   append)r   rN   ri   Zpart_numberrm   rg   r+   r+   r,   r   \  s    
r   )rR   r   rn   rG   c           	      C   sr   g }| j ddX}t|D ]H\}}t||| |d(}td||d}t| ||j W 5 Q R X qW 5 Q R X |S )NTr   )	seek_from
read_limitr   r.   )r   r   SliceFileObjr   r   r   rT   )	rR   r   rn   rT   r3   Zpart_idxZpart_upload_urlZfileobj_sliceZpart_upload_resr+   r+   r,   r   k  s    "r   c              
   C   sx   zddl m} W n tk
r,   tdY nX z|| j||ddddW S  tk
rr } ztd|W 5 d }~X Y nX d S )	Nr   )multipart_uploadzFast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not available in your environment. Try `pip install hf_transfer`.         )	file_pathZ
parts_urlsrn   Z	max_filesZparallel_failuresmax_retriesz~An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.)Zhf_transferr   ImportErrorrb   r   	ExceptionRuntimeError)rR   r   rn   r   er+   r+   r,   r   |  s*    
r   c                   @   sp   e Zd ZdZeeedddZdd Zdd Zded
ddZ	edddZ
ejfeeedddZdd ZdS )r   a  
    Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object.

    This is NOT thread safe

    Inspired by stackoverflow.com/a/29838711/593036

    Credits to @julien-c

    Args:
        fileobj (`BinaryIO`):
            A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course).
            `fileobj` will be reset to its original position when exiting the context manager.
        seek_from (`int`):
            The start of the slice (offset from position 0 in bytes).
        read_limit (`int`):
            The maximum number of bytes to read from the slice.

    Attributes:
        previous_position (`int`):
            The previous position

    Examples:

    Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327):
    ```python
    >>> with open("path/to/file", "rb") as file:
    ...     with SliceFileObj(file, seek_from=128, read_limit=200) as fslice:
    ...         fslice.read(...)
    ```

    Reading a file in chunks of 512 bytes
    ```python
    >>> import os
    >>> chunk_size = 512
    >>> file_size = os.getsize("path/to/file")
    >>> with open("path/to/file", "rb") as file:
    ...     for chunk_idx in range(ceil(file_size / chunk_size)):
    ...         with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice:
    ...             chunk = fslice.read(...)

    ```
    r3   r   r   c                 C   s   || _ || _|| _d S Nr   )selfr3   r   r   r+   r+   r,   __init__  s    zSliceFileObj.__init__c                 C   sF   | j  | _| j dtj}t| j|| j | _	| j | jt
j | S r   )r3   r7   _previous_positionr5   osSEEK_ENDminr   r   _lenr%   r6   )r   Zend_of_streamr+   r+   r,   	__enter__  s
    zSliceFileObj.__enter__c                 C   s   | j | jtj d S r   )r3   r5   r   r%   r6   )r   exc_type	exc_value	tracebackr+   r+   r,   __exit__  s    zSliceFileObj.__exit__nc                 C   sB   |   }|| jkrdS | j| }| j|dk r2|nt||}|S )Nr   r   )r7   r   r3   r4   r   )r   r   posZremaining_amountr/   r+   r+   r,   r4     s    

zSliceFileObj.read)rG   c                 C   s   | j  | j S r   )r3   r7   r   r   r+   r+   r,   r7     s    zSliceFileObj.tell)offsetwhencerG   c                 C   s   | j }|| j }|tjtjfkrR|tjkr2|| n|| }t|t||}tj}n>|tjkr| j	 }t|| t||| }nt
d| d| j||| j  S )Nzwhence value z is not supported)r   r   r   r6   r   maxr   SEEK_CURr3   r7   rb   r5   )r   r   r   startendZcur_posr+   r+   r,   r5     s    


zSliceFileObj.seekc                 c   s   | j ddV  d S )Ni  @ r   )r4   r   r+   r+   r,   __iter__  s    zSliceFileObj.__iter__N)r   )r9   r:   r;   r<   r   r?   r   r   r   r4   r7   r   r6   r5   r   r+   r+   r+   r,   r     s   ,r   )N)Dr<   r%   r   rer   
contextlibr   dataclassesr   mathr   os.pathr   pathlibr   typingr   r   r	   r
   r   r   r   r   Zrequests.authr   Zhuggingface_hub.constantsr   r   r   Zhuggingface_hub.utilsr   utilsr   r   r   r   r   Z	utils.shar   r   Z_commit_apir   Z
get_loggerr9   rt   compileZ	OID_REGEXZLFS_MULTIPART_UPLOAD_COMMANDr^   r   rA   r   rd   re   rh   r}   rv   rY   r\   ry   r?   rx   r   r   r   r   r   r+   r+   r+   r,   <module>   sp   (

) H<
#    