U
    +-eO                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZ d dlmZmZ d dlZddlmZmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& eG dd dZ'G dd dej(Z)G dd dej*j+Z,e-e-dddZ.dS )    N)	dataclass)datetime)	has_magic)AnyDictListOptionalTupleUnion)quoteunquote   )CommitOperationCopyCommitOperationDelete)DEFAULT_REVISIONENDPOINTREPO_TYPE_MODELREPO_TYPES_MAPPINGREPO_TYPES_URL_PREFIXES)HfApi)EntryNotFoundErrorHFValidationErrorRepositoryNotFoundErrorRevisionNotFoundErrorhf_raise_for_statushttp_backoffpaginateparse_datetimec                   @   s@   e Zd ZU dZeed< eed< eed< eed< edddZd	S )
HfFileSystemResolvedPathzUData structure containing information about a resolved Hugging Face file system path.	repo_typerepo_idrevisionpath_in_reporeturnc                 C   s2   t | jd| j  dt| j d| j dS )N @/)r   getr   r    
safe_quoter!   r"   rstripself r-   _/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/huggingface_hub/hf_file_system.py	unresolve$   s    ,z"HfFileSystemResolvedPath.unresolveN)__name__
__module____qualname____doc__str__annotations__r/   r-   r-   r-   r.   r      s   
r   c                       s  e Zd ZdZdZdZdddee ee d fddZeeee e	e
ee f dd	d
Zdd Zd/eee edddZd0ee ddddZd1eeee ddddZd2eee ddddZd3ee
ee ee ddddZd4ee
e
ee eeeeeef f  dd d!Zd5eee d"d#d$Zd6eeee dd%d&d'Zeedd(d)Zeeeef d fd*d+Zd7eeee f e
ee ee d,d-d.Z  Z S )8HfFileSystema  
    Access a remote Hugging Face Hub repository as if were a local file system.

    Args:
        endpoint (`str`, *optional*):
            The endpoint to use. If not provided, the default one (https://huggingface.co) is used.
        token (`str`, *optional*):
            Authentication token, obtained with [`HfApi.login`] method. Will default to the stored token.

    Usage:

    ```python
    >>> from huggingface_hub import HfFileSystem

    >>> fs = HfFileSystem()

    >>> # List files
    >>> fs.glob("my-username/my-model/*.bin")
    ['my-username/my-model/pytorch_model.bin']
    >>> fs.ls("datasets/my-username/my-dataset", detail=False)
    ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json']

    >>> # Read/write files
    >>> with fs.open("my-username/my-model/pytorch_model.bin") as f:
    ...     data = f.read()
    >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f:
    ...     f.write(data)
    ```
    r%   ZhfNendpointtokenc                   s6   t  j|| |pt| _|| _t||d| _i | _d S )Nr7   )super__init__r   r8   r9   r   _api_repo_and_revision_exists_cache)r,   r8   r9   argsZstorage_options	__class__r-   r.   r;   M   s    
zHfFileSystem.__init__)r   r    r!   r$   c              
   C   s   |||f| j krz| jj|||d W n ttfk
rt } z,d|f| j |||f< d|f| j ||d f< W 5 d }~X Y nf tk
r } z(d|f| j |||f< d| j ||d f< W 5 d }~X Y n"X d| j |||f< d| j ||d f< | j |||f S )N)r!   r   F)TN)r=   r<   Z	repo_infor   r   r   )r,   r   r    r!   er-   r-   r.   _repo_and_revision_exist_   s    $"z%HfFileSystem._repo_and_revision_existc              
   K   sN   z| j |f| W dS  tk
rH } zt|tr2 W Y dS d}~X Y nX dS )a  Is there a file at the given path

        Exact same implementation as in fsspec except that instead of catching all exceptions, we only catch when it's
        not a `NotImplementedError` (which we do want to raise). Catching a `NotImplementedError` can lead to undesired
        behavior.

        Adapted from https://github.com/fsspec/filesystem_spec/blob/f5d24b80a0768bf07a113647d7b4e74a3a2999e0/fsspec/spec.py#L649C1-L656C25
        TFN)info	Exception
isinstanceNotImplementedError)r,   pathkwargsrA   r-   r-   r.   existsp   s    	
zHfFileSystem.exists)rG   r!   r$   c                 C   s*  t t t t t t ddd}| |}|s6tdnH|dd d t krzd|kr`td|dd\}}t| }nt}|	ddkrd|kr|dd\}}d|kr|dd\}}nd	}t
|}|||}| |||\}}	|st||	nd|dd d
 }
d|dd
d  }|dd }d|ddd  }|
}|}| |||\}}	|s
t|	ttfr|}|}| |||\}}|st||	n
t||	nT|}d	}d|kr|dd\}}t
|}|||}| |||\}}|s
td|d k	r|nt}t||||S )N)revision_in_pathr!   r$   c                 S   s8   |d k	r0| d k	r4| |kr4t d|  d| dn| }|S )NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.)
ValueError)rJ   r!   r-   r-   r.   %_align_revision_in_path_with_revision   s    zHHfFileSystem.resolve_path.<locals>._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r'   r   r   r&   r%      )r   r4   _strip_protocolrF   splitr   valuesr   r   countr   rB   FileNotFoundErrorjoinrE   r   r   r   r   )r,   rG   r!   rL   r   r    rJ   r"   Zrepo_and_revision_existerrZrepo_id_with_namespaceZpath_in_repo_with_namespaceZrepo_id_without_namespaceZpath_in_repo_without_namespace_r-   r-   r.   resolve_path   sd     





zHfFileSystem.resolve_path)rG   r$   c                 C   sJ   |s| j   | j  n,| | }|rF| j |d  | |}q(d S )N)dircacheclearZ$_repository_type_and_id_exists_cacherV   r/   pop_parent)r,   rG   r-   r-   r.   invalidate_cache   s    
zHfFileSystem.invalidate_cacherbHfFileSystemFile)rG   moder!   r$   c                 K   s(   |dkrt dt| |f||d|S )Nabz/Appending to remote files is not yet supported.)r^   r!   )rF   r]   )r,   rG   r^   r!   rH   r-   r-   r.   _open   s    zHfFileSystem._openc              
   K   sR   | j ||d}| jj|j|j| j|j|j|d|dd | j	|
 d d S )Nr!   commit_messagecommit_description)r"   r    r9   r   r!   rb   rc   rG   )rV   r<   Zdelete_filer"   r    r9   r   r!   r(   r[   r/   )r,   rG   r!   rH   resolved_pathr-   r-   r.   _rm   s    	zHfFileSystem._rmF)rG   	recursivemaxdepthr!   r$   c              
      s   j ||d}t|jd|j  j||||jd} fdd|D }dd |D }	d| d}
|
|rnd	nd7 }
|
|d k	rd
| dnd7 }
jj|j|jj	|	|j|d|
|dd j
| d d S )Nra   r%   )rg   rh   r!   c                    s*   g | ]"} |s|t d  d qS )r   N)isdirlen).0rG   	root_pathr,   r-   r.   
<listcomp>   s     
 z#HfFileSystem.rm.<locals>.<listcomp>c                 S   s   g | ]}t |d qS ))r"   )r   )rk   r"   r-   r-   r.   rn      s     zDelete  zrecursively zup to depth rb   rc   )r    r   r9   
operationsr!   rb   rc   rd   )rV   r   r(   r   r    expand_pathr!   r<   create_commitr9   r[   r/   )r,   rG   rg   rh   r!   rH   re   pathsZpaths_in_reporp   rb   r-   rl   r.   rm   s$    
	zHfFileSystem.rmT)rG   detailrefreshr!   r$   c                    sr  | j ||d}dt|j   |k}| }|| jks<|r8t|j|j|jd d }|}	| j|	|jd}
zt	|
}W n< t
k
r   d|jkr| |}	| j|	|jd}
n Y nX t|g|
}
g }|
D ]`}||d  |d |d d}|d d	kr"||d
 |dt|d d d || q|| j|	< | |}|sZ fdd|D }|rd|S dd |D S )z!List the contents of a directory.ra   r&   r%   r'   rG   sizetypenamerw   rx   fileoidlfsZ
lastCommitdate)Zblob_idr}   last_modifiedc              	      s&   g | ]}|d |d    ddiqS )rz   r%   r   )replacerk   orJ   r-   r.   rn   3  s     z#HfFileSystem.ls.<locals>.<listcomp>c                 S   s   g | ]}|d  qS )rz   r-   r   r-   r-   r.   rn   4  s     )rV   r)   r!   r/   rW   r   r   r    
_iter_treenextr   r"   rZ   	itertoolschainupdater(   r   appendZ_ls_from_cache)r,   rG   ru   rv   r!   rH   re   has_revision_in_pathZpath_prefixZ	tree_pathZ	tree_iterZ	tree_itemZchild_infosZ
child_infooutr-   r   r.   ls  sV       




zHfFileSystem.ls)rG   r!   c              	   c   sj   | j ||d}| jj d|j d|j dt|j d|j 	d}| j	 }t
|ddi|dE d H  d S )	Nra   z/api/zs/z/tree/r'   expandT)paramsheaders)rV   r<   r8   r   r    r)   r!   r"   r*   _build_hf_headersr   )r,   rG   r!   re   r   r-   r-   r.   r   6  s    0
zHfFileSystem._iter_tree)path1path2r!   r$   c                 K   s0  | j ||d}| j ||d}|j|jko2|j|jk}|r| j||jdd d k	rd| d| }| jj|j|j|j|d||ddt|j	|j	|jdgd	 nj| j
|d
|jd}	|	 }
W 5 Q R X d| d| }| jj|
|j	|j| j|j|j|d||dd | j| d | j| d d S )Nra   r}   zCopy z to rb   rc   r%   )Zsrc_path_in_repor"   Zsrc_revision)r    r   r!   rb   rc   rp   r\   Zpath_or_fileobjr"   r    r9   r   r!   rb   rc   rd   )rV   r   r    rC   r!   r<   rr   r(   r   r"   openreadupload_filer9   r[   r/   )r,   r   r   r!   rH   Zresolved_path1Zresolved_path2Z	same_reporb   fcontentr-   r-   r.   cp_file@  sD    



zHfFileSystem.cp_filec                 K   s&   | j |f|}d|krt||d S )Nr   )rC   IsADirectoryError)r,   rG   rH   rC   r-   r-   r.   modifiedj  s    zHfFileSystem.modifiedc                    s`   |  |}|jsPdt|j }||k}| }|s@||ddn|}|dddS t j|f|S )Nr&   r%   r   r   	directoryry   )rV   r"   r)   r!   r/   r   r:   rC   )r,   rG   rH   re   rJ   r   rz   r?   r-   r.   rC   p  s    
zHfFileSystem.info)rG   rg   rh   r$   c           	         s  |d k	r|dk rt dt|tr2 |g||S t } fdd|D }|D ]}t|rt j|f|}||O }|rN|t jt|f||d|O }qNn*|rt j|f|ddd|}||O }||krN|dks 	|rN|
| qN|st|tt|S )	Nr   zmaxdepth must be at least 1c                    s   g | ]}  |qS r-   )rN   )rk   pr+   r-   r.   rn     s     z,HfFileSystem.expand_path.<locals>.<listcomp>)rg   rh   TF)rh   Zwithdirsru   )rK   rE   r4   rq   setr   globlistfindrI   addrR   sorted)	r,   rG   rg   rh   rH   r   r   bitZrecr-   r+   r.   rq   z  s*    
$zHfFileSystem.expand_path)N)N)r\   N)N)FNN)TFN)N)N)FN)!r0   r1   r2   r3   Zroot_markerprotocolr   r4   r;   r	   boolrD   rB   rI   r   rV   r[   r`   rf   intrt   r   r
   r   r   r   r   r   r   r   rC   rq   __classcell__r-   r-   r?   r.   r6   +   st     E             1
*     r6   c                       s^   e Zd Zdeeee d fddZeeedddZ	ddd	d
Z
deddddZ  ZS )r]   N)fsrG   r!   c                    s*   t  j||f| |  |j||d| _d S )Nra   )r:   r;   rV   re   )r,   r   rG   r!   rH   r?   r-   r.   r;     s    zHfFileSystemFile.__init__)startendr$   c                 C   s   dd| d|d  i| j j }| j j dt| jjd| jj  dt	| jj
 dt	| jj }td||d	}t| |jS )
Nrangezbytes=-r   r'   r%   z	/resolve/GET)r   )r   r<   r   r8   r   r(   re   r   r    r)   r!   r"   r   r   r   )r,   r   r   r   urlrr-   r-   r.   _fetch_range  s     
@zHfFileSystemFile._fetch_ranger#   c                 C   s   t jddd| _d S )Nzhffs-F)prefixdelete)tempfileNamedTemporaryFile	temp_filer+   r-   r-   r.   _initiate_upload  s    z!HfFileSystemFile._initiate_uploadF)finalr$   c                 C   s   | j d | j  }| j| |r| j  | jjj| jj	| j
j| j
j| jj| j
j| j
j| jd| jdd t| jj	 | jj| j
 d d S )Nr   rb   rc   r   rd   )bufferseekr   r   writecloser   r<   r   rz   re   r"   r    r9   r   r!   rH   r(   osremover[   r/   )r,   r   blockr-   r-   r.   _upload_chunk  s&    




zHfFileSystemFile._upload_chunk)N)F)r0   r1   r2   r6   r4   r   r;   r   bytesr   r   r   r   r   r-   r-   r?   r.   r]     s   r]   )sr$   c                 C   s   t | ddS )Nr%   )safe)r   )r   r-   r-   r.   r)     s    r)   )/r   r   r   dataclassesr   r   r   r   typingr   r   r   r   r	   r
   urllib.parser   r   ZfsspecZ_commit_apir   r   	constantsr   r   r   r   r   Zhf_apir   utilsr   r   r   r   r   r   r   r   r   ZAbstractFileSystemr6   specZAbstractBufferedFiler]   r4   r)   r-   r-   r-   r.   <module>   s&    (  o+