U
    -e=)                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZ d dlZd dlmZmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d d	lm Z  de!d
ddZ"de!d
ddZ#dS )    N)contextmanager)partial)Path)urljoinurlparse)DownloadConfigconfig)ExtractManager)
!_raise_if_offline_mode_is_enabledftp_getftp_head"get_authentication_headers_for_urlhash_url_to_filenamehttp_get	http_headis_local_pathis_remote_urllogger)FileLock)returnc                 K   s   |dkrt f |}|jptj}t|tr0t|}t| trBt| } t| rt| ||j	|j
|j|j|j|j|j|j|j|jd}n<tj| r| }n*t| rtd|  dntd|  d|dkr|S |jrt|jdj||jd}|S )	a  
    Given something that might be a URL (or might be a local path),
    determine which. If it's a URL, download the file and cache it, and
    return the path to the cached file. If it's already a local path,
    make sure the file exists and then return the path.

    Return:
        Local path (string)

    Raises:
        FileNotFoundError: in case of non-recoverable file
            (non-existent or no cache on disk)
        ConnectionError: in case of unreachable url
            and no cache on disk
        ValueError: if it couldn't parse the url or filename correctly
        requests.exceptions.ConnectionError: in case of internet connection issue
    N)	cache_dirforce_downloadproxiesresume_download
user_agentlocal_files_onlyuse_etagmax_retriesuse_auth_tokenignore_url_paramsdownload_desczLocal file z doesn't existzunable to parse z as a URL or as a local path)r   )force_extract)r   r   r   ZDOWNLOADED_DATASETS_PATH
isinstancer   strr   get_from_cacher   r   r   r   r   r   r   r   r   r    ospathexistsr   FileNotFoundError
ValueErrorZextract_compressed_filer	   extractr!   )Zurl_or_filenameZdownload_configZdownload_kwargsr   Zoutput_path r+   U/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/utils/cache.pycached_path   sF    


 r-   Fd   Tc           !         sp  |dkrt j}t|tr t|}tj|dd |rDt| t| j	}n| }d}d}d}d}d}t
|dd}tj	||}tj	|r|s|s|S t| |
d}|dk	r||d< |sB| drt| }zLt| d|||	|d	}|jd
krZ|r|jdnd}|j D ]2\}}|drd| kr| d| 7 } |j}qd}d| krd| kr| d7 } n|jdkrpd| ks|jdkrd| ks|jdkrtd| std|js|jdkrd| krd}td|   n2|jdkrt j| kr|
dkrtd|  dW n0 ttjj fk
r@ } z|}W 5 d}~X Y nX |stj	|r`|s`|S |rxt!d| dn$|dk	r|jdkrt!d|  t"d |   |dk	rtd!|  d"t#| d#n2|dk	rtd!|  d$|j d#ntd!|  t
||}tj	||}tj	|r2|s2|S |d% }t$|" |r|d&  t% fd'd(}|}tj	 rt& j'}nd)}nt(t)j*|dd*}d)}| L}t|  d+|j+  | drt,| | nt-| ||||||	|d, W 5 Q R X td-|  d.|  t./|j+| td/|  | |d0}|d1 }t0|d2d3d4} t12||  W 5 Q R X W 5 Q R X |S )5a  
    Given a URL, look for the corresponding file in the local cache.
    If it's not there, download it. Then return the path to the cached file.

    Return:
        Local path (string)

    Raises:
        FileNotFoundError: in case of non-recoverable file
            (non-existent or no cache on disk)
        ConnectionError: in case of unreachable url
            and no cache on disk
    NT)exist_okF)etag)r   z
user-agentzftp://)allow_redirectsr   timeoutr   headers   ETagZdownload_warningzdrive.google.comz	&confirm=zconfirm=z
&confirm=ti  zfirebasestorage.googleapis.comi  i  z7^https?://github.com/.*?/.*?/releases/download/.*?/.*?$z#^https://.*?s3.*?amazonaws.com/.*?$zndownloader.figstatic.comz"Couldn't get ETag version for url i  zUnauthorized for URL z^. Please use the parameter `use_auth_token=True` after logging in with `huggingface-cli login`z6Cannot find the requested files in the cached path at zi and outgoing traffic has been disabled. To enable file online look-ups, set 'local_files_only' to False.i  zCouldn't find file at zTried to reach zCouldn't reach z ()z (error z.lockz.incompletec               	   3   s    t  d} | V  W 5 Q R X d S )Nza+b)open)fZincomplete_pathr+   r,   _resumable_file_manager   s    z/get_from_cache.<locals>._resumable_file_managerr   )dirdeletezB not found in cache or force_download set to True, downloading to )r   resume_sizer3   cookiesr   desczstoring z in cache at zcreating metadata file for )urlr0   z.jsonwzutf-8)encoding)3r   ZHF_DATASETS_CACHEr"   r   r#   r%   makedirsr   r   r&   r   joinr'   r   
startswithr   r   status_coder3   getr>   itemsrematchr@   r   infoZHF_ENDPOINTConnectionErrorOSErrorrequests
exceptionsTimeoutr(   r
   reprr   r   statst_sizer   tempfileNamedTemporaryFilenamer   r   shutilmover7   jsondump)!r@   r   r   r   Zetag_timeoutr   r   r   r   r   r   r   r    Z
cached_url	connectedresponser>   r0   Z
head_errorfilename
cache_pathr3   kveZ	lock_pathr:   Ztemp_file_managerr=   	temp_filemeta	meta_pathZ	meta_filer+   r9   r,   r$   b   s    






"





 r$   )N)NFNr.   FNFTr   NFN)$rY   r%   rI   rW   rT   
contextlibr   	functoolsr   pathlibr   urllib.parser   r   rN   Zdatasetsr   r   Zdatasets.utils.extractr	   Zdatasets.utils.file_utilsr
   r   r   r   r   r   r   r   r   r   Zdatasets.utils.filelockr   r#   r-   r$   r+   r+   r+   r,   <module>   s@   0 F            