U -ee@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z mZddlmZddlmZddlmZddlmZmZmZmZmZddlmZmZddlZddlZddl Z dd lm!Z!dd l"m#Z#d d l$m%Z%m&Z&d d l'm(Z(ddl$m)Z)ddl*m+Z+ddl,m-Z-e).e/Z0dZ1ede2eZ3dgeeee2fe2dddZ4e2e5dddZ6e2e5dddZ7e2e5dddZ8e3e3ddd Z9dhe2e2e2d#d$d%Z:die2e2ee j;ee2e2e2d+d,d-Z?e2e2d.d/d0Z@dkd1d2ZAdle2d3d4d5ZBdmeee2eCfe2d6d7d8ZDdne2eee2e5feee2e5feCd:d;d<ZEGd=d>d>eFZGdoee2d?d@dAZHdde jIjJdddBd feeKeeCee jIjJeeeLeLeMeMdCdDdEZNdpe2e2eLeMeMeMe j;dGdHdIZOdqdJdKZPGdLdMdMejQjRZRdrdNdOZSdsdPdQZTdtdRdSZUdudUdVZVdve j;d3dWdXZWdwe2eee2e5feee2e5fee2d:dYdZZXdxe2d3d\d]ZYd^d_ZZd`daZ[dbdcZ\ej]dddedfZ^dS)yz Utilities for working with the local dataset cache. This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp Copyright by the AllenNLP authors. N)closingcontextmanager)partial)sha256)Path)ListOptionalTypeTypeVarUnion)urljoinurlparse)HfFolder)version) __version__config)DownloadConfig)logging)ExtractManager)FileLock .incompleteT)hf_modules_cachereturnc Csx|dk r |ntj}t|}|tjkrttj|tj|ddtjtj |dstt tj |ddW5QRX|S)z Add hf_modules_cache to the python path. By default hf_modules_cache='~/.cache/huggingface/modules'. It can also be set with the environment variable HF_MODULES_CACHE. This is used to add modules such as `datasets_modules` NTexist_okz __init__.pyw) rZHF_MODULES_CACHEstrsyspathappendosmakedirsexistsjoinopen)rr(Z/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/utils/file_utils.pyinit_hf_modules-s   r*)url_or_filenamercCs&t|jdko$tjt|jd SNz:/r schemer#r!ismountr+r(r(r) is_remote_url@sr2cCs$t|jdkp"tjt|jdSr,r.r1r(r(r) is_local_pathDsr3cCst|jdkotj| S)Nr-)r r/r#r!isabsr1r(r(r)is_relative_pathKsr5)r!rcCs6tjtjtjt|}t|tr2t|S|S)z'Convert relative path to absolute path.)r#r!abspath expanduser expandvarsr isinstancer)r!Z abs_path_strr(r(r)relative_to_absolute_pathOs r:FT) identifierfilenamercCs6|r|rtjntj}n|r tjntj}d|||fSN/)rZ"CLOUDFRONT_DATASETS_DISTRIB_PREFIXZS3_DATASETS_BUCKET_PREFIXZ!CLOUDFRONT_METRICS_DISTRIB_PREFIXZS3_METRICS_BUCKET_PREFIXr&)r;r<use_cdndatasetZendpointr(r(r) hf_bucket_urlUsrAcCstt||||d|dS)N)r;r<r?r@) max_retries) http_headrA)r;r<r?r@rBr(r(r) head_hf_s3]srD)r!namerevisionrcCsHttjrdnt}|p|}|r2tjj|||dStjj|||dSdS)Nmain)rFr!rE)rparser is_devreleaserZREPO_DATASETS_URLformatZREPO_METRICS_URL)r!rEr@rFZdefault_revisionr(r(r) hf_github_urlfs rK) base_name pathnamesrcGs8t|r"tj|fdd|DSt|f|SdS)Ncss&|]}t|tjddVqdSr>N)rreplacer#seplstrip).0pathnamer(r(r) qsz#url_or_path_join..)r2 posixpathr&ras_posix)rLrMr(r(r)url_or_path_joinosrW) url_or_pathrcCs*t|r|d|dStj|SdSr=)r2rindexr#r!dirname)rXr(r(r)url_or_path_parentvsr[cCsV|d}t|}|}|r@|d}t|}|d|7}|drR|d7}|S)a Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can identify it as a HDF5 file (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) utf-8.z.py)encoder hexdigestendswith)urletagZ url_bytesZurl_hashr<Z etag_bytesZ etag_hashr(r(r)hash_url_to_filename}s   rc)rcKs|dkrtf|}|jptj}t|tr0t|}t|trBt|}t|rt|||j |j |j |j |j |j|j|j|j|j|jd }n.)rrZ PY_VERSIONhuggingface_hubZPYARROW_VERSIONZTORCH_AVAILABLEZ TORCH_VERSIONZ TF_AVAILABLEZ TF_VERSIONZ JAX_AVAILABLEZ JAX_VERSIONZBEAM_AVAILABLEZ BEAM_VERSIONr9dictr&itemsr)rhZuar(r(r)get_datasets_user_agents"  $  rz deprecated)rarkuse_auth_tokenrcCsl|dkr td|dt|}i}|tjrh|dkr>d}nt|trN|}nt }|rhd||d<|S)zHandle the HF authenticationr{'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. You can remove this warning by passing 'token= ' instead.FNzBearer authorization) warningswarn FutureWarning startswithr HF_ENDPOINTr9rr get_token)rarkr|headersr(r(r)"get_authentication_headers_for_urls    rc@s eZdZdS)OfflineModeIsEnabledN)__name__ __module__ __qualname__r(r(r(r)rsrmsgcCs&tjr"t|dkrdn dt|dS)zaRaise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_DATASETS_OFFLINE is True.NzOffline mode is enabled.zOffline mode is enabled. )rZHF_DATASETS_OFFLINErrrr(r(r)!_raise_if_offline_mode_is_enabled sr?) func_args func_kwargs exceptions status_codesrBbase_wait_time max_wait_timec Cs|pd}|pi}d}z |||WS|k r} zf||ksJ|rP| jj|krP| nDt||d|} t|d| d||dt| |d7}W5d} ~ XYqXqdS)Nr(rrz timed out, retrying in zs... []r)response status_codeminloggerinfotimesleep) funcrrrrrBrrretryerr sleep_timer(r(r)_retrys    r$@)methodrarBrrtimeoutrc Kstd|d\}}|s|d7}z$tjf|||d|} d}Wqtjjtjjfk r} zR||krr| n@t|d|d||dt ||d |d} t | W5d } ~ XYqXq| S) aWrapper around requests to retry in case it fails with a ConnectTimeout, with exponential backoff. Note that if the environment variable HF_DATASETS_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised. Args: method (str): HTTP method, such as 'GET' or 'HEAD'. url (str): The URL of the resource to fetch. max_retries (int): Maximum number of retries, defaults to 0 (no retries). base_wait_time (float): Duration (in seconds) to wait before retrying the first time. Wait time between retries then grows exponentially, capped by max_wait_time. max_wait_time (float): Maximum amount of time between two retries, in seconds. **params (additional keyword arguments): Params to pass to :obj:`requests.request`. Tried to reach )rFr)rrarTz request to z timed out, retrying... [rrN) rrequestsrequestupperrConnectTimeoutConnectionErrorrrrrr) rrarBrrrparamstriessuccessrrrr(r(r)_request_with_retry-s rcCsJtd|tj||d\}}}t|dkrt|p i}t|dd|d<td|||||||d}|S)NrrHEAD)rrarfrrallow_redirectsrrB)rrrzrr)rarfrrrrrBrr(r(r)rCs rCcCsn|dkr td|dt|}t|jdkr2dSt||d}t||dd}||jrf|j dnd}|S) Nr{r}r~httphttpsrk)rrBETag) rrrr r/rrCraise_for_statusokrr)rarkr|rrrbr(r(r) request_etags  rdc$ s4| dkr td| dt| } |dkr.tj}t|tr@t|}tj |dd| rdt |t |j }n|}d}d}d}d}d}d}t |dd}tj ||}tj |r|s|s|St|| d }|dk r||d <|st |j}|d krt|}n>|d kr.t|| d }|r&|ddp(|ddnd}d}zNt|d||| |d}|jdkr|rd|jdnd}|jD]2\}}|drrd|krr|d|7}|j}qrd}d|kr|d|kr||d7}n|jdkrd|ks4|jdkrd|ks4|jdkrtd|s4td|js4|jdkrJd|krJd}td|n2|jdkr|tj |kr|| dkr|t!d |d!Wn0t"t#j$j%fk r}z|}W5d}~XYnX|sptj |r|s|S|rt&d"|d#n$|dk r |jd$kr t&d%|t'd&||dk r>t!d'|d(t(|d)n2|dk rbt!d'|d*|jd)nt!d'|t ||}tj ||}tj |r|s|S|d+}t)|xtj |r|s|W5QRS|d,t*d?fd.d/ }d0}|r"t+|d1d2}tj r"t,j-}|f}t|d3|j.|d krVt/||n4|d krrt0||| |d4nt1||||||| |d5W5QRXtd6|d7|t23|j.|t4d8} t4| t5|d8| @td9|||d:}!|d;}"t6|"d}#t78|!|#W5QRXW5QRX|S)@a Given a URL, look for the corresponding file in the local cache. If it's not there, download it. Then return the path to the cached file. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk r{r}r~NTrF)rbrrftprrrrb)rrfrrBrZdownload_warningzdrive.google.comz &confirm=zconfirm=z &confirm=tizfirebasestorage.googleapis.comiiz7^https?://github.com/.*?/.*?/releases/download/.*?/.*?$z#^https://.*?s3.*?amazonaws.com/.*?$zndownloader.figstatic.comz"Couldn't get ETag version for url izUnauthorized for URL zU. Please use the parameter `token=True` after logging in with `huggingface-cli login`z6Cannot find the requested files in the cached path at zi and outgoing traffic has been disabled. To enable file online look-ups, set 'local_files_only' to False.izCouldn't find file at rzCouldn't reach z ()z (error z.lockrw+bc 3s t| }|VW5QRXdSr)r')modefZincomplete_pathr(r)temp_file_managerns z)get_from_cache..temp_file_managerrza+b)rzB not found in cache or force_download set to True, downloading to )rmr)rfrrrrBrzstoring z in cache at izcreating metadata file for )rarbz.jsonrr\)encoding)r)9rrrrZHF_DATASETS_CACHEr9rrr#r$r r r!rcr&r%rr/rrrrCrrrryrrematchrarrrrOSErrorrrTimeoutrqrreprrrrstatst_sizerErrrrmoveumaskchmodr'jsondump)$rardrerfZ etag_timeoutrgrhrirjrBrkr|rlrmrnZ cached_url connectedrrrbZ head_errorr/r< cache_pathrrurvrZ lock_pathrrrrmeta meta_pathZ meta_filer(rr)rps        $         "                rpcsfdd}|S)Ncs(dd|jdk r|jnd|_|SNr-z )r&__doc__fndocstrr(r)docstring_decorators$z1add_start_docstrings..docstring_decoratorr(rrr(rr)add_start_docstringss rcsfdd}|S)Ncs(|jdk r|jnddd|_|Sr)rr&rrr(r)rs$z/add_end_docstrings..docstring_decoratorr(rr(rr)add_end_docstringss r cCstdd|DS)Ncss|]}|jVqdSr)r r )rRr!r(r(r)rTsz(estimate_dataset_size..)sum)rr(r(r)estimate_dataset_sizesr")rcCs4t}|d}|sq,||7}|drq,qt|S)Nr ) bytearrayrr`bytes)rresbr(r(r)readlines  r()N)FT)FTr)TN)N)N)N)Nr{)N)rrrr)N)NN)r)r)NrNNrrN)NNNTrr)Nr{)NFNrFNFTrNr{FNN)_rriorr#rUrrr rrr contextlibrr functoolsrhashlibrpathlibrtypingrrr r r urllib.parser r rrwrr packagingrr-rrZdownload.download_configrrrsrZfilelockrZ get_loggerrrZINCOMPLETE_SUFFIXrrr*boolr2r3r5r:rAResponserrDrKrWr[rcrtrxrzrrrrrRequestExceptiontuplerfloatrrr callbacksrrrrrrCrrprr r" RawIOBaser(r(r(r(r)s                 E      '   #   R