U 9%e@sddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZmZmZddlmZddlmZmZddlmZmZdd lmZmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&e#'e(Z)Gd d d Z*ee+e fe,dddZ-ee+e fe+e,dddZ.ee+e fe,dddZ/ee+e fe,dddZ0ee+e fe,dddZ1d*e+ee+e dfee+dddZ2ee+e fe,ddd Z3d+ee+e fee+e4d!d"d#Z5Gd$d%d%eZ6ed&d'Z7Gd(d)d)Z8dS),N)contextmanager)Path)CallableDictIteratorListOptionalTuple TypedDictUnion)urlparse)REPO_TYPES_URL_PREFIXES REPOCARD_NAME) metadata_load metadata_save)HfApirepo_type_and_id_from_hf_id)LFS_MULTIPART_UPLOAD_COMMAND)HfFolderSoftTemporaryDirectoryloggingrun_subprocesstqdmvalidate_hf_hub_argsc@seZdZdZdeeeejeedddZ e e dddZ e e dd d Ze e dd d Ze edd dZe edddZddZdS)CommandInProgressz= Utility to follow commands launched asynchronously. N)titleis_done_method status_methodprocess post_methodcCs.||_||_||_||_d|_d|_||_dS)N)r_is_done_status_process_stderr_stdout _post_method)selfrrrrr r)Y/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/huggingface_hub/repository.py__init__#szCommandInProgress.__init__returncCs(|}|r$|jdk r$|d|_|S)z. Whether the process is done. N)r"r')r(resultr)r)r*is_done3s zCommandInProgress.is_donecCs|S)z The exit code/status of the current action. Will return `0` if the command has completed successfully, and a number between 1 and 255 if the process errored-out. Will return -1 if the command is still ongoing. )r#r(r)r)r*status@s zCommandInProgress.statuscCs |jdkS)z2 Whether the process errored-out. rr1r0r)r)r*failedKszCommandInProgress.failedcCs(|jjdk r"|j|jj7_|jS)zC The current output message on the standard error. N)r$stderrr%readr0r)r)r*r4Rs zCommandInProgress.stderrcCs(|jjdk r"|j|jj7_|jS)zD The current output message on the standard output. N)r$stdoutr&r5r0r)r)r*r6[s zCommandInProgress.stdoutc Cs@|j}|dkrd}d|jd|d|js,dndd|jjd S) Nrunning[z command, status code: z, z in progress.z finished.z PID: ])r1rr/r$pid)r(r1r)r)r*__repr__ds ,zCommandInProgress.__repr__)N)__name__ __module__ __qualname____doc__strr subprocessPopenrr+propertyboolr/intr1r3r4r6r<r)r)r)r*rs(    r)folderr-cCs>tjtj|d}tjd|tjtjd}|o<|jdkS)a Check if the folder is the root or part of a git repository Args: folder (`str`): The folder in which to run the command. Returns: `bool`: `True` if the repository is part of a repository, `False` otherwise. z.gitz git branch)cwdr6r4r) ospathexistsjoinrBrunsplitPIPE returncode)rGZ folder_existsZ git_branchr)r)r* is_git_repoqs rQ)rG remote_urlr-cCs@t|s dStd|j}tdd|}dd|D}||kS)am Check if the folder is a local clone of the remote_url Args: folder (`str` or `Path`): The folder in which to run the command. remote_url (`str`): The url of a git repository. Returns: `bool`: `True` if the repository is a local clone of the remote repository specified, `False` otherwise. Fz git remote -v https://.*@https://cSsg|]}tdd|qS)rSrT)resub).0remoter)r)r* sz"is_local_clone..)rQrr6rUrVrN)rGrRremotesr)r)r*is_local_clones  r[)filenamer-c Cst|j}t|j}z"td|g|}|j}Wn@tjk rv}z t |s\WYdSt |j W5d}~XYnXt |dkrdSdddd}|dD]*}| D]}||krd|krd||<qqt|S) z Check if the file passed is tracked with git-lfs. Args: filename (`str` or `Path`): The filename to check. Returns: `bool`: `True` if the file passed is tracked with git-lfs, `False` otherwise. zgit check-attr -aFNr)diffmergefilter lfsT)rparentnamerrNr6striprBCalledProcessErrorrQOSErrorr4lenkeysallvalues)r\rGp attributesexcZ found_lfs_tag attributetagr)r)r*is_tracked_with_lfss"       rpc Cspt|j}t|j}z(td|g|dd}t|j }Wn.tjk rj}zt |j W5d}~XYnX|S)a Check if file is git-ignored. Supports nested .gitignore files. Args: filename (`str` or `Path`): The filename to check. Returns: `bool`: `True` if the file passed is ignored by `git`, `False` otherwise. zgit check-ignoreFcheckN) rrbrcrrNrErPrBrerfr4)r\rGrkZ is_ignoredrmr)r)r*is_git_ignoreds  rsc Csxz\t|d}|d}W5QRXtddddddd httd d d hB}t|d |WStk rrYdSXd S)z Check if file is a binary file. Args: filename (`str` or `Path`): The filename to check. Returns: `bool`: `True` if the file passed is a binary file, `False` otherwise. rbi  NT)openr5 bytearraysetrangerE translateUnicodeDecodeError)r\fcontentZ text_charsr)r)r*is_binary_files  *r.)patternrGr-c Cspzr?r__annotations__rFr)r)r)r*r5s rc csttjkr2z dVWntk r,YnXdStjddd}tj dd}t z}tj |dtjd<t dtjdt}tj||fd d }|z dVW5|| |tjd<XW5QRXdS) zv This is a context manager that will log the Git LFS progress of cleaning, smudging, pulling and pushing. N)stopping_eventc sFifddttdfdd }tjtjdsXrLdStdq(|tjdD]}z| \}}}}Wn2t k r}zt d ||W5d}~XYnX| d |}| d \} } t | } t | }  ||f} | dkr"t|| | d d ddt | d||f<qf| d| | d| | d<qfdS)zl To be launched as a separate thread with an event meaning it should stop the tail. csFD]8}|d|dj|d|d|dqdS)Nrr)rjupdatetotalrefreshclose)pbar)pbarsr)r* close_pbarsQs  z?_lfs_log_progress..output_progress..close_pbarsr,c 3sxt|dd}d}r qj|}|dk r^t|dks^||7}|drh|Vd}qtdqW5QRXdS)z Creates a generator to be iterated through, which will return each line one by one. Will stop tailing the file if the stopping_event is set. rr!Nrr`r)ris_setreadlinergrdendswithtimesleep)r\file current_lineZline_bit)rrr)r* tail_fileWs  z=_lfs_log_progress..output_progress..tail_fileGIT_LFS_PROGRESSNz!Cannot unpack LFS progress line: z file /BTi)descinitialrunitZ unit_scaleZ unit_divisor)rrrr)rrArIrJrKenvironrrrrN ValueError capitalizerFgetrr)rrlinestateZ file_progressZ byte_progressr\error descriptionZ current_bytes total_bytesZcurrent_bytes_intZtotal_bytes_intrr))rrrr*output_progressIs>  "  z*_lfs_log_progress..output_progressrr!Z lfs_progresszFollowing progress in T)targetargsdaemon)loggergetEffectiveLevelrERROR Exception threadingEventrIrrrrJrLdebugThreadstartr)rZcurrent_lfs_progress_valueZtmpdirZ exit_eventxr)r)r*_lfs_log_progress;s& L rc @seZdZUdZeeed<edYee e fe e e e ee e fe e e e e e e e e d ddZee d d d Zd d ZedZe ee e dfdddZd[e e e e dddZddZe d ddZe d ddZe d ddZee d ddZd\ee ee fe ddd Zee ee fd!d"d#Zd$d%Zd]e ee d'd(d)Zd^e ee d'd*d+Zd_d,d-Zd`e e d.d/d0Zdae e d1d2d3Z dbe d5d6d7Z!dce e e e ee e"e effd8d9d:Z#dde e d;dd?d@Z%dfe e e e d>dAdBZ&dge e e e e dCdDdEZ'e d dFdGZ(dhe e e e ede e"e effdHdIdJZ)e*die e e e e e dKdLdMZ+e e,d dNdOZ-e,ddPdQdRZ.edSdTZ/edUdVZ0dWdXZ1dS)j Repositoryz Helper class to wrap the git and git-lfs commands. The aim is to facilitate interacting with huggingface.co hosted model or dataset repos, though not a lot here (if any) is actually specific to huggingface.co. command_queueNTF) local_dir clone_from repo_typetokengit_user git_emailrevisionskip_lfs_filesclientc Csht|trt|}tj|ddtjt||_||_ g|_ ||_ | dk rR| nt |_ |t|trt||_n|dkrd|_n t|_|dk r|j|dnt|jrtdntd|jdk r|dks|dkr|j |j} |dkr| d}|dkr| d }|dk s$|dk r0||||||dk rX|j|dd t|jdS) a Instantiate a local clone of a git repo. If `clone_from` is set, the repo will be cloned from an existing remote repository. If the remote repo does not exist, a `EnvironmentError` exception will be thrown. Please create the remote repo first using [`create_repo`]. `Repository` uses the local git credentials by default. If explicitly set, the `token` or the `git_user`/`git_email` pair will be used instead. Args: local_dir (`str` or `Path`): path (e.g. `'my_trained_model/'`) to the local directory, where the `Repository` will be initialized. clone_from (`str`, *optional*): Either a repository url or `repo_id`. Example: - `"https://huggingface.co/philschmid/playground-tests"` - `"philschmid/playground-tests"` repo_type (`str`, *optional*): To set when cloning a repo from a repo_id. Default is model. token (`bool` or `str`, *optional*): A valid authentication token (see https://huggingface.co/settings/token). If `None` or `True` and machine is logged in (through `huggingface-cli login` or [`~huggingface_hub.login`]), token will be retrieved from the cache. If `False`, token is not sent in the request header. git_user (`str`, *optional*): will override the `git config user.name` for committing and pushing files to the hub. git_email (`str`, *optional*): will override the `git config user.email` for committing and pushing files to the hub. revision (`str`, *optional*): Revision to checkout after initializing the repository. If the revision doesn't exist, a branch will be created with that revision name from the default branch's current HEAD. skip_lfs_files (`bool`, *optional*, defaults to `False`): whether to skip git-LFS files or not. client (`HfApi`, *optional*): Instance of [`HfApi`] to use when calling the HF Hub API. A new instance will be created if this is left to `None`. Raises: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) if the remote repository set in `clone_from` does not exist. T)exist_okNF)repo_urlz [Repository] is a valid git repozNIf not specifying `clone_from`, you need to pass Repository a valid git clone.emailfullnamecreate_branch_ok) isinstancerrArImakedirsrJrLgetcwdr _repo_typerrrrcheck_git_versionshuggingface_tokenr get_tokenrrQrrrZwhoamigit_config_username_and_emaillfs_enable_largefilesgit_credential_helper_store git_checkoutatexitregisterwait_for_commands) r(rrrrrrrrruserr)r)r*r+s@;        zRepository.__init__r,c CsJztd|jj}Wn.tjk rD}zt|jW5d}~XYnX|S)zz Returns the current checked out branch. Returns: `str`: Current checked out branch. zgit rev-parse --abbrev-ref HEADNrrr6rdrBrerr4)r(r.rmr)r)r*current_branch s zRepository.current_branchcCs~ztd|jj}Wntk r2tdYnXztd|jj}Wntk rftdYnXt|d|dS)z Checks that `git` and `git-lfs` can be run. Raises: - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) if `git` or `git-lfs` are not installed. z git --versionz9Looks like you do not have git installed, please install.zgit-lfs --versionzLooks like you do not have git-lfs installed, please install. You can install from https://git-lfs.github.com/. Then run `git lfs install` (you only have to do this once).r`N)rrr6rdFileNotFoundErrorrrinfo)r(Z git_versionZ lfs_versionr)r)r*r/s zRepository.check_git_versions)rrc Csrt|tr|n|dkrdn|j}|dk r:|dr:td|jj}||ksdd|krt|ddkrt ||d\}}}|dk r|d|n|}|dk r||_ |d}|j t kr|t |j 7}|dk rt |j }||d |d |d }||7}td d |} z6td|jtt|jdkrtd| dtNtj} |jrd| ddit|jrrdndd|d|j| dW5QRXnt|jstd|jd|jdt|j|rt|jd| dnTtd|jdd} d | d!| d"} | jdkr2td#d$| j } | d%| 7} t| Wn0t!j"k rl}zt|j#W5d}~XYnXdS)&a Clone from a remote. If the folder already exists, will try to clone the repository within it. If this folder is a git repository with linked history, will try to update the repository. Args: repo_url (`str`): The URL from which to clone the repository token (`Union[str, bool]`, *optional*): Whether to use the authentication token. It can be: - a string which is the token itself - `False`, which would not use the authentication token - `True`, which would fetch the authentication token from the local folder and use it (you should be logged in for this to work). - `None`, which would retrieve the value of `self.huggingface_token`. Raises the following error: - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) if an organization token (starts with "api_org") is passed. Use must use your own personal access token (see https://hf.co/settings/tokens). - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) if you are trying to clone the repository in a non-empty folder, or if the `git` operations raise errors. FNZapi_orgzgYou must use your personal access token, not an Organization token (see https://hf.co/settings/tokens).httprr)hub_urlz://z://user:@z(https?)://.*@z\1://zgit lfs installrzCloning z into local empty directory.ZGIT_LFS_SKIP_SMUDGE1z git clonez git lfs clone z .)envzPTried to clone a repository in a non-empty folder that isn't a git repository ('z7'). If you really want to do this, do it manually: cd z && git init && git remote add origin && git pull origin main or clone repo to a new folder and move your existing files there afterwards.z is already a clone of z?. Make sure you pull the latest changes with `repo.git_pull()`.zgit remote get-url originrqzTried to clone zn in an unrelated git repository. If you believe this is an error, please add a remote with the following URL: rrSrTz' Local path has its origin defined as: )$rrAr startswithrrZendpointrgrNrrr r schemereplacerUrVrrrIlistdirrwarningrrcopyrrrQrr[rPr6rBrer4)r(rrrr namespace repo_nameZrepo_idrZclean_repo_urlroutput error_msgZclean_local_remote_urlrmr)r)r*rFsl& "        zRepository.clone_from)rrc Cstz@|dk r td|g|j|dk r>td||jWn.tjk rn}zt|jW5d}~XYnXdS)a Sets git username and email (only in the current repo). Args: git_user (`str`, *optional*): The username to register through `git`. git_email (`str`, *optional*): The email to register through `git`. Nzgit config user.namezgit config user.email )rrNrrBrerr4)r(rrrmr)r)r*rs z(Repository.git_config_username_and_emailc CsDztd|jWn.tjk r>}zt|jW5d}~XYnXdS)z; Sets the git credential helper to `store` z"git config credential.helper storeN)rrrBrerr4)r(rmr)r)r*rsz&Repository.git_credential_helper_storec CsLztd|j}|jWStjk rF}zt|jW5d}~XYnXdS)zy Get commit sha on top of HEAD. Returns: `str`: The current checked out commit SHA. zgit rev-parse HEADNr)r(rkrmr)r)r* git_head_hashs   zRepository.git_head_hashc CsZz&td|j}|j}tdd|WStjk rT}zt|j W5d}~XYnXdS)zp Get URL to origin remote. Returns: `str`: The URL of the `origin` remote. z"git config --get remote.origin.urlrSrTN) rrr6rdrUrVrBrerr4)r(rkurlrmr)r)r*git_remote_urls   zRepository.git_remote_urlcCs4|}|}|dr&|dd}|d|S)z Get URL to last commit on HEAD. We assume it's been pushed, and the url scheme is the same one as for GitHub or HuggingFace. Returns: `str`: The URL to the current checked-out commit. rNr7z/commit/)rrr)r(sharr)r)r*git_head_commit_urls   zRepository.git_head_commit_urlc Csztd|jj}Wn.tjk rD}zt|jW5d}~XYnXt|dkrVgSdd| dD}dd|D}dd|D}|S) z Returns a list of the files that are deleted in the working directory or index. Returns: `List[str]`: A list of files that have been deleted in the working directory or index. z git status -sNrcSsg|] }|qSr))rdrWr1r)r)r*rY%sz1Repository.list_deleted_files..r`cSs g|]}d|dkr|qS)Dr)rNrr)r)r*rY(scSsg|]}|dqS)r7)rNrdrr)r)r*rY+s rrr6rdrBrerr4rgrN)r( git_statusrmZmodified_files_statusesZdeleted_files_statuses deleted_filesr)r)r*list_deleted_files s   zRepository.list_deleted_files)patternsr\c Csrt|tr|g}z.|D]$}td|r&dndd||jqWn.tjk rl}zt|jW5d}~XYnXdS)a9 Tell git-lfs to track files according to a pattern. Setting the `filename` argument to `True` will treat the arguments as literal filenames, not as patterns. Any special glob characters in the filename will be escaped when writing to the `.gitattributes` file. Args: patterns (`Union[str, List[str]]`): The pattern, or list of patterns, to track with git-lfs. filename (`bool`, *optional*, defaults to `False`): Whether to use the patterns as literal filenames. zgit lfs track z --filenamer!rN)rrArrrBrerr4)r(rr\rrmr)r)r* lfs_track/s  zRepository.lfs_track)rc Csht|tr|g}z$|D]}td|g|jqWn.tjk rb}zt|jW5d}~XYnXdS)z Tell git-lfs to untrack those files. Args: patterns (`Union[str, List[str]]`): The pattern, or list of patterns, to untrack with git-lfs. zgit lfs untrackN) rrArrNrrBrerr4)r(rrrmr)r)r* lfs_untrackHs zRepository.lfs_untrackc Csdz0d}t|d|jt|dt|jWn.tjk r^}zt|jW5d}~XYnXdS)zI HF-specific. This enables upload support of files >5GB. z'git config lfs.customtransfer.multipartz.path huggingface-cliz.args N)rrrrBrerr4)r(Z lfs_configrmr)r)r*rXs z Repository.lfs_enable_largefilesr)rr-cCsg}|}t||jdD]v}||kr(qtjt|j|}t|st|stj |d}|dkrpt dt |}|r| |||q|||S)aH Automatically track binary files with git-lfs. Args: pattern (`str`, *optional*, defaults to "."): The pattern with which to track files that are binary. Returns: `List[str]`: List of filenames that are now tracked due to being binary files rGrxzParsing a large file to check if binary or not. Tracking large files using `repository.auto_track_large_files` is recommended so as to not load the full file in memory.)rrrrIrJrLrrprsgetsizerrrrappendr)r(rfiles_to_be_tracked_with_lfsrr\ path_to_file size_in_mbZ is_binaryr)r)r*auto_track_binary_filesfs$    z"Repository.auto_track_binary_filescCsg}|}t||jdD]`}||kr(qtjt|j|}tj|d}|dkrt|st |s| || |q| ||S)ap Automatically track large files (files that weigh more than 10MBs) with git-lfs. Args: pattern (`str`, *optional*, defaults to "."): The pattern with which to track files that are above 10MBs. Returns: `List[str]`: List of filenames that are now tracked due to their size. rrrx) rrrrIrJrLrr rprsrr r)r(rr rr\r r r)r)r*auto_track_large_filess    z!Repository.auto_track_large_filesc Cspz.status_methodpushcs dk SNrr)r r)r*Zz%Repository.git_push..)rrrr )rrrrrrBrCrNrO communicaterkillrgrerrr4rrrr r) r(rrrrZnumber_of_commitsr6r4Z return_codermrZcommand_in_progressr)r r*git_pushsR    "    zRepository.git_push)rrc Csz10MB). Args: commit_message (`str`): Message to use for the commit. blocking (`bool`, *optional*, defaults to `True`): Whether the function should return only when the `git push` has finished. clean_ok (`bool`, *optional*, defaults to `True`): If True, this function will return None if the repo is untouched. Default behavior is to fail because the git command fails. auto_lfs_prune (`bool`, *optional*, defaults to `False`): Whether to automatically prune files once they have been pushed to the remote. z*Repo currently clean. Ignoring push_to_hubNTrorigin rrr)r/rrrrr'r)r(rrr0rr)r)r* push_to_hubs     zRepository.push_to_hub)rbranchtrack_large_filesrrc cstd|jd}t|rPt|dkr>t|ddddd}td|d|dk rf|j|d d t|jrtd |j d d ntd |j dt }t t j||jz |VW5|j|dz||Wn6tk r} zdt| kr| W5d} ~ XYnXz|jd|j ||dWnBtk rv} z"dt| krbtd| n| W5d} ~ XYnXt |XdS)a Context manager utility to handle committing to a repository. This automatically tracks large files (>10Mb) with git-lfs. Set the `track_large_files` argument to `False` if you wish to ignore that behavior. Args: commit_message (`str`): Message to use for the commit. branch (`str`, *optional*): The branch on which the commit will appear. This branch will be checked-out before any operation. track_large_files (`bool`, *optional*, defaults to `True`): Whether to automatically track large files or not. Will do so by default. blocking (`bool`, *optional*, defaults to `True`): Whether the function should return only when the `git push` has finished. auto_lfs_prune (`bool`, defaults to `True`): Whether to automatically prune files once they have been pushed to the remote. Examples: ```python >>> with Repository( ... "text-files", ... clone_from="/text-files", ... token=True, >>> ).commit("My first file :)"): ... with open("file.txt", "w+") as f: ... f.write(json.dumps({"hey": 8})) >>> import torch >>> model = torch.nn.Transformer() >>> with Repository( ... "torch-model", ... clone_from="/torch-model", ... token=True, >>> ).commit("My cool model :)"): ... torch.save(model.state_dict(), "model.pt") ``` rrNr7z, ...]zPThere exists some updated files in the local repository that are not committed: z|. This may lead to errors if checking out a branch. These files and their modifications will be added to the current commit.TrzPulling changes ...)rz@The current branch has no upstream branch. Will push to 'origin 'r1znothing to commitr2r3zcould not read UsernamezCCouldn't authenticate user for push. Did you set `token` to `True`?)rrrgrArrrrrrrrIrchdirrJrLrrrfr') r(rr5r6rrZfiles_to_stageZ files_in_msgZcurrent_working_directoryer)r)r*commit sD7         zRepository.commitcCs(tj|jt}tj|r$t|SdSr")rIrJrLrrisfiler)r(filepathr)r)r*repocard_metadata_loads z!Repository.repocard_metadata_load)datar-cCsttj|jt|Sr")rrIrJrLrr)r(r?r)r)r*repocard_metadata_savesz!Repository.repocard_metadata_savecCsdd|jDS)z@ Returns the asynchronous commands that failed. cSsg|]}|jdkr|qS)rr2rWcr)r)r*rYs z.Repository.commands_failed..rr0r)r)r*commands_failedszRepository.commands_failedcCsdd|jDS)zS Returns the asynchronous commands that are currently in progress. cSsg|]}|js|qSr))r/rAr)r)r*rYsz3Repository.commands_in_progress..rCr0r)r)r*commands_in_progressszRepository.commands_in_progresscCsxd}|jD].}td|jd|jjdt|jq |jrt|ddkr`td|jd|d7}t dq:d S) zr Blocking method: blocks all subsequent execution until all commands have been processed. rzThe z command with PID z failed.rxzCWaiting for the following commands to finish before shutting down: rrN) rDrrrr$r;r4rErrr)r(indexZcommand_failedr)r)r*rs   zRepository.wait_for_commands)NNTNNNFN)N)NN)F)r)r)F)FF)rF)r)NTF)F)N)N)NN)rTTF)NTTF)2r=r>r?r@rrrrr rArrrErr+rDrrrrrrrrrrrrrrrrrrr r'rr)r,r.r/r4rr;rr>r@rDrErr)r)r)r*rs    l  #+! " Z %" $j  r)rN)N)9rrIrUrBrr contextlibrpathlibrtypingrrrrrr r r urllib.parser Zhuggingface_hub.constantsr rZhuggingface_hub.repocardrrZhf_apirrrarutilsrrrrrrZ get_loggerr=rrrArErQr[rprsrrrrFrrrrr)r)r)r*s8  (   S%"  l