U
    9%e                     @   s   d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZmZ ddlmZmZmZ eeZG d	d
 d
eZG dd dZedddZe
e dddZG dd dZdS )a  
Implementation of a custom transfer agent for the transfer type "multipart" for
git-lfs.

Inspired by:
github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]
path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
--wait-for-client
/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
lfs-multipart-upload ```    N)_SubParsersAction)DictListOptional)BaseHuggingfaceCLICommand)LFS_MULTIPART_UPLOAD_COMMANDSliceFileObj   )get_sessionhf_raise_for_statusloggingc                   @   s"   e Zd ZdZeedddZdS )LfsCommandsu  
    Implementation of a custom transfer agent for the transfer type "multipart"
    for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom
    transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ huggingface-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file
    >5GB. It's documented in the error message you get if you just try to git
    push a 5GB file without having enabled it before.

    2. $ huggingface-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the
    user.
    )parserc                 C   sP   | j ddd}|jdtdd |jdd d	 | j td
d}|jdd d	 d S )Nzlfs-enable-largefilesz:Configure your repository to enable upload of files > 5GB.)helppathz/Local path to repository you want to configure.)typer   c                 S   s   t | S N)LfsEnableCommandargs r   [/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/huggingface_hub/commands/lfs.py<lambda>?       z1LfsCommands.register_subcommand.<locals>.<lambda>)funcz<Command will get called by git-lfs, do not call it directly.c                 S   s   t | S r   )LfsUploadCommandr   r   r   r   r   E   r   )
add_parseradd_argumentstrset_defaultsr   )r   Zenable_parserZupload_parserr   r   r   register_subcommand8   s    zLfsCommands.register_subcommandN)__name__
__module____qualname____doc__staticmethodr   r    r   r   r   r   r   #   s   r   c                   @   s   e Zd Zdd Zdd ZdS )r   c                 C   s
   || _ d S r   r   selfr   r   r   r   __init__I   s    zLfsEnableCommand.__init__c                 C   sf   t j| jj}t j|s,td td tjd	 d|d tjdt
 	 d|d td d S )Nz)This does not look like a valid git repo.   z<git config lfs.customtransfer.multipart.path huggingface-cliT)checkcwdz-git config lfs.customtransfer.multipart.args z Local repo set up for largefiles)osr   abspathr   isdirprintexit
subprocessrunsplitr   )r'   Z
local_pathr   r   r   r2   L   s    zLfsEnableCommand.runNr!   r"   r#   r(   r2   r   r   r   r   r   H   s   r   msgc                 C   s(   t | d }tj| tj  dS )z-Write out the message in Line delimited JSON.
N)jsondumpssysstdoutwriteflush)r6   Zmsg_strr   r   r   	write_msg^   s    r>   returnc                  C   sV   t tj  } d| d| dfkr0dS | ddkrRtd t	d | S )z$Read Line delimited JSON from stdin.	terminater   eventN)downloaduploadzReceived unexpected messager)   )
r8   loadsr:   stdinreadlinestripgetloggercriticalr0   r5   r   r   r   read_msge   s    

rL   c                   @   s(   e Zd ZddddZddddZdS )r   Nr?   c                 C   s
   || _ d S r   r   r&   r   r   r   r(   u   s    zLfsUploadCommand.__init__c                 C   s  t tj  }|ddkr0|ddksLtddddi td	 ti  t	 }|d krltd
 |d }|d }|d d }|d d }t
|d}t| }td|d	d
d g }	t|d}
t|D ]t\}}t|
|| |dT}t j||d}t| |	|jd|d	 d td||d	 | |d W 5 Q R X qW 5 Q R X t j|||	dd}t| td|d qTd S )NrB   initZ	operationrD   error    zWrong lfs init operation)codemessager)   r   oidr   actionhrefheader
chunk_sizeprogress)rB   rR   Z
bytesSoFarZbytesSinceLastrb)Z	seek_fromZ
read_limit)dataetag)rZ   Z
partNumber)rR   parts)r8   Zcomplete)rB   rR   )r8   rE   r:   rF   rG   rH   rI   r>   r0   rL   intpoplistvaluesopen	enumerater   r
   putr   appendheaderspost)r'   Zinit_msgr6   rR   filepathZcompletion_urlrU   rV   Zpresigned_urlsr[   fileiZpresigned_urlrY   rr   r   r   r2   x   sh    

	


zLfsUploadCommand.runr4   r   r   r   r   r   t   s   r   )r$   r8   r,   r1   r:   argparser   typingr   r   r   Zhuggingface_hub.commandsr   Zhuggingface_hub.lfsr   r   utilsr
   r   r   Z
get_loggerr!   rJ   r   r   r>   rL   r   r   r   r   r   <module>   s   
%