U
    9%eA                     @   s   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZmZmZ ddlZddlmZ dd	lmZ eeZd
ZG dd deZG dd dZedddZee dddZG dd de	ZG dd dZdS )a  
Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs.

Inspired by: github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]
path = /path/to/transformers/.env/bin/python args = -m debugpy --listen 5678 --wait-for-client
/path/to/transformers/src/transformers/commands/transformers_cli.py lfs-multipart-upload ```    N)ArgumentParser)AbstractContextManager)DictListOptional   )logging   )BaseTransformersCLICommandzlfs-multipart-uploadc                   @   s"   e Zd ZdZeedddZdS )LfsCommandsu  
    Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload
    large files >5GB 🔥. Spec for LFS custom transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ transformers-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file >5GB. It's documented in the error
    message you get if you just try to git push a 5GB file without having enabled it before.

    2. $ transformers-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the user.
    )parserc                 C   sP   | j ddd}|jdtdd |jdd d	 | j td
d}|jdd d	 d S )Nzlfs-enable-largefileszeDeprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB.)helppathz/Local path to repository you want to configure.)typer   c                 S   s   t | S N)LfsEnableCommandargs r   X/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/commands/lfs.py<lambda>?       z1LfsCommands.register_subcommand.<locals>.<lambda>)funczgDeprecated: use `huggingface-cli` instead. Command will get called by git-lfs, do not call it directly.c                 S   s   t | S r   )LfsUploadCommandr   r   r   r   r   H   r   )
add_parseradd_argumentstrset_defaultsLFS_MULTIPART_UPLOAD_COMMAND)r   Zenable_parserZupload_parserr   r   r   register_subcommand6   s    zLfsCommands.register_subcommandN)__name__
__module____qualname____doc__staticmethodr   r   r   r   r   r   r   $   s   r   c                   @   s   e Zd Zdd Zdd ZdS )r   c                 C   s
   || _ d S r   r   selfr   r   r   r   __init__L   s    zLfsEnableCommand.__init__c                 C   sp   t d tj| jj}tj|s6td td t	j
d d|d t	j
dt  d|d td d S )	NzcManaging repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead.z)This does not look like a valid git repo.r	   z=git config lfs.customtransfer.multipart.path transformers-cliT)checkcwdz-git config lfs.customtransfer.multipart.args z Local repo set up for largefiles)warningswarnosr   abspathr   isdirprintexit
subprocessrunsplitr   )r&   Z
local_pathr   r   r   r2   O   s$      zLfsEnableCommand.runNr    r!   r"   r'   r2   r   r   r   r   r   K   s   r   msgc                 C   s(   t | d } tj|  tj  dS )z-Write out the message in Line delimited JSON.
N)jsondumpssysstdoutwriteflushr5   r   r   r   	write_msgb   s    r>   )returnc                  C   sV   t tj  } d| d| dfkr0dS | ddkrRtd t	d | S )z$Read Line delimited JSON from stdin.	terminater   eventN)downloaduploadzReceived unexpected messager	   )
r8   loadsr:   stdinreadlinestripgetloggercriticalr0   r5   r   r   r   read_msgi   s    

rK   c                   @   sL   e Zd ZdZeeedddZdd Zdd Zdd
dZ	dd Z
dd ZdS )	FileSlicezq
    File-like object that only reads a slice of a file

    Inspired by stackoverflow.com/a/29838711/593036
    )filepath	seek_from
read_limitc                 C   s   || _ || _|| _d| _d S )Nr   )rM   rN   rO   n_seen)r&   rM   rN   rO   r   r   r   r'      s    zFileSlice.__init__c                 C   s    t | jd| _| j| j | S )Nrb)openrM   fseekrN   r&   r   r   r   	__enter__   s    zFileSlice.__enter__c                 C   s$   t | j j}t| j|| j S r   )r,   fstatrS   filenost_sizeminrO   rN   )r&   total_lengthr   r   r   __len__   s    zFileSlice.__len__c                 C   sP   | j | jkrdS | j| j  }| j|dk r.|nt||}|  j t|7  _ |S )Nr   r   )rP   rO   rS   readrZ   len)r&   nZremaining_amountdatar   r   r   r^      s    zFileSlice.readc                 c   s   | j ddV  d S )Ni  @ )r`   )r^   rU   r   r   r   __iter__   s    zFileSlice.__iter__c                 G   s   | j   d S r   )rS   closer%   r   r   r   __exit__   s    zFileSlice.__exit__N)r]   )r    r!   r"   r#   r   intr'   rV   r\   r^   rb   rd   r   r   r   r   rL   x   s   
rL   c                   @   s   e Zd Zdd Zdd ZdS )r   c                 C   s
   || _ d S r   r   r%   r   r   r   r'      s    zLfsUploadCommand.__init__c              
   C   s^  t tj  }|ddkr0|ddksLtddddi td	 ti  t	 }|d krltd
 |d }|d }|d d }|d d }t
|d}t| }g }	t|D ]r\}
}t||
| |dR}tj||d}|  |	|jd|
d	 d td||
d	 | |d W 5 Q R X qtj|||	dd}|  td|d qTd S )NrA   initZ	operationrC   error    zWrong lfs init operation)codemessager	   r   oidr   actionhrefheader
chunk_size)rN   rO   )ra   etag)rp   Z
partNumberprogress)rA   rk   Z
bytesSoFarZbytesSinceLast)rk   parts)r8   Zcomplete)rA   rk   )r8   rD   r:   rE   rF   rG   rH   r>   r0   rK   re   poplistvalues	enumeraterL   requestsputraise_for_statusappendheaderspost)r&   Zinit_msgr6   rk   rM   Zcompletion_urlrn   ro   Zpresigned_urlsrr   iZpresigned_urlra   rr   r   r   r2      sN    




zLfsUploadCommand.runNr4   r   r   r   r   r      s   r   )r#   r8   r,   r1   r:   r*   argparser   
contextlibr   typingr   r   r   rw   utilsr    r
   Z
get_loggerr    rI   r   r   r   r>   rK   rL   r   r   r   r   r   <module>   s&   
'%