U
    W+d<                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ G dd	 d	eZdddZG dd deZdS )    N)configstorage_uri_for_key)AWSAuthConnection)ResumableDownloadException)ResumableTransferDisposition)KeyFile)Keyc                   @   s    e Zd ZdZdd Zdd ZdS )ByteTranslatingCallbackHandlerz
    Proxy class that translates progress callbacks made by
    boto.s3.Key.get_file(), taking into account that we're resuming
    a download.
    c                 C   s   || _ || _d S N
proxied_cbdownload_start_point)selfr   r    r   F/tmp/pip-unpacked-wheel-dlxw5sjy/boto/s3/resumable_download_handler.py__init__?   s    z'ByteTranslatingCallbackHandler.__init__c                 C   s   |  | j| | d S r
   r   )r   Ztotal_bytes_uploadedZ
total_sizer   r   r   callC   s    z#ByteTranslatingCallbackHandler.callN)__name__
__module____qualname____doc__r   r   r   r   r   r   r	   9   s   r	   Fc                 C   sP   t | tr|s|  jS |s$|  }| dtj |  }|sL| |tj |S )zH
    Returns size of file, optionally leaving fp positioned at EOF.
    r   )	
isinstancer   ZgetkeysizetellseekosSEEK_ENDSEEK_SET)fpposition_to_eofZcur_poscur_file_sizer   r   r   get_cur_file_sizeH   s    
r!   c                   @   sZ   e Zd ZdZdZejeej	ej
fZdddZdd Zdd	 Zd
d Zdd ZdddZdS )ResumableDownloadHandlerz*
    Handler for resumable downloads.
       Nc                 C   s(   || _ || _d| _|r|   d| _dS )a  
        Constructor. Instantiate once for each downloaded file.

        :type tracker_file_name: string
        :param tracker_file_name: optional file name to save tracking info
            about this download. If supplied and the current process fails
            the download, it can be retried in a new process. If called
            with an existing file containing an unexpired timestamp,
            we'll resume the transfer for this file; else we'll start a
            new resumable download.

        :type num_retries: int
        :param num_retries: the number of times we'll re-try a resumable
            download making no progress. (Count resets every time we get
            progress, so download can span many more than this number of
            retries.)
        N)tracker_file_namenum_retriesetag_value_for_current_download_load_tracker_file_etagr   )r   r$   r%   r   r   r   r   b   s    z!ResumableDownloadHandler.__init__c              
   C   s   d }zz>t| jd}| d| _t| j| jk rBtd| j  W nB t	k
r } z$|j
t
jkrvtd| j|jf  W 5 d }~X Y nX W 5 |r|   X d S )Nr
zJCouldn't read etag in tracker file (%s). Restarting download from scratch.zJCouldn't read URI tracker file (%s): %s. Restarting download from scratch.)closeopenr$   readlinerstripr&   lenMIN_ETAG_LENprintIOErrorerrnoENOENTstrerror)r   fer   r   r   r'   ~   s    

z0ResumableDownloadHandler._load_tracker_file_etagc              
   C   s   |j d| _| jsd S d }z`z t| jd}|d| j  W n: tk
rx } ztd| j|j	f t
jW 5 d }~X Y nX W 5 |r|  X d S )N"'wz%s
zCouldn't write tracker file (%s): %s.
This can happenif you're using an incorrectly configured download tool
(e.g., gsutil configured to save tracker files to an unwritable directory))etagstripr&   r$   r*   r+   writer1   r   r4   r   ABORT)r   keyr5   r6   r   r   r   _save_tracker_info   s     
z+ResumableDownloadHandler._save_tracker_infoc                 C   s$   | j r tj| j r t| j  d S r
   )r$   r   pathexistsunlink)r   r   r   r   _remove_tracker_file   s    z-ResumableDownloadHandler._remove_tracker_filec	           
   
   C   s@  t |dd}	|	r| jr| j|jdkr|	|jkrXtd|j|	tt||jf t	j
n$|	|jkr||jjjdkrxtd dS |jjjdkrtd | }d	|	|jd f |d
< t||	j}|	| _n0|jjjdkrtd | | d| _|d t|tr|j||||||d|d n|j||||||dd |  dS )zr
        Attempts a resumable download.

        Raises ResumableDownloadException if any problems occur.
        T)r   r7   zp%s is larger (%d) than %s (%d).
Deleting tracker file, so if you re-try this download it will start from scratch   zDownload complete.NzResuming download.zbytes=%d-%dZRangez Starting new resumable download.r   override_num_retries	hash_algsrE   )r!   r&   r9   r:   r   r   namestrr   r   r<   bucket
connectiondebugr0   copyr	   r   r   r>   truncater   GSKeyget_fileflush)
r   r=   r   headerscbnum_cbtorrent
version_idrF   r    r   r   r   _attempt_resumable_download   sN    



 z4ResumableDownloadHandler._attempt_resumable_download
   Fc	                 C   s$  |j jj}	|si }| jdkr,tddd| _d}
t|}z6| |||||||| |   |	dkrjt	d W dS  | j
k
r  } zp|	dkrt	d|   t|tr|jtjkrt|tr|j||||||d|d	 n|j||||||dd
 W 5 d}~X Y n tk
r } zx|jtjkr<|	dkr8t	d|j   nJ|jtjkrn|	dkrbt	d|j  |    n|	dkrt	d|j  W 5 d}~X Y nX t||krd}
n|
d7 }
|
| jkrtdtjz|  W n tjk
r   Y nX d|
 }|	dkrt	d|
|f  t| q0dS )a  
        Retrieves a file from a Key
        :type key: :class:`boto.s3.key.Key` or subclass
        :param key: The Key object from which upload is to be downloaded
        
        :type fp: file
        :param fp: File pointer into which data should be downloaded
        
        :type headers: string
        :param: headers to send when retrieving the files
        
        :type cb: function
        :param cb: (optional) a callback function that will be called to report
             progress on the download.  The callback should accept two integer
             parameters, the first representing the number of bytes that have
             been successfully transmitted from the storage service and
             the second representing the total number of bytes that need
             to be transmitted.
        
        :type num_cb: int
        :param num_cb: (optional) If a callback is specified with the cb
             parameter this parameter determines the granularity of the callback
             by defining the maximum number of times the callback will be
             called during the file transfer.
             
        :type torrent: bool
        :param torrent: Flag for whether to get a torrent for the file

        :type version_id: string
        :param version_id: The version ID (optional)

        :type hash_algs: dictionary
        :param hash_algs: (optional) Dictionary of hash algorithms and
            corresponding hashing class that implements update() and digest().
            Defaults to {'md5': hashlib/md5.md5}.

        Raises ResumableDownloadException if a problem occurs during
            the transfer.
        NZBotor%      r   rC   zResumable download complete.zCaught exception (%s)rD   rG   z4Caught non-retryable ResumableDownloadException (%s)zXCaught non-retryable ResumableDownloadException (%s); aborting and removing tracker filez3Caught ResumableDownloadException (%s) - will retryzeToo many resumable download attempts failed without progress. You might try this download again later   zWGot retryable failure (%d progress-less in a row).
Sleeping %d seconds before re-trying)rJ   rK   rL   r%   r   getintr!   rW   rB   r0   RETRYABLE_EXCEPTIONS__repr__r   r1   r2   EPIPErO   rP   r   dispositionr   ZABORT_CUR_PROCESSmessager<   r*   httplibIncompleteReadtimesleep)r   r=   r   rR   rS   rT   rU   rV   rF   rL   Zprogress_less_iterationsZhad_file_bytes_before_attemptr6   Zsleep_time_secsr   r   r   rP      s    *

  
 



z!ResumableDownloadHandler.get_file)NN)NrX   FNN)r   r   r   r   r/   ra   HTTPExceptionr1   socketerrorgaierrorr\   r   r'   r>   rB   rW   rP   r   r   r   r   r"   X   s   

0    r"   )F)r2   ra   r   rerf   rc   Zbotor   r   Zboto.connectionr   Zboto.exceptionr   r   Zboto.s3.keyfiler   Zboto.gs.keyr   rO   objectr	   r!   r"   r   r   r   r   <module>   s   
