U 3 dYC@sddlZddlZddlZddlZddlZddlZddlmZddlZddl m Z m Z m Z m Z mZddlmZmZmZeZedZGdddeZGdd d eZGd d d ejZGd d d eZGdddeZGdddeZdS)N)Queue)DEFAULT_PART_SIZEminimum_part_size chunk_hashes tree_hash bytes_to_hex)UploadArchiveErrorDownloadArchiveErrorTreeHashDoesNotMatchErrorzboto.glacier.concurrentc@s2eZdZedfddZddZddZdd Zd S) ConcurrentTransferer cCs||_||_g|_dSN) _part_size _num_threads_threads)self part_size num_threadsr;/tmp/pip-unpacked-wheel-d7dsrkjd/boto/glacier/concurrent.py__init__+szConcurrentTransferer.__init__cCsLt|}|j|kr|j}n|}td|j|tt|t|}||fS)NzfThe part size specified (%s) is smaller than the minimum required part size. Using a part size of: %s)rrlogdebugintmathceilfloat)r total_sizeZmin_part_size_requiredr total_partsrrr_calculate_required_part_size0s z2ConcurrentTransferer._calculate_required_part_sizecCs>td|jD] }d|_q|jD] }|q"tddS)NzShutting down threads.FzThreads have exited.)rrrshould_continuejoin)rthreadrrr_shutdown_threads<s     z&ConcurrentTransferer._shutdown_threadscCsDtdt|D]}|||fqt|jD]}|tq0dS)NzAdding work items to queue.)rrrangeputr _END_SENTINEL)rr worker_queuerirrr_add_work_items_to_queueDs   z-ConcurrentTransferer._add_work_items_to_queueN)__name__ __module__ __qualname__rrrr#r)rrrrr *s r cs@eZdZdZedffdd Zd ddZdd Zd d ZZ S) ConcurrentUploaderaConcurrently upload an archive to glacier. This class uses a thread pool to concurrently upload an archive to glacier using the multipart upload API. The threadpool is completely managed by this class and is transparent to the users of this class. r cs"tt|||||_||_dS)au :type api: :class:`boto.glacier.layer1.Layer1` :param api: A layer1 glacier object. :type vault_name: str :param vault_name: The name of the vault. :type part_size: int :param part_size: The size, in bytes, of the chunks to use when uploading the archive parts. The part size must be a megabyte multiplied by a power of two. :type num_threads: int :param num_threads: The number of threads to spawn for the thread pool. The number of threads will control how much parts are being concurrently uploaded. N)superr-r_api _vault_name)rapi vault_namerr __class__rrrVszConcurrentUploader.__init__Nc Cst|j}||\}}dg|}t}t}|j|j||} | d} ||||| || ||z| |||Wn@t k r} z"t d|j|j| | W5d} ~ XYnXt d|j|j| tt||} t d| dS)a^Concurrently create an archive. The part_size value specified when the class was constructed will be used *unless* it is smaller than the minimum required part size needed for the size of the given file. In that case, the part size used will be the minimum part size required to properly upload the given file. :type file: str :param file: The filename to upload :type description: str :param description: The description of the archive. :rtype: str :return: The archive id of the newly created archive. NZUploadIdzHAn error occurred while uploading an archive, aborting multipart upload.zCompleting upload.zUpload finished.Z ArchiveId)osstatst_sizerrr/Zinitiate_multipart_uploadr0r)_start_upload_threads_wait_for_upload_threadsrrrZabort_multipart_uploadZcomplete_multipart_uploadrr) rfilename descriptionrrr hash_chunksr' result_queueresponse upload_iderrruploadns@       zConcurrentUploader.uploadcCs\t|D]F}|}t|tr>td||td||\}}|||<q|dS)N?An error was found in the result queue, terminating threads: %s0An error occurred while uploading an archive: %s)r$get isinstance Exceptionrrr#r)rr<r=r_result part_numberZ tree_sha256rrrr9s   z+ConcurrentUploader._wait_for_upload_threadscCsRtdt|jD]8}t|j|j||||}td| |j |qdSNzStarting threads.g?) rrr$rUploadWorkerThreadr/r0timesleepstartrappend)rr=r?r'r:rGr"rrrr8s   z(ConcurrentUploader._start_upload_threads)N) r*r+r,__doc__rrrAr9r8 __classcell__rrr3rr-Ls  2r-cs4eZdZfddZddZddZddZZS) TransferThreadcs$tt|||_||_d|_dS)NT)r.rRr _worker_queue _result_queuer )rr'r=r3rrrszTransferThread.__init__cCsj|jr^z|jjdd}Wntk r0YqYnX|tkrF|dS||}|j|q|dS)N)timeout) r rSrDEmptyr&_cleanup_process_chunkrTr%)rworkrHrrrruns  zTransferThread.runcCsdSr r)rrZrrrrYszTransferThread._process_chunkcCsdSr rrrrrrXszTransferThread._cleanup)r*r+r,rr[rYrXrQrrr3rrRs  rRcs<eZdZddeffdd ZddZddZdd ZZS) rKc sLtt|||||_||_||_t|d|_||_||_ ||_ | |_ dS)Nrb) r.rKrr/r0 _filenameopen_fileobj _upload_id _num_retries_time_between_retries_retry_exceptions) rr1r2r:r?r'r= num_retriestime_between_retriesretry_exceptionsr3rrrs zUploadWorkerThread.__init__cCsd}t|jdD]z}z||}WqWq|jk r}z@td|d|j|d|jd|j|j|t |j |}W5d}~XYqXq|S)NrUzpException caught uploading part number %s for vault %s, attempt: (%s / %s), filename: %s, exception: %s, msg: %sr) r$rc _upload_chunkrererrorr0r_r4rLrMrd)rrZrHr(r@rrrrYs$   z!UploadWorkerThread._process_chunkc Cs|\}}||}|j||j|}t|}tt|}||t|df}t d|||j |j |j|t|||} | ||fS)NrUzUploading chunk %s of size %s)raseekreadhashlibsha256 hexdigestrrlenrrr/Z upload_partr0rbr) rrZrIr start_bytecontentsZ linear_hashZtree_hash_bytes byte_ranger>rrrris    z UploadWorkerThread._upload_chunkcCs|jdSr )racloser\rrrrXszUploadWorkerThread._cleanup) r*r+r,rFrrYrirXrQrrr3rrKsrKcs>eZdZdZedffdd ZddZddZd d ZZ S) ConcurrentDownloaderz Concurrently download an archive from glacier. This class uses a thread pool to concurrently download an archive from glacier. The threadpool is completely managed by this class and is transparent to the users of this class. r cstt|||||_dS)a :param job: A layer2 job object for archive retrieval object. :param part_size: The size, in bytes, of the chunks to use when uploading the archive parts. The part size must be a megabyte multiplied by a power of two. N)r.rur_job)rjobrrr3rrrs zConcurrentDownloader.__init__c Cs|jj}||\}}t}t}|||||||z||||Wn2tk r}zt d||W5d}~XYnXt ddS)z Concurrently download an archive. :param filename: The filename to download the archive to :type filename: str z2An error occurred while downloading an archive: %sNzDownload completed.) rvZ archive_sizerrr)_start_download_threads_wait_for_download_threadsr rr)rr:rrrr'r=r@rrrdownload$s  zConcurrentDownloader.downloadc Csdg|}t|d~}t|D]n}|}t|trTtd||td||\}} } } | ||<|| } | | | | | qW5QRXt t |} td|jj| |jj| kr|td|jj| f|dS)a Waits until the result_queue is filled with all the downloaded parts This indicates that all part downloads have completed Saves downloaded parts into filename :param filename: :param result_queue: :param total_parts: NwbrBrCz?Verifying final tree hash of archive, expecting: %s, actual: %szBTree hash for entire archive does not match, expected: %s, got: %s)r`r$rDrErFrrr#r rkwriteflushrrrvZsha256_treehashr )rr:r=rr<frGrHrIr actual_hashdatarqZ final_hashrrrry9sF         z/ConcurrentDownloader._wait_for_download_threadscCsJtdt|jD]0}t|j||}td||j |qdSrJ) rrr$rDownloadWorkerThreadrvrLrMrNrrO)rr=r'rGr"rrrrx`s   z,ConcurrentDownloader._start_download_threads) r*r+r,rPrrrzryrxrQrrr3rru s  'rucs4eZdZddeffdd ZddZddZZS)rr]cs.tt|||||_||_||_||_dS)a  Individual download thread that will download parts of the file from Glacier. Parts to download stored in work queue. Parts download to a temp dir with each part a separate file :param job: Glacier job object :param work_queue: A queue of tuples which include the part_number and part_size :param result_queue: A priority queue of tuples which include the part_number and the path to the temp file that holds that part's data. N)r.rrrvrcrdre)rrwr'r=rfrgrhr3rrrjs zDownloadWorkerThread.__init__c Csvd}t|jD]b}z||}WqrWq|jk rn}z(td|d|jt|j |}W5d}~XYqXq|S)z Attempt to download a part of the archive from Glacier Store the result in the result_queue :param work: Nz6Exception caught downloading part number %s for job %sr) r$rc_download_chunkrerrjrvrLrMrd)rrZrHrGr@rrrrYs   z#DownloadWorkerThread._process_chunkc Cs|\}}||}|||df}td|||j|}|}ttt|}|d|krttd||d|f||t ||fS)z Downloads a chunk of archive from Glacier. Saves the data to a temp file Returns the part number and temp file location :param work: rUzDownloading chunk %s of size %sZTreeHashzBTree hash for part number %s does not match, expected: %s, got: %s) rrrvZ get_outputrlrrrr binascii unhexlify) rrZrIrrqrsr>rrrrrrs   z$DownloadWorkerThread._download_chunk)r*r+r,rFrrYrrQrrr3rris r)r5r threadingrmrLloggingZ boto.compatrrZboto.glacier.utilsrrrrrZboto.glacier.exceptionsrr r objectr& getLoggerrr r-ThreadrRrKrurrrrrs"  "o4]