U
    MfU                     @   s   d dl Zd dlZd dlZd dlZd dlZd dlZd dlZddlm	Z	 ddl
mZ ddlmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZ ddlmZ ddlmZ G dd deZG dd de	Z dS )    N   )FileDownloader)HttpFD   )aes_cbc_decrypt_bytesunpad_pkcs7)compat_os_name)Request)	HTTPErrorIncompleteRead)DownloadErrorRetryManagerencodeFilenametraverse_obj)HTTPHeaderDict)ProgressCalculatorc                   @   s   e Zd Zdd ZeZdS )HttpQuietDownloaderc                 O   s   d S N )selfargsZkargsr   r   >/tmp/pip-unpacked-wheel-zk9bgjdq/yt_dlp/downloader/fragment.py	to_screen   s    zHttpQuietDownloader.to_screenN)__name__
__module____qualname__r   Zto_console_titler   r   r   r   r      s   r   c                   @   s   e Zd ZdZdd Zd*ddZdd Zd	d
 Zdd Zdd Z	dd Z
d+ddZdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ d%d$ ddd&d'd(d)ZdS ),
FragmentFDa  
    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).

    Available options:

    fragment_retries:   Number of times to retry a fragment for HTTP error
                        (DASH and hlsnative only). Default is 0 for API, but 10 for CLI
    skip_unavailable_fragments:
                        Skip unavailable fragments (DASH and hlsnative only)
    keep_fragments:     Keep downloaded fragments on disk after downloading is
                        finished
    concurrent_fragment_downloads:  The number of threads to use for native hls and dash downloads
    _no_ytdl_file:      Don't use .ytdl file

    For each incomplete fragment download yt-dlp keeps on disk a special
    bookkeeping file with download state and metadata (in future such files will
    be used for any incomplete download handled by yt-dlp). This file is
    used to properly handle resuming, check download file consistency and detect
    potential errors. The file has a .ytdl extension and represents a standard
    JSON file of the following format:

    extractor:
        Dictionary of extractor related data. TBD.

    downloader:
        Dictionary of downloader related data. May contain following data:
            current_fragment:
                Dictionary with current (being downloaded) fragment data:
                index:  0-based index of current fragment among all fragments
            fragment_count:
                Total count of fragments

    This feature is experimental and file format may change in future.
    c                 C   s   |  d | ||||S )Nz{yt_dlp.downloader.FragmentFD.report_retry_fragment is deprecated. Use yt_dlp.downloader.FileDownloader.report_retry instead)Zdeprecation_warningreport_retry)r   err
frag_indexcountretriesr   r   r   report_retry_fragment?   s    
z FragmentFD.report_retry_fragmentNc                 C   s2   |rd| dnd}|  d| d|dd d S )N ; z
[download]z Skipping fragment dz ...)r   )r   r   r   r   r   r   report_skip_fragmentD   s    zFragmentFD.report_skip_fragmentc                 C   s   | d}|rt|d |S |S )Nhttp_headers)getr	   )r   	info_dicturlheadersr   r   r   _prepare_urlH   s    
zFragmentFD._prepare_urlc                 C   s   |  | | || d S r   )_prepare_frag_download_start_frag_download)r   ctxr*   r   r   r    _prepare_and_start_frag_downloadL   s    
z+FragmentFD._prepare_and_start_frag_downloadc                 C   s&   |d dk	o$|d dko$| j d S )NliveTtmpfilename-Z_no_ytdl_file)paramsr)   )r   r0   r   r   r   Z__do_ytdl_fileP   s    zFragmentFD.__do_ytdl_filec                 C   s   d|kst | | |d d\}}zdzBt| }|d d d |d< d|d krh|d d |d< W n tk
r   d	|d< Y nX W 5 |  X d S )
Nytdl_corruptfilenamer
downloadercurrent_fragmentindexfragment_indexextra_stateT)AssertionErrorsanitize_openytdl_filenameclosejsonloadsread	Exception)r   r0   stream_Z	ytdl_datar   r   r   _read_ytdl_fileS   s    zFragmentFD._read_ytdl_filec                 C   s   |  | |d d\}}zVdd|d ii}d|kr@|d |d< |dd k	rZ|d |d< |td|i W 5 |  X d S )	Nr7   wr:   r;   r<   r=   fragment_countr9   )r?   r@   rA   r)   writerB   dumps)r   r0   Zfrag_index_streamrG   r9   r   r   r   _write_ytdl_file`   s     zFragmentFD._write_ytdl_filec                 C   s   d|d |d f }||p"| d|| dd}d}|d j d	d
rX| | |}| |d< |d< |d ||\}	}
|	sdS | dr| d|d< ||d< d
S )Nz	%s-Frag%dr3   r<   r(   ctx_id)r+   r(   request_datarN   r   dl
continuedlTfrag_resume_lenFfiletimefragment_filetimefragment_filename_sanitized)r)   r5   filesize_or_none	temp_namedownload)r   r0   Zfrag_urlr*   r,   rO   Zfragment_filenamefragment_info_dictrR   successrG   r   r   r   _download_fragmentp   s"    
zFragmentFD._download_fragmentc                 C   sj   | dsd S z| |d d\}}W n& tk
rL   | drFY d S  Y nX ||d< | }|  |S )NrU   rbr2   )r)   r?   FileNotFoundErrorrD   rA   )r   r0   ZdownZfrag_sanitizedfrag_contentr   r   r   _read_fragment   s    

zFragmentFD._read_fragmentc                 C   s`   z|d | |d   W 5 |  |r4| | | jddsT| t|d  |d= X d S )NZkeep_fragmentsFrU   dest_stream)_FragmentFD__do_ytdl_filerM   r5   r)   
try_remover   rK   flush)r   r0   r^   r   r   r   _append_fragment   s    

zFragmentFD._append_fragmentc              	   C   s  | dds6d|d  }|dd}|r:|d| 7 }nd}| d	| j d
|  | |d  t| j| jdddddd}| |d }d}| 	|}|dkrd}|
|dd | |rtjt| |d }| jdd}	|	rv|rv| | |ddk}
|d dko"|dk}|
s0|r|
r:dnd}| | d d |d< }d|krj|d= | | n>|	s|r| | d |d< }| | |d dkst| ||\}}|
||||d d S )Nr2   F%dtotal_fragsad_fragsr    (not including %d ad)unknown (live)[] Total fragments: r7   T)Z
noprogresstestZsleep_intervalZmax_sleep_intervalZsleep_interval_subtitleswbabr3   r<   rQ   r6   r<   z.ytdl file is corruptz2Inconsistent state of incomplete fragment downloadz#. Restarting from the beginning ...)rP   r`   r3   complete_frags_downloaded_bytes)
setdefaultr)   r   FD_NAMEZreport_destinationr   ydlr5   rW   rV   updatera   ospathisfiler   r@   rH   Zreport_warningrM   r>   r?   )r   r0   total_frags_strrg   rP   r3   Z	open_mode
resume_lenZytdl_file_existsrQ   Z
is_corruptZis_inconsistentmessager`   r   r   r   r.      sp    






z!FragmentFD._prepare_frag_downloadc                    sz    d } d   dd| d  d  d dt  d	< t| fd
d} d |  d	 S )Nrp   rf   rN   downloadingr<   r7   r3   )statusdownloaded_bytesr<   rJ   r7   r3   startedc                    sR  | d dkrd S s*  dr* d d< d k	rD|  dkrDd S   dd<   dd< jd< |  dpvd	}| d
i | d<  d sΈ d | d d   }|_|  d jd< n|  d | d dkrd  d7  < d  d<   j d<  d< jj d<  d< j	jd< 
 d S )Nr|   )r{   finishedrJ   rN   max_progressprogress_idxelapsedtotal_bytesr   r*   rY   r2   rp   r<   r   r}   Ztotal_bytes_estimater   speedeta)r)   r   poptotalrt   Zthread_reset
downloadedr   Zsmoothr   _hook_progress)sZfrag_total_bytesZestimated_sizer0   rN   r*   progressr   staterf   r   r   frag_progress_hook   s<    


z;FragmentFD._start_frag_download.<locals>.frag_progress_hookrP   )r)   timer   Zadd_progress_hook)r   r0   r*   ry   r   r   r   r   r/      s    
	'zFragmentFD._start_frag_downloadc                 C   s   |d    | |r*| | |d  t |d  }|d dk}|rZ| |d }n|d }|s|rx| |d  | d dS |r| |d |d  |d	}| j	d
dr|rt
t t|d t |f W 5 Q R X | |||d d||d|d|dd| dS )Nr`   r7   r~   r3   r4   rp   zThe downloaded file is emptyFrT   Z
updatetimeTr   rN   r   r   )r}   r   r7   r|   r   rN   r   r   )rA   ra   rb   r@   r   rV   report_errorZ
try_renamer)   r5   
contextlibsuppressrE   ru   utimer   )r   r0   r*   r   Zto_filer}   rS   r   r   r   _finish_frag_download  s@    


"	
z FragmentFD._finish_frag_downloadc                 C   s   d|krd|d< |d sBd|d  }| dd}|rF|d| 7 }nd}| d	| j d
|  | |d }||dd d S )Nr2   Fre   rf   rg   r   rh   ri   rj   rk   r7   ro   )r)   r   rr   rW   rt   )r   r0   rx   rg   r3   r   r   r   _prepare_external_frag_downloadB  s    z*FragmentFD._prepare_external_frag_downloadc                    s(   i fdd  fdd}|S )Nc                    s,   |  kr$j |   | <  |  S r   )rs   urlopenr-   rD   )r+   )
_key_cacher*   r   r   r   _get_keyY  s    z&FragmentFD.decrypter.<locals>._get_keyc                    s   |d krd S |  d}|r&|d dkr*|S | dpBtd| d }| dpb tdp`|d	 |d< j d
drz|S tt||d |S )Ndecrypt_infoZMETHODzAES-128ZIVz>8xqZmedia_sequenceZKEY)Zhls_aesuriURIrl   F)r)   structpackr   r5   r   r   )fragmentr^   r   Ziv)r   r*   r   r   r   decrypt_fragment^  s    

z.FragmentFD.decrypter.<locals>.decrypt_fragmentr   )r   r*   r   r   )r   r   r*   r   r   	decrypterV  s    zFragmentFD.decrypterc              
      sd  dg t |dkr&j|d S jdd}dkrF tt|d} fdd}G dd	 d	tjj	}t
d
krdd }ndd } fdd}g }	t|D ]F\}
\}}}|t| }|||
|||||}|	||f qd}|	D ]N\}}z6z|o||}W n tk
r6   d d< Y nX W 5 |jdd X q d s`|s`t|S )z
        @params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ...
                all args must be either tuple or list
        Tr   r   concurrent_fragment_downloads).r   is_livec                    s,   |d< | |d< j |||f| dS )Nr   r   )tpeinterrupt_trigger)download_and_append_fragments)idxr0   	fragmentsr*   r   r   kwargsr   r   r   r   thread_func~  s        zFFragmentFD.download_and_append_fragments_multiple.<locals>.thread_funcc                   @   s   e Zd Zdd ZdS )z?FragmentFD.download_and_append_fragments_multiple.<locals>.FTPEc                 S   s   d S r   r   )r   exc_typeexc_valexc_tbr   r   r   __exit__  s    zHFragmentFD.download_and_append_fragments_multiple.<locals>.FTPE.__exit__N)r   r   r   r   r   r   r   r   FTPE  s   r   ntc                 S   sD   z|  dW S  tk
r"    Y q  tjjk
r<   Y q Y q X q d S )Ng?)resultKeyboardInterrupt
concurrentfuturesTimeoutErrorfuturer   r   r   future_result  s    zHFragmentFD.download_and_append_fragments_multiple.<locals>.future_resultc                 S   s   |   S r   )r   r   r   r   r   r     s    c                 3   s    | D ]} d s q|V  qd S )Nr   r   )Zfgf)r   r   r   interrupt_trigger_iter  s    zQFragmentFD.download_and_append_fragments_multiple.<locals>.interrupt_trigger_iterwaitF)lenr   r5   r)   Z_prepare_multiline_statusanyr   r   r   ThreadPoolExecutorr   	enumeratemathceilZsubmitappendshutdownr   )r   r   r   max_workersr   r   r   r   r   Zspinsr   r0   r   r*   r   Zjobr   r   r   r   &download_and_append_fragments_multiplep  s:    

	z1FragmentFD.download_and_append_fragments_multiplec                 C   s   dS )NFr   )r   r   r   r   <lambda>      zFragmentFD.<lambda>c                 C   s   | S r   r   )contentr   r   r   r   r     r   )T)is_fatal	pack_funcfinish_funcr   r   c             	      s  j ddsdd fddfdd}	}
tj d	d
 dd
 }|d
kr, fdd}|ptj|}zV|||D ]D\}}} 	||d |	|
|
 | s W W 5 Q R  dS qW n< tk
r     jdddd |jdd  Y nX W 5 Q R X n|D ]z}d sD qz*|  |	|
|
 |d  }W n, tk
r   drY  q Y nX |s0 dS q0|d k	r҈ d |   d    S )NZskip_unavailable_fragmentsTc                 S   s   dS )NTr   )rG   r   r   r   r     r   z:FragmentFD.download_and_append_fragments.<locals>.<lambda>c                    s*  d sd S | d   d< d  d< t d}| d}|r\d|d |d	 d
 f |d< | dpnd
  fdd}tjd|D ]}z6| d d<  | d |dsW  d S W q ttfk
r } z||_W Y qW 5 d }~X Y q tk
r"   r Y qX qd S )Nr   r   r<   
last_errorr(   
byte_rangezbytes=%d-%dstartendr   ZRanger;   c                    s6   r||kr d    | || |  d< d S )Nr`   r   )rA   r   )r   r    r!   )r0   fatalr   r   r   r   error_callback  s    z[FragmentFD.download_and_append_fragments.<locals>.download_fragment.<locals>.error_callbackZfragment_retriesrJ   r+   rO   )	r   r)   r   r5   r[   r
   r   errorr   )r   r0   r,   r   r   retryr   )r*   r   r   r   )r0   r   r   r   download_fragment  s6    
    zCFragmentFD.download_and_append_fragments.<locals>.download_fragmentc                    sX   | r || | n< |d s2|d n"|d   d| d dS dS )Nr   zfragment not foundr`   z	fragment z not found, unable to continueFT)rd   r'   rA   r   )r^   r   r0   )r   r   r   r   r   append_fragment  s    zAFragmentFD.download_and_append_fragments.<locals>.append_fragmentr   r   r   c                    s&      }| | | | d |dfS )Nr   rU   )copyr)   )r   Zctx_copy)r0   r   r   r   r[     s    
zDFragmentFD.download_and_append_fragments.<locals>._download_fragment)rU   r<   Fz;Interrupted by user. Waiting for all threads to shutdown...)Zis_errortbr   r   r   r   r`   )r5   r)   r   r   r   r   r   r   maprt   r_   r   Z_finish_multiline_statusr   r   rK   rc   r   )r   r0   r   r*   r   r   r   r   r   r   r   r   r[   poolr   r   Zfrag_filenamer   r   )r0   r   r*   r   r   r   r   r   r     s`    !

  

  

z(FragmentFD.download_and_append_fragments)N)NN)r   r   r   __doc__r"   r'   r-   r1   ra   rH   rM   r[   r_   rd   r.   r/   r   r   r   r   r   r   r   r   r   r      s.   #

D<$A  r   )!concurrent.futuresr   r   rB   r   ru   r   r   commonr   httpr   Zaesr   r   compatr   Z
networkingr	   Znetworking.exceptionsr
   r   utilsr   r   r   r   Zutils.networkingr   Zutils.progressr   r   r   r   r   r   r   <module>   s"   