U
    $Pf¸                    @   sj  d dl mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZ ddlmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZ ddl[m\Z\ ddl]m^Z^m_Z_m`Z` dd	lambZb dd
lcmdZd ddlemfZf ddlgmhZhmiZimjZjmkZkmlZlmmZm ddlnmoZo e"dkrVd dlpZpG dd deqZrdS )    )absolute_importunicode_literalsN)ascii_letters   )compat_basestringcompat_cookiejarcompat_get_terminal_sizecompat_http_clientcompat_kwargscompat_numeric_typescompat_os_name
compat_strcompat_tokenize_tokenizecompat_urllib_errorcompat_urllib_request!compat_urllib_request_DataHandler)2age_restrictedargs_to_strContentTooShortErrordate_from_str	DateRangeDEFAULT_OUTTMPLdetermine_extdetermine_protocolDownloadErrorencode_compat_strencodeFilenameerror_to_compat_strexpand_pathExtractorErrorformat_bytesformatSecondsGeoRestrictedErrorint_or_noneISO3166Utilslocked_filemake_HTTPS_handlerMaxDownloadsReached
orderedSet	PagedListparse_filesizePerRequestProxyHandlerplatform_namePostProcessingErrorpreferredencodingprepend_extensionregister_socks_protocolsrender_tablereplace_extensionSameFileErrorsanitize_filenamesanitize_pathsanitize_urlsanitized_Requeststd_headersstr_or_nonesubtitles_filenameUnavailableVideoErrorurl_basenameversion_tuplewrite_json_filewrite_stringYoutubeDLCookieJarYoutubeDLCookieProcessorYoutubeDLHandlerYoutubeDLRedirectHandler)Cache)get_info_extractorgen_extractor_classes_LAZY_LOADER)PhantomJSwrapper)get_suitable_downloader)rtmpdump_version)FFmpegFixupM3u8PPFFmpegFixupM4aPPFFmpegFixupStretchedPPFFmpegMergerPPFFmpegPostProcessorget_postprocessor)__version__ntc                   @   sD  e Zd ZdZedZdZg Zg ZdZ	dZ
dZe ZdZd}ddZdd	 Zd
d Zdd Zdd Zdd Zdd Zdd Zd~ddZdddZdddZdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zdd)d*Zd+d, Z dd-d.Z!d/d0 Z"d1d2 Z#d3d4 Z$e%d5d6 Z&ddi ddfd7d8Z'd9d: Z(e(d;d< Z)d=d> Z*di fd?d@Z+dAdB Z,e(dCdD Z-dEdF Z.ddGdHZ/dIdJ Z0dKdL Z1dMdN Z2ddOdPZ3dQdR Z4dSdT Z5dUdV Z6dWdX Z7dYdZ Z8e%d[d\ Z9d]d^ Z:d_d` Z;dadb Z<dcdd Z=e%ddfdgZ>dhdi Z?djdk Z@dldm ZAddodpZBdqdr ZCdsdt ZDdudv ZEdwdx ZFdydz ZGd{d| ZHdS )	YoutubeDLaM1  YoutubeDL class.

    YoutubeDL objects are the ones responsible of downloading the
    actual video file and writing it to disk if the user has requested
    it, among some other tasks. In most cases there should be one per
    program. As, given a video URL, the downloader doesn't know how to
    extract all the needed information, task that InfoExtractors do, it
    has to pass the URL to one of them.

    For this, YoutubeDL objects have a method that allows
    InfoExtractors to be registered in a given order. When it is passed
    a URL, the YoutubeDL object handles it to the first InfoExtractor it
    finds that reports being able to handle it. The InfoExtractor extracts
    all the information about the video or videos the URL refers to, and
    YoutubeDL process the extracted information, possibly using a File
    Downloader to download the video.

    YoutubeDL objects accept a lot of parameters. In order not to saturate
    the object constructor with arguments, it receives a dictionary of
    options instead. These options are available through the params
    attribute for the InfoExtractors to use. The YoutubeDL also
    registers itself as the downloader in charge for the InfoExtractors
    that are added to it, so this is a "mutual registration".

    Available options:

    username:          Username for authentication purposes.
    password:          Password for authentication purposes.
    videopassword:     Password for accessing a video.
    ap_mso:            Adobe Pass multiple-system operator identifier.
    ap_username:       Multiple-system operator account username.
    ap_password:       Multiple-system operator account password.
    usenetrc:          Use netrc for authentication instead.
    verbose:           Print additional info to stdout.
    quiet:             Do not print messages to stdout.
    no_warnings:       Do not print out anything for warnings.
    forceurl:          Force printing final URL.
    forcetitle:        Force printing title.
    forceid:           Force printing ID.
    forcethumbnail:    Force printing thumbnail URL.
    forcedescription:  Force printing description.
    forcefilename:     Force printing final filename.
    forceduration:     Force printing duration.
    forcejson:         Force printing info_dict as JSON.
    dump_single_json:  Force printing the info_dict of the whole playlist
                       (or video) as a single JSON line.
    simulate:          Do not download the video files.
    format:            Video format code. See options.py for more information.
    outtmpl:           Template for output names.
    outtmpl_na_placeholder: Placeholder for unavailable meta fields.
    restrictfilenames: Do not allow "&" and spaces in file names
    ignoreerrors:      Do not stop on download errors.
    force_generic_extractor: Force downloader to use the generic extractor
    nooverwrites:      Prevent overwriting files.
    playliststart:     Playlist item to start at.
    playlistend:       Playlist item to end at.
    playlist_items:    Specific indices of playlist to download.
    playlistreverse:   Download playlist items in reverse order.
    playlistrandom:    Download playlist items in random order.
    matchtitle:        Download only matching titles.
    rejecttitle:       Reject downloads for matching titles.
    logger:            Log messages to a logging.Logger instance.
    logtostderr:       Log messages to stderr instead of stdout.
    writedescription:  Write the video description to a .description file
    writeinfojson:     Write the video description to a .info.json file
    writeannotations:  Write the video annotations to a .annotations.xml file
    writethumbnail:    Write the thumbnail image to a file
    write_all_thumbnails:  Write all thumbnail formats to files
    writesubtitles:    Write the video subtitles to a file
    writeautomaticsub: Write the automatically generated subtitles to a file
    allsubtitles:      Downloads all the subtitles of the video
                       (requires writesubtitles or writeautomaticsub)
    listsubtitles:     Lists all available subtitles for the video
    subtitlesformat:   The format code for subtitles
    subtitleslangs:    List of languages of the subtitles to download
    keepvideo:         Keep the video file after post-processing
    daterange:         A DateRange object, download only if the upload_date is in the range.
    skip_download:     Skip the actual download of the video file
    cachedir:          Location of the cache files in the filesystem.
                       False to disable filesystem cache.
    noplaylist:        Download single video instead of a playlist if in doubt.
    age_limit:         An integer representing the user's age in years.
                       Unsuitable videos for the given age are skipped.
    min_views:         An integer representing the minimum view count the video
                       must have in order to not be skipped.
                       Videos without view count information are always
                       downloaded. None for no limit.
    max_views:         An integer representing the maximum view count.
                       Videos that are more popular than that are not
                       downloaded.
                       Videos without view count information are always
                       downloaded. None for no limit.
    download_archive:  File name of a file where all downloads are recorded.
                       Videos already present in the file are not downloaded
                       again.
    cookiefile:        File name where cookies should be read from and dumped to.
    nocheckcertificate:Do not verify SSL certificates
    prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
                       At the moment, this is only supported by YouTube.
    proxy:             URL of the proxy server to use
    geo_verification_proxy:  URL of the proxy to use for IP address verification
                       on geo-restricted sites.
    socket_timeout:    Time to wait for unresponsive hosts, in seconds
    bidi_workaround:   Work around buggy terminals without bidirectional text
                       support, using fridibi
    debug_printtraffic:Print out sent and received HTTP traffic
    include_ads:       Download ads as well
    default_search:    Prepend this string if an input url is not valid.
                       'auto' for elaborate guessing
    encoding:          Use this encoding instead of the system-specified.
    extract_flat:      Do not resolve URLs, return the immediate result.
                       Pass in 'in_playlist' to only show this behavior for
                       playlist items.
    postprocessors:    A list of dictionaries, each with an entry
                       * key:  The name of the postprocessor. See
                               youtube_dl/postprocessor/__init__.py for a list.
                       as well as any further keyword arguments for the
                       postprocessor.
    progress_hooks:    A list of functions that get called on download
                       progress, with a dictionary with the entries
                       * status: One of "downloading", "error", or "finished".
                                 Check this first and ignore unknown values.

                       If status is one of "downloading", or "finished", the
                       following properties may also be present:
                       * filename: The final filename (always present)
                       * tmpfilename: The filename we're currently writing to
                       * downloaded_bytes: Bytes on disk
                       * total_bytes: Size of the whole file, None if unknown
                       * total_bytes_estimate: Guess of the eventual file size,
                                               None if unavailable.
                       * elapsed: The number of seconds since download started.
                       * eta: The estimated time in seconds, None if unknown
                       * speed: The download speed in bytes/second, None if
                                unknown
                       * fragment_index: The counter of the currently
                                         downloaded video fragment.
                       * fragment_count: The number of fragments (= individual
                                         files that will be merged)

                       Progress hooks are guaranteed to be called at least once
                       (with status "finished") if the download is successful.
    merge_output_format: Extension to use when merging formats.
    fixup:             Automatically correct known faults of the file.
                       One of:
                       - "never": do nothing
                       - "warn": only emit a warning
                       - "detect_or_warn": check whether we can do anything
                                           about it, warn otherwise (default)
    source_address:    Client-side IP address to bind to.
    call_home:         Boolean, true iff we are allowed to contact the
                       youtube-dl servers for debugging.
    sleep_interval:    Number of seconds to sleep before each download when
                       used alone or a lower bound of a range for randomized
                       sleep before each download (minimum possible number
                       of seconds to sleep) when used along with
                       max_sleep_interval.
    max_sleep_interval:Upper bound of a range for randomized sleep before each
                       download (maximum possible number of seconds to sleep).
                       Must only be used along with sleep_interval.
                       Actual sleep time will be a random float from range
                       [sleep_interval; max_sleep_interval].
    listformats:       Print an overview of available video formats and exit.
    list_thumbnails:   Print a table of all thumbnails and exit.
    match_filter:      A function that gets called with the info_dict of
                       every video.
                       If it returns a message, the video is ignored.
                       If it returns None, the video is downloaded.
                       match_filter_func in utils.py is one example for this.
    no_color:          Do not emit color codes in output.
    geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
                       HTTP header
    geo_bypass_country:
                       Two-letter ISO 3166-2 country code that will be used for
                       explicit geographic restriction bypassing via faking
                       X-Forwarded-For HTTP header
    geo_bypass_ip_block:
                       IP range in CIDR notation that will be used similarly to
                       geo_bypass_country

    The following options determine which downloader is picked:
    external_downloader: Executable of the external downloader to call.
                       None or unset for standard (built-in) downloader.
    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
                       if True, otherwise use ffmpeg/avconv if False, otherwise
                       use downloader suggested by extractor if None.

    The following parameters are not used by YoutubeDL itself, they are used by
    the downloader (see youtube_dl/downloader/common.py):
    nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
    noresizebuffer, retries, continuedl, noprogress, consoletitle,
    xattr_set_filesize, external_downloader_args, hls_use_mpegts,
    http_chunk_size.

    The following options are used by the post processors:
    prefer_ffmpeg:     If False, use avconv instead of ffmpeg if both are available,
                       otherwise prefer ffmpeg.
    ffmpeg_location:   Location of the ffmpeg/avconv binary; either the path
                       to the binary or its containing directory.
    postprocessor_args: A list of additional command-line arguments for the
                        postprocessor.

    The following options are used by the Youtube extractor:
    youtube_include_dash_manifest: If True (default), DASH manifests and related
                        data will be downloaded and processed by extractor.
                        You can reduce network I/O by disabling it if you don't
                        care about DASH.
    )widthheighttbrabrasrvbrfpsfilesizefilesize_approx	timestampZupload_yearZupload_monthZ
upload_dayduration
view_countZ
like_countZdislike_countZrepost_countZaverage_ratingZcomment_count	age_limit
start_timeZend_timeZchapter_numberZseason_numberZepisode_numberZtrack_numberZdisc_numberZrelease_yearplaylist_indexNr   Tc              
      s  |dkri }g  _ i  _g  _g  _d _d _tjtjg|	dd  _
tj _ddi _ j| t  _ fdd}|dd	d
r j	ddkr jd  jd< |ddd |ddd |ddd |	ddrzddl}| \}}t j}|dkr
g }ndt|g}ttj| jd}	ztjdg| f|	 _W n0 tk
rr   tjdddg| f|	 _Y nX t|d _W n@ tk
r }
 z |
jtjkr  d n W 5 d}
~
X Y nX tj!dkrt" dkr|	d ds  d! d" jd < t#|	d#t$r  d$  %  |r< &   '   j	d%g D ]:}t(|d& }t|}|d&= | ft)|} *| qJ j	d'g D ]} +| qt,  dS )(z6Create a FileDownloader object with the given options.Nr   ZlogtostderrFZnocheckcertificatec                    s*    j | d k	r& d||f  dS dS )Nz!%s is deprecated. Use %s instead.TF)paramsgetreport_warning)paramoptionZ
suggestionself 8/tmp/pip-unpacked-wheel-7zdooeg3/youtube_dl/YoutubeDL.pycheck_deprecatedm  s    
z,YoutubeDL.__init__.<locals>.check_deprecatedZcn_verification_proxyz--cn-verification-proxyz--geo-verification-proxyZgeo_verification_proxyautonumber_sizez--autonumber-sizezFoutput template with %(autonumber)0Nd, where N in the number of digits
autonumberz--auto-numberz%-o "%(autonumber)s-%(title)s.%(ext)s"Zusetitlez--titlez-o "%(title)s-%(id)s.%(ext)s"Zbidi_workaroundz-w)stdinstdoutstderrZbidivZfribidiz-czUTF-8rbzCould not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.win32)asciizANSI_X3.4-1968restrictfilenameszAssuming --restrict-filenames since file system encoding cannot encode all characters. Set the LC_ALL environment variable to fix this.TouttmplzParameter outtmpl is bytes, but should be a unicode string. Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.postprocessorskeyZprogress_hooks)-_ies_ies_instances_pps_progress_hooks_download_retcode_num_downloadssysrp   rq   rd   _screen_file	_err_filerc   updaterD   cacheptyopenptyr   columnsstrdict
subprocessPIPEPopen_output_processOSErrorosfdopen_output_channelerrnoENOENTre   platformgetfilesystemencoding
isinstancebytes_setup_openerprint_debug_headeradd_default_info_extractorsrP   r
   add_post_processoradd_progress_hookr0   )ri   rc   Z	auto_initrl   r   ZmasterZslaverT   Z
width_argsZ	sp_kwargsZoseZ
pp_def_rawZpp_classZpp_defppphrj   rh   rk   __init__Z  s     





zYoutubeDL.__init__c                    s`   dd t  D r\dgfddt  D  dg  fddD  }| dt|  d S )Nc                 S   s    g | ]\}}t d |r|qS )z^-[0-9A-Za-z_-]{10}$)rematch.0iarj   rj   rk   
<listcomp>  s   z.YoutubeDL.warn_if_short_id.<locals>.<listcomp>z
youtube-dlc                    s   g | ]\}}| kr|qS rj   rj   r   )idxsrj   rk   r     s      z--c                    s   g | ]} | qS rj   rj   r   r   )argvrj   rk   r     s     zULong argument string detected. Use -- to separate parameters and URLs, like this:
%s
)	enumeratere   r   )ri   r   Zcorrect_argvrj   )r   r   rk   warn_if_short_id  s"    zYoutubeDL.warn_if_short_idc                 C   s2   | j | t|ts.|| j| < ||  dS )z3Add an InfoExtractor object to the end of the list.N)ry   appendr   typerz   ie_keyset_downloaderri   ierj   rj   rk   add_info_extractor  s    
zYoutubeDL.add_info_extractorc                 C   s,   | j |}|dkr(t| }| | |S )z
        Get an instance of an IE with name ie_key, it will try to get one from
        the _ies list, if there's no instance it will create a new one and add
        it to the extractor list.
        N)rz   rd   rE   r   )ri   r   r   rj   rj   rk   rE     s
    

zYoutubeDL.get_info_extractorc                 C   s   t  D ]}| | qdS )zZ
        Add the InfoExtractors returned by gen_extractors to the end of the list
        N)rF   r   r   rj   rj   rk   r     s    
z%YoutubeDL.add_default_info_extractorsc                 C   s   | j | ||  dS )z3Add a PostProcessor object to the end of the chain.N)r{   r   r   )ri   r   rj   rj   rk   r     s    zYoutubeDL.add_post_processorc                 C   s   | j | dS )z>Add the progress hook (currently only for the file downloader)N)r|   r   )ri   r   rj   rj   rk   r     s    zYoutubeDL.add_progress_hookc                    s   t  ds|S t  dstt|ts*t|dd } jj|d d  jj	  d
 fddt|D }|d td  S )	Nr   r   
r   utf-8 c                 3   s   | ]} j  d V  qdS )r   N)r   readlinedecoder   _rh   rj   rk   	<genexpr>  s   z-YoutubeDL._bidi_workaround.<locals>.<genexpr>)hasattrAssertionErrorr   r   countr   ro   writeencodeflushjoinrangelen)ri   message
line_countresrj   rh   rk   _bidi_workaround  s    
zYoutubeDL._bidi_workaroundFc                 C   s   | j ||ddS )-Print message to stdout if not in quiet mode.T)check_quiet)	to_stdout)ri   r   skip_eolrj   rj   rk   	to_screen  s    zYoutubeDL.to_screenc                 C   s   t ||| jdd d S )Nencoding)outr   )r?   rc   rd   )ri   sr   rj   rj   rk   _write_string  s    zYoutubeDL._write_stringc                 C   s`   | j dr| j d | n>|r0| j dds\| |}ddg| }|| }| || j dS )r   loggerquietFr   r   N)rc   rd   debugr   r   r   )ri   r   r   r   
terminatoroutputrj   rj   rk   r     s    
zYoutubeDL.to_stdoutc                 C   sP   t |tst| jdr,| jd | n | |}|d }| || j dS )zPrint message to stderr.r   r   N)	r   r   r   rc   rd   errorr   r   r   )ri   r   r   rj   rj   rk   	to_stderr
  s    
zYoutubeDL.to_stderrc                 C   s\   | j ddsd S tdkr<tjj rXtjjt| ndt	j
krX| d| | j d S )NconsoletitleFrR   TERMz]0;%s)rc   rd   r   ctypeswindllkernel32ZGetConsoleWindowZSetConsoleTitleW	c_wchar_pr   environr   r   )ri   r   rj   rj   rk   to_console_title  s    
zYoutubeDL.to_console_titlec                 C   sH   | j ddsd S | j ddr$d S tdkrDdtjkrD| d| j d S )Nr   FsimulaterR   r   z[22;0trc   rd   r   r   r   r   r   rh   rj   rj   rk   save_console_title  s    zYoutubeDL.save_console_titlec                 C   sH   | j ddsd S | j ddr$d S tdkrDdtjkrD| d| j d S )Nr   Fr   rR   r   z[23;0tr   rh   rj   rj   rk   restore_console_title(  s    zYoutubeDL.restore_console_titlec                 C   s   |    | S N)r   rh   rj   rj   rk   	__enter__1  s    zYoutubeDL.__enter__c                 G   s,   |    | jdd k	r(| jjddd d S )N
cookiefileTignore_discardignore_expires)r   rc   rd   	cookiejarsave)ri   argsrj   rj   rk   __exit__5  s    zYoutubeDL.__exit__c                 C   s  |dk	r|  | | jdr|dkrt d rd}tt d drxt d jd rx|dtjt d j 7 }|t	t
 7 }ntt }d|}|  | | jddst d rtt d drt d jd rt d j}nt }t||d| _dS )	aI  Determine action to take when a download problem appears.

        Depending on if the downloader has been configured to ignore
        download errors or not, this method may throw an exception or
        not when errors are found, after printing the message.

        tb, if given, is additional traceback information.
        Nverboser   r   r   exc_infoignoreerrorsF)r   rc   rd   r   r   r   r   	tracebackformat_exceptionr   
format_excformat_listextract_stackr   r}   )ri   r   tbZtb_datar   rj   rj   rk   trouble;  s$    	
$

0
zYoutubeDL.troublec                 C   st   | j ddk	r"| j d | nN| j dr2dS | j dsV| j rVtdkrVd}nd}d||f }| | dS )	z
        Print the message to stderr, it will be prefixed with 'WARNING:'
        If stderr is a tty file the 'WARNING:' will be colored
        r   NZno_warningsno_colorrR   z[0;33mWARNING:[0mzWARNING:%s %s)rc   rd   warningr   isattyr   r   )ri   r   _msg_headerZwarning_messagerj   rj   rk   re   Y  s    zYoutubeDL.report_warningc                 C   sD   | j ds$| j r$tdkr$d}nd}d||f }| || dS )z
        Do the same as trouble, but prefixes the message with 'ERROR:', colored
        in red if stderr is a tty file.
        r   rR   z[0;31mERROR:[0mzERROR:r   N)rc   rd   r   r   r   r   )ri   r   r   r   error_messagerj   rj   rk   report_errorj  s
    zYoutubeDL.report_errorc                 C   s6   z|  d|  W n tk
r0   |  d Y nX dS )z.Report file has already been fully downloaded.z)[download] %s has already been downloadedz/[download] The file has already been downloadedN)r   UnicodeEncodeError)ri   	file_namerj   rj   rk   report_file_already_downloadedv  s    z(YoutubeDL.report_file_already_downloadedc              
      sZ  zt |}tt |d< jd}|dkr4d}jddd j |d< |ddkr|d	r|d
rd|d	 |d
 f |d< n6|d
rd|d
  |d< n|d	rd|d	  |d< fdd t  fdd| D }tfdd|}jdt	}t
t|d |d}d}t||}|rRt|d||d  |}jD ].}||krXd}	t|	|d||}qXddd tdD }
|dd |
d!d"|
}t||
d| }tjd#k rtjd$krt|d%t }t|W S  tk
rT } z.d&t| d' tt  d(  W Y dS d}~X Y nX dS ))zGenerate the output filename.epochrm   N   Zautonumber_startr   rn   
resolutionrT   rU   %dx%d%sp%dx?c                    s(   t t| jd| dkp"| ddS )Nru   idZ_id)Z
restrictedZis_id)r4   r   rc   rd   endswith)kvrh   rj   rk   <lambda>  s   
z,YoutubeDL.prepare_filename.<locals>.<lambda>c                 3   sF   | ]>\}}|d k	rt |tttfs|t |tr2|n ||fV  qd S r   )r   listtupler   r   r   r	  r
  )sanitizerj   rk   r     s    z-YoutubeDL.prepare_filename.<locals>.<genexpr>c                      s    j ddS )NZouttmpl_na_placeholderZNA)rc   rd   rj   rh   rj   rk   r        rv   	n_entries)rb   rn   z0(?<!%)%\((?P<field>autonumber|playlist_index)\)sz
%%(\1)0%ddfielda  (?x)
                        (?<!%)
                        %
                        \({0}\)  # mapping key
                        (?:[#0\-+ ]+)?  # conversion flags (optional)
                        (?:\d+)?  # minimum field width (optional)
                        (?:\.\d+)?  # precision (optional)
                        [hlL]?  # length modifier (optional)
                        [diouxXeEfFgGcrs%]  # conversion type
                    z%({0})sr   c                 S   s   g | ]}t tqS rj   )randomchoicer   r   rj   rj   rk   r     s     z.YoutubeDL.prepare_filename.<locals>.<listcomp>    z%%z%{0}%z$$z${0}$)   r   rs   TzError in output template: z (encoding: )) r   inttimerc   rd   r~   itemscollectionsdefaultdictr   r   r   r   searchsubgroup_NUMERIC_FIELDSformatr   r   replacer   r   version_infor   r   r   r.   r5   
ValueErrorr   repr)ri   	info_dictZtemplate_dictrm   rv   Zfield_size_compat_mapZFIELD_SIZE_COMPAT_REZmobjnumeric_fieldZ	FORMAT_REsepfilenameerrrj   )r  ri   rk   prepare_filename}  s`    

	


  
$zYoutubeDL.prepare_filenamec                 C   s  | d| dd}d|kr|d }| j dd}|rZt||tjsZd| d | d S | j dd}|rt||tjrd| d	 | d S | d
}|dk	r| j dt }||krdt| |f S | d}	|	dk	r>| j d}
|
dk	r|	|
k rd||	|
f S | j d}|dk	r>|	|kr>d||	|f S t| d| j dr`d| S | 	|rtd| S |s| j d}|dk	r||}|dk	r|S dS )z0 Returns None iff the file should be downloaded titler  video
matchtitleF"z" title did not match pattern "rejecttitlez " title matched reject pattern "upload_dateNZ	daterangez!%s upload date is not in range %sr_   	min_viewszBSkipping %s, because it has not reached minimum view count (%d/%d)	max_viewszCSkipping %s, because it has exceeded the maximum view count (%d/%d)r`   z*Skipping "%s" because it is age restrictedz'%s has already been recorded in archivematch_filter)
rd   rc   r   r  
IGNORECASEr   r   	isoformatr   in_download_archive)ri   r&  
incompleteZvideo_titler,  r.  r0  dateZ	dateRanger_   r2  r3  r4  retrj   rj   rk   _match_entry  sF    




zYoutubeDL._match_entryc                 C   s"   |  D ]\}}| || qdS )z=Set the keys from extra_info in info dict if they are missingN)r  
setdefault)r&  
extra_inforx   valuerj   rj   rk   add_extra_info   s    zYoutubeDL.add_extra_infoc           	      C   s   |s|rd}|r|  |g}n| j}|D ]D}||s8q(|  | }| sX| d | |||||  S | d|  dS )a  
        Return a list with a dictionary for each video extracted.

        Arguments:
        url -- URL to extract

        Keyword arguments:
        download -- whether to download videos during extraction
        ie_key -- extractor key hint
        extra_info -- dictionary containing the extra values to add to each result
        process -- whether to resolve all unresolved references (URLs, playlist items),
            must be True for download to work.
        force_generic_extractor -- force using the generic extractor
        Genericz^The program functionality for this site has been marked as broken, and will probably not work.z$no suitable InfoExtractor for URL %sN)rE   ry   suitabler   Zworkingre   _YoutubeDL__extract_infor   )	ri   urldownloadr   r=  processforce_generic_extractorZiesr   rj   rj   rk   extract_info  s    

zYoutubeDL.extract_infoc                    s    fdd}|S )Nc              
      s
  z | f||W S  t k
rn } z>|j}|jrL|ddttj|j 7 }|d7 }| | W 5 d }~X Y n tk
r } z| t	||
  W 5 d }~X Y nd tk
r    Y nP tk
r } z0| jddr| jt|tt d n W 5 d }~X Y nX d S )Nz
This video is available in %s., zL
You might want to use a VPN or a proxy server (with --proxy) to workaround.r   F)r   )r"   msgZ	countriesr   mapr$   Z
short2fullr   r   r   Zformat_tracebackr'   	Exceptionrc   rd   r   r   r   r   )ri   r   kwargserI  funcrj   rk   wrapper-  s$    $z9YoutubeDL.__handle_extraction_exceptions.<locals>.wrapperrj   )rO  rP  rj   rN  rk   Z__handle_extraction_exceptions,  s    z(YoutubeDL.__handle_extraction_exceptionsc                 C   sR   | |}|d krd S t|tr*d|d}| ||| |rJ| |||S |S d S )Ncompat_list)_typeentries)extractr   r  add_default_extra_infoprocess_ie_result)ri   rC  r   rD  r=  rE  	ie_resultrj   rj   rk   Z__extract_infoB  s    

zYoutubeDL.__extract_infoc                 C   s$   |  ||j|t|| d d S )N	extractorwebpage_urlwebpage_url_basenameextractor_key)r?  ZIE_NAMEr<   r   )ri   rW  r   rC  rj   rj   rk   rU  S  s    z YoutubeDL.add_default_extra_infoc              	      s4   dd}|dkrdtd d< j dd}|dkrBdksJ|d	krdjd	d
 S |dkr jdS |dkrjd  ddS |dkrBjd  dddd}|s|S tdd 	 D }dD ]}||kr||= q|
 }	|	| |	 ddkr2d|	d< j|	dS |dkrԈd }
|
jkrd d p| d dS  jd7  _j|
 zW S  jd8  _jsЈj  X n\|dkr$d d  fdd  fd d!d" D d"< S td#| dS )$z
        Take the result of the ie(may be modified) and resolve all unresolved
        references (URLs, playlist items).

        It will also download the videos if 'download'.
        Returns the resolved ie_result.
        rR  r-  )rC  url_transparentrC  extract_flatFZin_playlistplaylistTr8  rD  r   )r   r=  r]  )r   r=  rD  rE  c                 s   s"   | ]\}}|d k	r||fV  qd S r   rj   r  rj   rj   rk   r     s     z.YoutubeDL.process_ie_result.<locals>.<genexpr>)rR  rC  r  rY  r\  r   rD  r=  )r_  Zmulti_videorZ  z3[download] Skipping already downloaded playlist: %sr,  r  Nr   rQ  zCExtractor %s returned a compat_list result. It needs to be updated.rY  c                    s.    |  d  d t d  d d | S )NrY  rZ  r\  rX  )r?  r<   )r)rW  ri   rj   rk   _fixup  s    
	z+YoutubeDL.process_ie_result.<locals>._fixupc                    s   g | ]}  |qS rj   rV  )r   rc  )rd  rD  r=  ri   rj   rk   r     s   z/YoutubeDL.process_ie_result.<locals>.<listcomp>rS  zInvalid result type: %s)rd   r6   rc   _YoutubeDL__forced_printingsr+  r?  process_video_resultrG  r   r  copyr   rV  _playlist_urlsr   _playlist_leveladdclear_YoutubeDL__process_playlistre   rK  )ri   rW  rD  r=  Zresult_typer^  infoZforce_propertiesfZ
new_resultrZ  rj   )rd  rD  r=  rW  ri   rk   rV  [  s     

   
  


zYoutubeDL.process_ie_resultc                    s    dp  dd  g }j ddd }j d}|dkrPd }j d}d |d k	r|d	d
 }t|| d }fdd}	 fdd}
t|trt|}r|	|}n||| }t|}d d ||f  nt|trHr*g }D ]}||	|d | q
n|	||}t|}|
| nDrj|	tt
|dt}ntt
|||}t|}|
| j ddr|d d d }j ddrt|   d}t|dD ]\}}d||f  |r ||d< |  d  d  d  dr6|d  n||  d  d t d  d d}j|dd}|d k	rd|  q؈|||}|| q| d< d    S )!Nr,  r  z#[download] Downloading playlist: %splayliststartr   playlistendZplaylist_itemsc                 s   s\   |  dD ]L}d|krL| d\}}tt|t|d D ]}t|V  q:q
t|V  q
d S )N,-r   )splitr   r  )r!  Zstring_segmentstartenditemrj   rj   rk   iter_playlistitems  s    z8YoutubeDL.__process_playlist.<locals>.iter_playlistitemsrS  c                    s   t   fddD S )Nc                    s6   g | ].} |d    kr"k rn q |d   qS )r   rj   r   list_ie_entriesnum_entriesrj   rk   r     s
     zTYoutubeDL.__process_playlist.<locals>.make_playlistitems_entries.<locals>.<listcomp>)r   )r{  )playlistitemsrz  rk   make_playlistitems_entries  s    z@YoutubeDL.__process_playlist.<locals>.make_playlistitems_entriesc                    s    d d | f  d S )Nz'[%s] playlist %s: Downloading %d videosrY  )r   )r|  )rW  r_  ri   rj   rk   report_download  s
    z5YoutubeDL.__process_playlist.<locals>.report_downloadzA[%s] playlist %s: Collected %d video ids (downloading %d of them)rY  r   ZplaylistreverseFZplaylistrandom__x_forwarded_for_ipz%[download] Downloading video %s of %sZuploaderZuploader_idrZ  r\  )r  r_  Zplaylist_idZplaylist_titleZplaylist_uploaderZplaylist_uploader_idrb   rY  rZ  r[  r\  Tr`  [download] z,[download] Finished downloading playlist: %s)rd   r   rc   r(   r   r  r   r)   extendZgetslice	itertoolsislicemaxr  shuffler   r<   r;  "_YoutubeDL__process_iterable_entryr   )ri   rW  rD  Zplaylist_resultsrp  rq  Zplaylistitems_strry  Z
ie_entriesr~  r  Zn_all_entriesrS  r  rx  Zx_forwarded_forr   entryextrareasonZentry_resultrj   )rW  r_  r}  ri   rk   Z__process_playlist  s    

  
  
  



zYoutubeDL.__process_playlistc                 C   s   | j |||dS )Nrb  re  )ri   r  rD  r=  rj   rj   rk   Z__process_iterable_entry2  s
      z"YoutubeDL.__process_iterable_entryc              	      sv  t jt jt jt jt jt jd}tdd	t
tj|  }||rztd W nZ tk
r   td  dkrtdd   dkrtdd|f Y nX |d sPt jd	d
 dd
 dd
 d}tdd	t
tj|  }||rPd |d drLfdd
nsbtd|  fdd}|S )zG Returns a function to filter the formats according to the filter_spec )<z<=>z>==z!=z(?x)\s*
            (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)
            \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
            (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
            $
            |r>  NBz+Invalid value %r in format specification %ropc                 S   s
   |  |S r   )
startswithattrr>  rj   rj   rk   r  Y  r  z0YoutubeDL._build_format_filter.<locals>.<lambda>c                 S   s
   |  |S r   )r  r  rj   rj   rk   r  Z  r  c                 S   s   || kS r   rj   r  rj   rj   rk   r  [  r  )r  z^=z$=z*=z(?x)
                \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id|language)
                \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?
                \s*(?P<value>[a-zA-Z0-9._-]+)
                \s*$
                Znegationc                    s    | | S r   rj   r  )str_oprj   rk   r  h  r  zInvalid filter specification %rc                    s,   |  d}|d kr"dS | S )Nrx   Znone_inclusive)rd   r  )ro  Zactual_value)comparison_valuemr  rj   rk   _filtero  s    
z/YoutubeDL._build_format_filter.<locals>._filter)operatorltlegtgeeqner   compiler   rJ  escapekeysr  r  r  r$  r*   )ri   Zfilter_specZ	OPERATORSZoperator_rexZSTR_OPERATORSZstr_operator_rexr  rj   )r  r  r  r  rk   _build_format_filter7  s\    
 


zYoutubeDL._build_format_filterc                    s>   fdd  fdd}ddg}| r4|   d|S )Nc                     s   t  } | jo|  S r   )rN   	available	can_merge)mergerrh   rj   rk   r  x  s    z1YoutubeDL._default_format_spec.<locals>.can_mergec                      sL   j ddrdS sdS j dtdkr0dS dr>dS   sHdS dS )Nr   Frv   rt  TZis_live)rc   rd   r   rj   r  rD  r&  ri   rj   rk   prefer_best|  s    
z3YoutubeDL._default_format_spec.<locals>.prefer_bestzbestvideo+bestaudiobest/)reverser   )ri   r&  rD  r  Zreq_format_listrj   r  rk   _default_format_specv  s    zYoutubeDL._default_format_specc              	      s   fdd
ddddt ddd	d
g dd dd }d 
fdd		fddtd}zt|t|j}W n( tj	k
r   
ddt
fY nX G dd dt}t||}|S )Nc                    s   d |  d|d  }t|S )Nz,Invalid format specification: {0}
	{1}
	{2}^ r   )r!  SyntaxError)noterv  r   format_specrj   rk   syntax_error  s      
z5YoutubeDL.build_format_selector.<locals>.syntax_error	PICKFIRSTMERGESINGLEGROUPFormatSelectorr   selectorfiltersc                 S   sF   g }| D ]8\}}}}}|t jkr6|dkr6d|  S || qd S )N]r   )tokenizeOPr   r   )tokensZfilter_partsr   stringrv  r   rj   rj   rk   _parse_filter  s
    z6YoutubeDL.build_format_selector.<locals>._parse_filterc                 s   s&  d}d\}}}}| D ]\}}}}	}
|t jkr|dkr|rNt j||||fV  d }||||	|
fV  | D ]4\}}}}	}
||||	|
fV  |t jkrb|dkrb qqbq|t jkr||kr|rt j||||fV  d }||||	|
fV  q|t jt jt jfkr|s |}|}|	}q||7 }q|r"t j||||fV  d S )N)r  +rs  (r  )NNNN[r  )r  r  NAMENUMBER)r  ZALLOWED_OPSZlast_stringZ
last_startZlast_end	last_liner   r  rv  rw  linerj   rj   rk   _remove_unused_ops  s2    
z;YoutubeDL.build_format_selector.<locals>._remove_unused_opsFc                    s  g }d }| D ]\}}}}	}	|t tdd kr0qq|tjtjfkrN |g }q|tjkr|dkrx|sn|    qq|r|dkr|    qq|r|dkr|    qq|dkr|sʈd||| d }q|dkr|sd||}
| dd	} |
|fg }n|d
krH|s2 dg }| }|j| n|dkr||rbd|| dd} |g }nb|dkr|rd||}| dd}|r|sd| ||fg }nd||q|tj	kr qq|r|| |S )NENCODINGr  )r  rs  rs  z!"," must follow a format selectorr  z!"/" must follow a format selectorT)inside_choicer  r  r  zUnexpected "(")inside_groupr  zUnexpected "+")inside_mergez("+" must be between two format selectorszOperator not recognized: "{0}")
getattrr  r  r  r  restore_last_tokenr   r  r!  	ENDMARKER)r  r  r  r  	selectorsZcurrent_selectorr   r  rv  r   Zfirst_choiceZsecond_choiceZformat_filterr  video_selectoraudio_selector)r  r  r  r  r  r  _parse_format_selectionr  rj   rk   r    sj    














z@YoutubeDL.build_format_selector.<locals>._parse_format_selectionc                    s   t | tr,fdd| D fddS | jkrB| jn| j	krnfdd| jD fddnT| j
kr| jfddn6| jkrfdd	 t| j\ fd
dfdd| jD fdd}|S )Nc                    s   g | ]} |qS rj   rj   r   r   _build_selector_functionrj   rk   r     s     zUYoutubeDL.build_format_selector.<locals>._build_selector_function.<locals>.<listcomp>c                 3   s"    D ]}|| D ]
}|V  qqd S r   rj   )ctxro  r!  fsrj   rk   selector_function   s    z\YoutubeDL.build_format_selector.<locals>._build_selector_function.<locals>.selector_functionc                    s   g | ]} |qS rj   rj   r  r  rj   rk   r     s     c                    s&    D ]}t || }|r|  S qg S r   )r  )r  ro  Zpicked_formatsr  rj   rk   r  
  s
    
c           
   	   3   sr  t | d }|sd S  dkr0|D ]
}|V  q n> dkrz dkrDdnd}dd |D }|rf|| V  n| d	 rx|| V  n d
krdd |D }|r|d V  nΈ dkrdd |D }|r|d V  n dkrdd |D }|r|d V  n dkrdd |D }|rn|d V  nXdddddddddg	} |krD fdd}n fdd}t t||}	|	rn|	d V  d S )Nformatsall)r  worstNr  r   rr  c                 S   s,   g | ]$}| d dkr| ddkr|qS )vcodecnoneacodecrd   r   ro  rj   rj   rk   r     s    zpYoutubeDL.build_format_selector.<locals>._build_selector_function.<locals>.selector_function.<locals>.<listcomp>incomplete_formatsZ	bestaudioc                 S   s   g | ]}| d dkr|qS r  r  r  r  rj   rj   rk   r   '  s   Z
worstaudioc                 S   s   g | ]}| d dkr|qS r  r  r  rj   rj   rk   r   -  s   Z	bestvideoc                 S   s   g | ]}| d dkr|qS r  r  r  r  rj   rj   rk   r   3  s   Z
worstvideoc                 S   s   g | ]}| d dkr|qS r  r  r  rj   rj   rk   r   9  s   mp4ZflvwebmZ3gpm4amp3ZoggZaacZwavc                    s   | d  kS Nextrj   ro  r  rj   rk   r  A  r  znYoutubeDL.build_format_selector.<locals>._build_selector_function.<locals>.selector_function.<locals>.<lambda>c                    s   | d  kS )N	format_idrj   r  r  rj   rk   r  C  r  )r  filter)
r  r  ro  Z
format_idxZaudiovideo_formatsZaudio_formatsZvideo_formats
extensionsZfilter_fmatchesr  rj   rk   r    s\    

c                    sL  dd | D \}}| d  ddkr: d||f  d S | d  ddkrt| d  ddkrt d	||f  d S  j d
d kr| d d n jd
 }| d| d  d| d  df d| d  d| d  df | d  d| d  d| d  d| d  d| d  d| d  d| d  d| d  d| d  d|dS )Nc                 S   s   g | ]}|d  qS )r  rj   r  rj   rj   rk   r   I  s     zeYoutubeDL.build_format_selector.<locals>._build_selector_function.<locals>._merge.<locals>.<listcomp>r   r  r  z=The first format must contain the video, try using "-f %s+%s"r  r   zHBoth formats %s and %s are video-only, you must specify "-f video+audio"merge_output_formatr  z%s+%sr!  r  rT   rU   r  rZ   rY   stretched_ratiorW   )requested_formatsr!  r  rT   rU   r  rZ   r  rY   r  r  rW   r  )rd   r   rc   )Zformats_infoZformat_1Zformat_2Z
output_extrh   rj   rk   _mergeH  sF    $zQYoutubeDL.build_format_selector.<locals>._build_selector_function.<locals>._mergec                 3   s4   t t| t| D ]} |V  q d S r   )r  productrh  deepcopy)r  pair)r  r  r  rj   rk   r  n  s
     c                    s   g | ]}  |qS rj   )r  r  rh   rj   rk   r   s  s     c                    s2   t | } D ]}tt||d |d< q|S )Nr  )rh  r  r  r  )r  Zctx_copyr  )r  r  rj   rk   final_selectoru  s    
zYYoutubeDL.build_format_selector.<locals>._build_selector_function.<locals>.final_selector)r   r  r   r  rJ  r  )r  r  )r  r  r  r  r  ri   )r  r  r  r  r  r  r  rk   r    s&    



4
$zAYoutubeDL.build_format_selector.<locals>._build_selector_functionr   z/Missing closing/opening brackets or parenthesisr   c                   @   s0   e Zd Zdd Zdd Zdd ZeZdd Zd	S )
z6YoutubeDL.build_format_selector.<locals>.TokenIteratorc                 S   s   || _ d| _d S )Nr   )r  counter)ri   r  rj   rj   rk   r     s    z?YoutubeDL.build_format_selector.<locals>.TokenIterator.__init__c                 S   s   | S r   rj   rh   rj   rj   rk   __iter__  s    z?YoutubeDL.build_format_selector.<locals>.TokenIterator.__iter__c                 S   s4   | j t| jkrt | j| j  }|  j d7  _ |S Nr   )r  r   r  StopIteration)ri   r>  rj   rj   rk   __next__  s
    z?YoutubeDL.build_format_selector.<locals>.TokenIterator.__next__c                 S   s   |  j d8  _ d S r  )r  rh   rj   rj   rk   r    s    zIYoutubeDL.build_format_selector.<locals>.TokenIterator.restore_last_tokenN)__name__
__module____qualname__r   r  r  nextr  rj   rj   rj   rk   TokenIterator  s
   r   )FFF)r  
namedtupleioBytesIOr   r  r   r   r  
TokenErrorr   objectiter)ri   r  r  streamr  r   Zparsed_selectorrj   )r  r  r  r  r  r  r  r  r  ri   r  rk   build_format_selector  s&    : zYoutubeDL.build_format_selectorc                 C   sX   t  }|d}|r || | |}|r6||d< d|krT|d}|rT||d< |S )Nhttp_headersCookiezX-Forwarded-Forr  )r8   rh  rd   r   _calc_cookies)ri   r&  r   add_headerscookiesZx_forwarded_for_iprj   rj   rk   _calc_headers  s    



zYoutubeDL._calc_headersc                 C   s"   t |d }| j| |dS )NrC  r
  )r7   r   add_cookie_header
get_header)ri   r&  prrj   rj   rk   r    s    zYoutubeDL._calc_cookiesc                     sl  | dddkstd|kr$tdd|kr4tdfdd  fd	d
 fdd}|d || d|krd |d< d |d< | d}|d kr| d}|rd|ig |d< }|r8|jdd d t|D ]b\}}t|d |d< | dr| drd|d |d f |d< | dd krd| |d< qԈj drT| d S | d}|rrt||d< n|r|d d |d< d|krd|kr|d |d< dD ]j\}}	| |	d kr| |d k	rz"tj	|| }
|

d||	< W n tttfk
r   Y nX qdD ]B}| d | d k	r| |sd!| |d |  f ||< qd"D ]v}| |}|rd| D ]V\}}|D ]F}| drt|d |d< | d#d krt|d  |d#< qqqd| d$}| d%}j d&d'r2d$|kr|d |d( |d |d% d S |d |||d)< | d*d kr`|g}n|d* }|svtd+fd,d-}tt||}i }t|D ]\}}|d. || t|d |d< | d.st||d.< ntd/d0|d. |d.< |d. }||krg ||< || | q| D ]<\}}t|d1kr,t|D ]\}}d2||f |d.< qJq,t|D ]\}}| d3d krd4j|d. || d5d k	rd6|d5 nd7d8|d3< | d#d krt|d  |d#< | d9d krt||d9< | }||  ||d:< qrd;|kr:|d;= |d< |k	rP||d*< j d=rl!| d S j d3}|d krj"||d>}j d?r#d@|  $|}t%dAdB |D pt%dCdB |D }||dD}t||}|stdEdFdG|rZt|d1kr2&dH|d t|f  |D ]"}t'|}|| (| q6||d  |S )INrR  r-  r  z&Missing "id" field in extractor resultr,  z)Missing "title" field in extractor resultc                    s     d| ||f  d S )NzL"%s" field is not %s - forcing %s conversion, there is an error in extractor)re   )r  Z	field_not
conversionrh   rj   rk   report_force_conversion  s
    z?YoutubeDL.process_video_result.<locals>.report_force_conversionc                    s<   |  |}|d kst|tr d S  |dd t|| |< d S )Nza stringr  )rd   r   r   )rn  Zstring_fieldr  )r  rj   rk   sanitize_string_field  s
    
z=YoutubeDL.process_video_result.<locals>.sanitize_string_fieldc                    sF   j D ]:}| |}|d kst|tr(q |dd t|| |< qd S )Nnumericr  )r   rd   r   r   r#   )rn  r'  r  )r  ri   rj   rk   sanitize_numeric_fields  s    

z?YoutubeDL.process_video_result.<locals>.sanitize_numeric_fieldsr_  rb   
thumbnails	thumbnailrC  c                 S   st   |  dd k	r|  dnd|  dd k	r2|  dnd|  dd k	rL|  dnd|  dd k	rf|  dnd|  dfS )N
preferencerr  rT   rU   r  r   rC  r  )trj   rj   rk   r    s     z0YoutubeDL.process_video_result.<locals>.<lambda>)rx   rT   rU   r  r  z%dlist_thumbnailsrr  Z
display_id))r]   r1  )Zrelease_timestampZrelease_datez%Y%m%d)ZchapterZseasonZepisodez	%s_numberz%s %d)	subtitlesautomatic_captionsr  r  r  ZlistsubtitlesFzautomatic captionsrequested_subtitlesr  zNo video formats found!c                    s4   |  d}|sd dS t|tr0 | d dS )NrC  zQ"url" field is missing or empty - skipping format, there is an error in extractorFT)rd   re   r   r   )ro  rC  )r  ri   rj   rk   is_wellformed'  s    


z5YoutubeDL.process_video_result.<locals>.is_wellformedr  z[\s,/+\[\]()]r   r   z%s-%dr!  z{id} - {res}{note}format_notez ({0})r   )r  r   r  protocolr	  r  r   Zlistformatsra  r   z [debug] Default format spec: %s
c                 s   s*   | ]"}| d dko | ddkV  qdS r  r  r  Nr  r  rj   rj   rk   r     s     z1YoutubeDL.process_video_result.<locals>.<genexpr>c                 s   s*   | ]"}| d dko | ddkV  qdS r"  r  r  rj   rj   rk   r     s     )r  r  zrequested format not availableT)expectedz*[info] %s: downloading video in %s formats))rd   r   r   sortr   r6   rc   r  datetimeutcfromtimestampstrftimer$  OverflowErrorr   
capitalizer  r   lowerlist_subtitlesprocess_subtitlesr  r  r   r   r  r   r   r!  format_resolutionr   rh  r   r  list_formatsr  r   r  r  r   r   process_info) ri   r&  rD  r  r  r  r   r  Zts_keyZdate_keyr1  r  Zcc_kindccr   ZsubtitleZsubtitle_formatr  r  r  r  Zformats_dictr!  r  Zambiguous_formatsZfull_format_infoZ
req_formatZformat_selectorr  r  Zformats_to_downloadnew_inforj   )r  r  ri   rk   rg    s   




 
  
 


    

 






zYoutubeDL.process_video_resultc                    s  i }|r| j dr|| |rP| j drP| D ]\}}||kr6|||< q6| j dsh| j drl|spdS | j ddr| }n>| j ddr| j d}n"d|krdg}nt| d g}| j d	d
}|r|dng }	i }
|D ]}||}|dkr| d||f  q|	D ]H  d
kr:|d } qtt fdd|}|r|d } qq|d }| d|||d f  ||
|< q|
S )z/Select the requested subtitles and their formatwritesubtitleswriteautomaticsubNZallsubtitlesFZsubtitleslangsenr   Zsubtitlesformatr  r  z!%s subtitles not available for %srr  c                    s   | d  kS r  rj   r  r  rj   rk   r    r  z-YoutubeDL.process_subtitles.<locals>.<lambda>z@No subtitle format found matching "%s" for language %s, using %sr  )	rc   rd   r   r  r  r  ru  re   r  )ri   video_idZnormal_subtitlesr  Zavailable_subslangZcap_infoZrequested_langsZformats_queryZformats_preferencesubsr  ro  r  rj   r5  rk   r,    sX    








zYoutubeDL.process_subtitlesc                    s   fdd}fdd}|d |d j ddr sd	d k	rxd	 D ]}|d
 |dd  qVnd
 dd  |d |d j ddr|d k	r| j ddrdd k	rtd  |d j ddrt d S )Nc                    s6   j d|  dr2 r$| d k	r2|   d S Nzforce%sFrc   rd   r   r  r8  r&  ri   rj   rk   print_mandatory  s    z5YoutubeDL.__forced_printings.<locals>.print_mandatoryc                    s2   j d|  dr. | d k	r. |   d S r9  r:  r;  )r&  ri   rj   rk   print_optional  s    z4YoutubeDL.__forced_printings.<locals>.print_optionalr,  r  ZforceurlFr  rC  Z	play_pathr   r  descriptionZforcefilenameZforcedurationr^   r!  Z	forcejson)rc   rd   r   r!   jsondumps)ri   r&  r)  r8  r=  r>  ro  rj   r<  rk   Z__forced_printings  s$    
zYoutubeDL.__forced_printingsc           '         s	  | dddkst j d}|dk	r< jt|kr<t |d |d< d|kr\|d |d<  j|d	d
}|dk	r d|  dS   jd7  _ | |d< } j	||d	d
  j dd	rdS |dkrdS  fdd}|t
t|sdS  j dd	rt|d| d} j dd	rBtjt|rB d n| ddkr^ d nlz@ d|  tjt|ddd}||d  W 5 Q R X W n* ttfk
r    d|  Y dS X  j dd	rt|d| d} j dd	rtjt|r d n| ds2 d nz@ d |  tjt|ddd}	|	|d  W 5 Q R X W nL ttfk
r    d Y n* ttfk
r    d!|  Y dS X t j d"d	 j d#g}
|
r| d$r|d$ } |d% }| D ]t\}}|d }t|||| d} j dd	rltjt|rl d&||f  q d'|  | d(dk	rz4tjt|ddd)d*}||d(  W 5 Q R X W n, ttfk
r    d+|  Y  dS X nzF|j|d, |d- d	d. }tt|d/}|| W 5 Q R X W nL tttt fk
r } z$ d0|t!|f  W Y qW 5 d}~X Y nX q j d1d	r$t|d2| d} j dd	rtjt|rֈ d3 nN d4|  zt" #|| W n* ttfk
r"    d5|  Y dS X  $||  j d6d		sz fd7d8}| d9dk	rg }d:}t% }|j&sg } d; n|g}d<d= }tj'|d dd }||d krtj'|d> n|}|d9 } j d?dkr||sd@|d<  dA dB||d f }tjt|r8 dC|  n||D ]f}t(|}|)| t* |dD|dE  |d } || s W dS |+|  || |}!|o|!}q<||dF< ||dG< n
|||}W n t,j-t.j/t0j1fk
r
 } z dHt!|  W Y dS d}~X Y np ttfk
r8 } zt2|W 5 d}~X Y nB t3fk
rx } z  dI|j4|j5f  W Y dS d}~X Y nX |	r|dJk	r j dK}"|"dkrdL}"dM}#| dN}$|$dk	rF|$dkrF|"dOkr dP|d- |$f  n^|"dLkr8t6 }%|%j&r|7dFg  |dF +|% n dQ|d- |$|#f  n|"dRksFt| d9dkr| dSdTkr|"dOkr dU|d-   n\|"dLkrt8 }&|&j&r|7dFg  |dF +|& n dV|d- |#f  n|"dRkst| dWdXk	s| dWdYk	r j dZ	r|"dOk	r, d[|d-   n\|"dLk	rzt9 }&|&j&	rb|7dFg  |dF +|& n d\|d- |#f  n|"dRk	stz :|| W n< t;k
	r } z d]t<|  W Y dS d}~X Y nX  =| dS )^z$Process a single resolved IE result.rR  r-  max_downloadsNr,  Z	fulltitler!  r  Fr`  r  r   	_filenamer   c              
      s   z,t j| }|r(t j|s(t | W dS  ttfk
r } z<t|trb|jtj	krbW Y  dS  
dt|  W Y dS d }~X Y nX d S )NTzunable to create directory F)r   pathdirnameexistsmakedirsr   IOErrorr   r   EEXISTr   r   )rD  dnr*  rh   rj   rk   ensure_dir_exists  s    

z1YoutubeDL.process_info.<locals>.ensure_dir_existsZwritedescriptionr?  nooverwritesz+[info] Video description is already presentz There's no description to write.z%[info] Writing video description to: wr   r   zCannot write description file Zwriteannotationszannotations.xmlz,[info] Video annotations are already presentannotationsz"There are no annotations to write.z%[info] Writing video annotations to: zCannot write annotations file: r2  r3  r  r\  z.[info] Video subtitle %s.%s is already presentz#[info] Writing video subtitles to: datar   )r   newlinezCannot write subtitles file rC  r  )r  wbz(Unable to download subtitle for "%s": %sZwriteinfojsonz	info.jsonz4[info] Video description metadata is already presentz6[info] Writing video description metadata as JSON to: z#Cannot write metadata to JSON file Zskip_downloadc                    sV   t | j  j} jD ]}|| q jdrJ d|d  || |S )Nr   z![debug] Invoking downloader on %rrC  )rI   rc   r|   r   rd   r   rD  )namern  fdr   rh   rj   rk   dlu  s    
z"YoutubeDL.process_info.<locals>.dlr  TzhYou have requested multiple formats but ffmpeg or avconv are not installed. The formats won't be merged.c                 S   sN   | \}}| d| d }}|rJ|rJd}|D ]}||kr.||kr. dS q.dS )Nr  ))	r  r  r  Zm4pZm4bZm4rZm4vZismvZismar  TFr  )r  r-  audioZ	video_extZ	audio_extZCOMPATIBLE_EXTSextsrj   rj   rk   compatible_formats  s    z2YoutubeDL.process_info.<locals>.compatible_formatsr   r  ZmkvzIRequested formats are incompatible for merge and will be merged into mkv.z%s.%sz4[download] %s has already been downloaded and mergedzf%sr  __postprocessorsZ__files_to_mergez!unable to download video data: %sz3content too short (expected %s bytes and served %s)rt  fixupZdetect_or_warnz3Install ffmpeg or avconv to fix this automatically.r  warnz %s: Non-uniform pixel ratio (%s)z$%s: Non-uniform pixel ratio (%s). %s)ignorenever	containerZm4a_dashz?%s: writing DASH m4a. Only some players support this container.zB%s: writing DASH m4a. Only some players support this container. %sr!  Zm3u8_nativeZm3u8Zhls_prefer_nativez%%s: malformed AAC bitstream detected.z(%s: malformed AAC bitstream detected. %szpostprocessing: %s)>rd   r   rc   r~   r  r'   r;  r   r+  rf  r5   r   r2   r   rD  rF  re   r  openr   r   rH  r   KeyError	TypeErroranyrE   r  r:   Z_request_webpagereadr   r$  r   r>   filter_requested_info_write_thumbnailsrN   r  splitextr   r   r/   r   r   URLErrorr	   HTTPExceptionsocketr   r;   r   r#  
downloadedrM   r<  rL   rK   post_processr-   r   record_download_archive)'ri   r&  rB  r  r)  rK  ZdescfnZdescfileZannofnZannofileZsubtitles_are_requestedr  r   Zsub_langZsub_infoZ
sub_formatZsub_filenameZsubfileZsub_datar*  ZinfofnrU  rj  successr  rw   rX  Zfilename_real_extZfilename_wo_extr  ro  r1  fnameZpartial_successZfixup_policyZINSTALL_FFMPEG_MESSAGEr  Zstretched_ppZfixup_pprj   rh   rk   r/    s   ""
"  

 "


 






 








zYoutubeDL.process_infoc              	   C   s   | j dt}t|dkrB|dkrBd|krB| j ddkrBt||D ]|}z| j|| j ddd}W n< tk
r   | d	 Y qF tk
r   | 	d
  Y qFX | j ddrF| 
t| qF| jS )zDownload a given list of URLs.rv   r   rt  %rB  rF  F)rF  zunable to download videoz2[info] Maximum number of downloaded files reached.Zdump_single_json)rc   rd   r   r   r3   rG  r;   r   r'   r   r   r@  rA  r}   )ri   Zurl_listrv   rC  r   rj   rj   rk   rD    s.     

zYoutubeDL.downloadc              	   C   s   t tj|gdtdd}| td|}W 5 Q R X z| j	|dd W nF t
k
r   |d}|d k	r| d|  | |g Y S  Y nX | jS )	Nrc  r   )modeZopenhookr   Tra  rZ  z-The info failed to download, trying with "%s")
contextlibclosing	fileinputZ	FileInputZhook_encodedrd  r@  loadsr   rV  r   rd   re   rD  r}   )ri   Zinfo_filenamero  rn  rZ  rj   rj   rk   download_with_info_file!  s       
z!YoutubeDL.download_with_info_filec                 C   s   t dd |  D S )Nc                 s   s"   | ]\}}|d kr||fV  qdS ))r  r  Nrj   r  rj   rj   rk   r   4  s   z2YoutubeDL.filter_requested_info.<locals>.<genexpr>)r   r  )r&  rj   rj   rk   rd  2  s    zYoutubeDL.filter_requested_infoc           	      C   s   t |}||d< g }|ddk	r0||d  || j |D ]}g }z||\}}W n. tk
r } z| |j W 5 d}~X Y nX |r@| jdds@|D ]H}| 	d|  zt
t| W q ttfk
r   | d Y qX qq@dS )z-Run all the postprocessors on the given file.filepathrY  NZ	keepvideoFz+Deleting original file %s (pass -k to keep)z)Unable to remove downloaded original file)r   rd   r  r{   runr-   r   rI  rc   r   r   remover   rH  r   re   )	ri   r)  Zie_inforn  Z	pps_chainr   Zfiles_to_deleterM  Zold_filenamerj   rj   rk   rk  8  s&    zYoutubeDL.post_processc                 C   sz   | d}|sd S | dp$| d}|d krjt| d}|sDd S | jD ]}||rJ| } qjqJd S | d | S )Nr  r\  r   rC  r  )rd   r9   ry   rA  r   r*  )ri   r&  r6  rY  rC  r   rj   rj   rk   _make_archive_idN  s    


zYoutubeDL._make_archive_idc              
   C   s   | j d}|d krdS | |}|s*dS zFt|ddd.}|D ]"}| |kr@ W 5 Q R  W dS q@W 5 Q R X W n0 tk
r } z|jtjkr W 5 d }~X Y nX dS )Ndownload_archiveFrc  r   rN  T)rc   rd   ry  r%   striprH  r   r   )ri   r&  fnvid_idarchive_filer  Zioerj   rj   rk   r7  b  s    
"zYoutubeDL.in_download_archivec              	   C   sV   | j d}|d krd S | |}|s*tt|ddd}||d  W 5 Q R X d S )Nrz  r   r   rN  r   )rc   rd   ry  r   r%   r   )ri   r&  r|  r}  r~  rj   rj   rk   rl  u  s    
z!YoutubeDL.record_download_archiveunknownc                 C   s   |  ddkrdS |  dd k	r(| d S |  dd k	rh|  dd k	rZd| d | d f }qd| d  }n |  dd k	rd	| d  }n|}|S )
Nr  r  z
audio onlyr  rU   rT   z%sx%sr  r  r  )r!  defaultr   rj   rj   rk   r-  ~  s    zYoutubeDL.format_resolutionc                 C   s|  d}| ddkr|d7 }| dr@|r0|d7 }|d|d  7 }| dd k	r^||d d 7 }| d	d k	r||d
|d	  7 }| dd k	r|r|d7 }|d|d  7 }| dd k	r| ddkr|r|d7 }||d 7 }| dd k	r|d7 }n(| dd k	r| dd k	r|d7 }| dd k	r:|d|d  7 }| dd k	rh|rX|d7 }|d|d  7 }| dd k	r|r|d7 }|d dkr|d7 }n|d|d  7 }n&| dd k	r|r|d7 }|d7 }| dd k	r|d|d  7 }| dd k	r|d|d  7 }| dd k	rF|r4|d7 }|t|d 7 }n2| dd k	rx|rd|d7 }|d t|d  7 }|S )!Nr   r  )Zf4fZf4mz(unsupported) languager  z[%s] r   rV   z%4dk r^  rH  z%s containerr  r  rY   @rW   zvideo@z%4dkrZ   z%sfpsr  z
video onlyz%-5srV  z@%3dkrX   z (%5dHz)r[   r\   ~)rd   r    )ri   Zfdictr   rj   rj   rk   _format_note  sl    

 
zYoutubeDL._format_notec                    s   | d|g} fdd|D }t|dkrT|d d  |d d rHdndd 7  < d	d
ddg} d|d t||f  d S )Nr  c                    sF   g | ]>}| d dks"|d  dkr|d |d  | |gqS )r  Nir  r  )rd   r-  r  r  rh   rj   rk   r     s    z*YoutubeDL.list_formats.<locals>.<listcomp>r   rr  r  r   z(best)zformat code	extensionr  r  z#[info] Available formats for %s:
%sr  )rd   r   r   r1   )ri   r&  r  tableZheader_linerj   rh   rk   r.    s    
(zYoutubeDL.list_formatsc                 C   s\   | d}|s$| d|d   d S | d|d   | tddddgd	d
 |D  d S )Nr  z#[info] No thumbnails present for %sr  z[info] Thumbnails for %s:ZIDrT   rU   URLc                 S   s0   g | ](}|d  | dd| dd|d gqS )r  rT   r  rU   rC  r  )r   r  rj   rj   rk   r     s     z-YoutubeDL.list_thumbnails.<locals>.<listcomp>)rd   r   r1   )ri   r&  r  rj   rj   rk   r    s    


zYoutubeDL.list_thumbnailsr  c                 C   sR   |s|  d||f  d S |  d||f  |  tddgdd | D  d S )Nz%s has no %szAvailable %s for %s:ZLanguager  c                 S   s,   g | ]$\}}|d  dd t|D gqS )rH  c                 s   s   | ]}|d  V  qdS )r  Nrj   r  rj   rj   rk   r     s     z6YoutubeDL.list_subtitles.<locals>.<listcomp>.<genexpr>)r   reversed)r   r7  r  rj   rj   rk   r     s   z,YoutubeDL.list_subtitles.<locals>.<listcomp>)r   r1   r  )ri   r6  r  rS  rj   rj   rk   r+    s    
zYoutubeDL.list_subtitlesc                 C   s$   t |trt|}| jj|| jdS )z Start an HTTP download )timeout)r   r   r7   _openerr_  _socket_timeout)ri   reqrj   rj   rk   urlopen  s    
zYoutubeDL.urlopenc                 C   sP  | j dsd S tdtk	r&| d ttjddttjj }dt	
 t ||  f }t|d d | dt d	  tr| d
 zhtjddddgtjtjtjtjtd}| \}}|  }td|r| d| d	  W n: tk
r*   zt  W n tk
r$   Y nX Y nX dd }| dt ! | t" f  t#$| }t% |d< t&' |d< d(dd t)|* D }|sd}| d|  i }	| j+j,D ]}
t-|
dr|	.|
j/ q| dt|	 d	  | j ddrL| 0d 1 d!}| d"|  | 0d#1 d!}t2|t2tkrL| d$|  d S )%Nr   r   z>Your Python is broken! Update to a newer and supported versionr   zmissing (%s)z5[debug] Encodings: locale %s, fs %s, out %s, pref %s
rN  z[debug] youtube-dl version r   z([debug] Lazy loading extractors enabled
gitz	rev-parsez--shortHEAD)rp   rq   cwdz	[0-9a-f]+z[debug] Git HEAD: c                  S   s4   t  } | dkr0ttdr0| dtjd d   S | S )NPyPypypy_version_infoz version %d.%d.%dr  )r   python_implementationr   r   r  )Z	impl_namerj   rj   rk   r  	  s    z;YoutubeDL.print_debug_header.<locals>.python_implementationz$[debug] Python version %s (%s) - %s
ZrtmpdumpZ	phantomjsrH  c                 s   s"   | ]\}}|rd ||f V  qdS )r   Nrj   )r   Zexer
  rj   rj   rk   r   $	  s   z/YoutubeDL.print_debug_header.<locals>.<genexpr>r  z[debug] exe versions: %s
proxiesz[debug] Proxy map: Z	call_homeFzhttps://yt-dl.org/ipr   z[debug] Public IP address: %s
z https://yt-dl.org/latest/versionzoYou are using an outdated version (newest version: %s)! See https://yt-dl.org/update if you need help updating.)3rc   rd   r   r   re   r  r   rp   r  localegetpreferredencodingr   get_encodingr?   r   rQ   rG   r   r   r   r   rD  rE  abspath__file__communicater   r{  r   r   rK  Z	exc_clearr   python_versionr,   rO   get_versionsrJ   rH   _versionr   sortedr  r  handlersr   r   r  r  rc  r=   )ri   Zstdout_encodingZencoding_strspr   r*  r  Zexe_versionsZexe_strZ	proxy_maphandleripaddrZlatest_versionrj   rj   rk   r     s      

  



zYoutubeDL.print_debug_headerc              	   C   sN  | j d}|d krdnt|| _| j d}| j d}|d krNt | _n0t|}t|| _t	
|t	jr~| jjddd t| j}|d k	r|dkri }q||d}n$t }d	|krd
|kr|d	 |d
< t|}| j drdnd}t| j |d}t| j |d}	t }
t }t }dd }||_t||||	|
||}g |_|| _d S )Nsocket_timeoutiX  r   proxyTr   r   )httphttpsr  r  Zdebug_printtrafficr   r   )
debuglevelc                  _   s   t dd S )NzHfile:// scheme is explicitly disabled in youtube-dl for security reasons)r   rg  )r   rL  rj   rj   rk   	file_openf	  s    z*YoutubeDL._setup_opener.<locals>.file_open)rc   rd   floatr  r   	CookieJarr   r   r@   r   accessR_OKloadrA   r   
getproxiesr+   r&   rB   rC   r   FileHandlerr  build_opener
addheadersr  )ri   Ztimeout_valZopts_cookiefileZ
opts_proxyZcookie_processorr  Zproxy_handlerr  https_handlerZydlhZredirect_handlerZdata_handlerZfile_handlerr  openerrj   rj   rk   r   >	  sL    

      zYoutubeDL._setup_openerc              
   C   sT   t |tr|S z||  W S  tk
rN } z|jd |_ W 5 d }~X Y nX d S )NzH. Check your system encoding configuration or use the --encoding option.)r   r   r   r  r   r  )ri   r   r*  rj   rj   rk   r   s	  s    
zYoutubeDL.encodec                 C   s   | j d}|d krt }|S )Nr   )rc   rd   r.   )ri   r   rj   rj   rk   r  }	  s    zYoutubeDL.get_encodingc                 C   s  | j ddr(|d}|rF|d g}n| j ddrB|d}nd S |sNd S |D ]b}t|d d}t|dkr~d	|d
  nd}t|dkrd|d
  nd}t|| ||d |d< }| j ddrtjt|r| 	d|d |d
 |f  qR| 	d|d |d
 |f  zV| 
|d }	tt|d}
t|	|
 W 5 Q R X | 	d|d |d
 ||f  W qR tjtjtjfk
r } z| d|d t|f  W 5 d }~X Y qRX qRd S )NZwritethumbnailFr  rr  Zwrite_all_thumbnailsrC  Zjpgr   z_%sr  r   z%s r  r)  rL  z'[%s] %s: Thumbnail %sis already presentrY  z$[%s] %s: Downloading thumbnail %s...rR  z#[%s] %s: Writing thumbnail %sto: %sz%Unable to download thumbnail "%s": %s)rc   rd   r   r   r2   r   rD  rF  r   r   r  r_  shutilcopyfileobjr   rg  r	   rh  ri  r   re   r   )ri   r&  r)  r  r  Z	thumb_extsuffixZthumb_display_idZthumb_filenameZufZthumbfr*  rj   rj   rk   re  	  s@    


zYoutubeDL._write_thumbnails)NT)F)N)FF)NN)N)T)T)r  )r  )Ir  r  r  __doc__setr   rc   ry   r{   r}   r~   rj  ri  r   r   r   r   rE   r   r   r   r   r   r   r   r   r   r   r   r   r   r   re   r   r   r+  r;  staticmethodr?  rG  Z(_YoutubeDL__handle_extraction_exceptionsrB  rU  rV  rm  r  r  r  r  r  r  rg  r,  rf  r/  rD  ru  rd  rk  ry  r7  rl  r-  r  r.  r  r+  r  r   r   r   r  re  rj   rj   rj   rk   rS   t   s    R
^



		

Y*
 
&
gp
?
  
 r1  
	8
L5
rS   )s
__future__r   r   r  rq  rh  r%  r   rs  r  r  r@  r  r  r   r   r   r  r   ri  r   r  r  r   r  r  r   compatr   r   r   r	   r
   r   r   r   r   r   r   r   utilsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   r   rD   rY  rE   rF   rG   Zextractor.openloadrH   
downloaderrI   Zdownloader.rtmprJ   ZpostprocessorrK   rL   rM   rN   rO   rP   versionrQ   r   r  rS   rj   rj   rj   rk   <module>   sF   84 
