U
    $Pfu                     @   s   d dl mZ d dlZd dlZd dlZddlmZ ddlmZm	Z	m
Z
mZmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd	 d	eZG d
d deZdS )    )unicode_literalsN   )InfoExtractor   )compat_etree_fromstringcompat_http_client
compat_strcompat_urllib_errorcompat_urllib_parse_unquote compat_urllib_parse_unquote_plus)
clean_htmlerror_to_compat_strExtractorErrorfloat_or_noneget_element_by_idint_or_none
js_to_jsonlimit_lengthparse_count	qualitiessanitized_Requesttry_geturlencode_postdataurljoinc                   @   s2  e Zd ZdZdZdZdZdZdZdZ	ddd	d
dddddddddd
dddddddgdddddd
d d!d"d#ddd$d%d&d'd
d(d)d*d+dd,d-d.d
d/d0d*d+dd1d2d3d4d
d5dd6d7d8d9d:d;d<d=d
d>d?d@dAde
dBdCdDdEd
dFdGdHdIdJdKdLidMdNdOd
dPdQdRdSddTdKdLidMdUdVd
dWdXdYdZdJdKdLidMd[dLd\d]dLd\d^dLd\d_dLd\d`dLd\dadLd\dbdLd\dcdLd\ddded
dfdgd*dKdLidMdhdidjidkdldmdLd\dndidoidpdldqdrd
dsdtdudvdwddxdydzd
d{d|d}d~dwdddLd\ddididddgZdZddiZedd Zdd Zdd Zdd Zdd ZdS )
FacebookIEay  (?x)
                (?:
                    https?://
                        (?:[\w-]+\.)?(?:facebook\.com|facebookcorewwwi\.onion)/
                        (?:[^#]*?\#!/)?
                        (?:
                            (?:
                                video/video\.php|
                                photo\.php|
                                video\.php|
                                video/embed|
                                story\.php|
                                watch(?:/live)?/?
                            )\?(?:.*?)(?:v|video_id|story_fbid)=|
                            [^/]+/videos/(?:[^/]+/)?|
                            [^/]+/posts/|
                            groups/[^/]+/permalink/|
                            watchparty/
                        )|
                    facebook:
                )
                (?P<id>[0-9]+)
                z\https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1z]https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1Zfacebookz-https://www.facebook.com/video/video.php?v=%szZhttps://www.facebook.com/video/tahoe/async/%s/?chain=true&isvideo=true&payloadtype=primaryz<https://www.facebook.com/video.php?v=637842556329505&fref=nfZ 6a40d33c0eccbb1af76cf0485a052659Z637842556329505mp4zOre:Did you know Kei Nishikori is the first Asian man to ever reach a Grand SlamzTennis on FacebookZ20140908iTidexttitleuploaderupload_date	timestampzRequires logging in)urlmd5	info_dictskipz4https://www.facebook.com/video.php?v=274175099429670Z274175099429670z"re:^Asif Nawab Butt posted a videozAsif Nawab ButtZ20140506iV"iSzre:^https?://.*)r   r   r   r    r!   r"   	thumbnailr   )r#   r%   Zexpected_warningszVideo with DASH manifestz4https://www.facebook.com/video.php?v=957955867617029Z b2c28d528273b323abe5c6ab59f0f030Z957955867617029zPWhen you post epic content on instagram.com/433 8 million followers, this is ...zDemy de ZeeuwZ20160110iYV)noter#   r$   r%   r&   z8https://www.facebook.com/maxlayn/posts/10153807558977570Z 037b1fa7f3c2d02b7a0d7bc16031ecc6Z544765982287235z)"What are you doing running in the snow?"ZFailArmy)r   r   r   r    z
Video gonezOhttps://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903Z 1deb90b6ac27f7efcf6d747c8a27f5e3Z1035862816472149z;What the Flock Is Going On In New Zealand  Credit: ViralHogzS. Saintzswf params escapedz<https://www.facebook.com/barackobama/posts/10153664894881749Z 97ba073838964d12c70566e0085c2b91Z10153664894881749zPAverage time to confirm recent Supreme Court nominees: 67 days Longest it's t...i,VZ20160223zBarack Obama)r   r   r   r'   r"   r!   r    )r(   r#   r$   r%   z6https://www.facebook.com/cnn/videos/10155529876156509/Z 9571fae53d4165bbbadb17a94651dcdcZ10155529876156509uT   She survived the holocaust — and years later, she’s getting her citizenship s...iXZ20161030ZCNN)r   r   r   r"   r!   r    r'   
view_count)r#   r$   r%   zAhttps://www.facebook.com/yaroslav.korpan/videos/1417995061575415/Z1417995061575415z$md5:1db063d6a8c13faa8da727817339c857isXZ20170209zYaroslav Korpan)r   r   r   r"   r!   r    Zskip_downloadT)r#   r%   paramsz>https://www.facebook.com/LaGuiaDelVaron/posts/1072691702860471Z1072691702860471z$md5:ae2d22a93fbb12dad20dc393a869739diXZ20161024u   La Guía Del Varón)r   r   r   r"   r!   r    r'   zLhttps://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/Z1396382447100162z$md5:19a428bbde91364e3de815383b54a235i&XZ20170202zElisabeth Ahtnz6https://www.facebook.com/video.php?v=10204634152394104r#   Zonly_matchingzAhttps://www.facebook.com/amogood/videos/1618742068337349/?fref=nfzbhttps://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theaterzfacebook:544765982287235zJhttps://www.facebook.com/groups/164828000315060/permalink/764967300301124/z@https://zh-hk.facebook.com/peoplespower/videos/1135894589806027/z>https://www.facebookcorewwwi.onion/video.php?v=274175099429670zIhttps://www.facebook.com/onlycleverentertainment/videos/1947995502095005/z<https://www.facebook.com/WatchESLOne/videos/359649331226507/Z359649331226507z?#ESLOne VoD - Birmingham Finals Day#1 Fnatic vs. @Evil GeniuseszESL One Dota 2z@https://www.facebook.com/100033620354545/videos/106560053808006/r   Z106560053808006r   )r#   r%   playlist_countz1https://www.facebook.com/watch/?v=647537299265662zAhttps://www.facebook.com/PankajShahLondon/posts/10157667649866271Z10157667649866271   zHhttps://m.facebook.com/Alliance.Police.Department/posts/4048563708499330Z117576630041613zFacebook video #117576630041613Z189393014416438Z20201123i_)r   r   r   uploader_idr!   r"   )r#   r%   r&   zMhttps://www.facebook.com/groups/ateistiskselskab/permalink/10154930137678856/Z211567722618337zFacebook video #211567722618337Z127875227654254Z20161122i3X)r#   r%   z7https://www.facebook.com/watch/live/?v=1823658634322275z3https://www.facebook.com/watchparty/211641140192478Z211641140192478r   )r#   r%   r,   r&   zK(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)graphURIz/api/graphql/c                 C   sL   g }t d| D ]}||d qt d| D ]}||d q2|S )Nzd<iframe[^>]+?src=(["\'])(?P<url>https?://www\.facebook\.com/(?:video/embed|plugins/video\.php).+?)\1r#   z(?x)<div[^>]+
                class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
                data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2))refinditerappendgroup)webpageurlsZmobj r6   A/tmp/pip-unpacked-wheel-7zdooeg3/youtube_dl/extractor/facebook.py_extract_urls+  s    zFacebookIE._extract_urlsc              
   C   s  |   \}}|d krd S t| j}| ddd | j|d ddd}| d|d}| d	|d
}||||dddddd	}t| jt|}|dd z| j|d ddd}	t	d|	d k	r| j
d|	dd dd}
|
rtd|
 dd| jd W d S | jd|	dd d}| jd |	d!d d}|r |s&W d S ||d"d#}t| jt|}|dd | j|d d$d%}t	d&|d k	r|| jd' W nJ tjtjtjfk
r } z| jd(t|  W Y d S d }~X Y nX d S ))Nzfacebook.comlocaleen_USzDownloading login pagezUnable to download login page)r(   Zerrnotez/<input type="hidden" name="lsd" value="([^"]*)"lsdzname="lgnrnd" value="([^"]*?)"lgnrndzhttp://facebook.com/home.php01z-60)	emailpassr;   r<   nextZdefault_persistentZlegacy_returntimezoneZtrynumContent-Type!application/x-www-form-urlencodedz
Logging inzunable to fetch login pagez <form(.*)name="login"(.*)</form>zh(?s)<div[^>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>zlogin errorerror)defaultr3   zUnable to login: %sTexpectedzjunable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.zname="fb_dtsg" value="(.+?)"fb_dtsgrF   z.name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"hZ	dont_save)rI   rK   Zname_action_selectedzConfirming login)r(   zid="checkpointSubmitButton"zSUnable to confirm login, you have to login in your browser and authorize the login.zunable to log in: %s)Z_get_login_infor   
_LOGIN_URLZ_set_cookie_download_webpage_search_regexr   
add_headerr0   search_html_search_regexr   Z_downloaderZreport_warning_CHECKPOINT_URLr	   URLErrorr   HTTPExceptionsocketrE   r   )selfZ	useremailpasswordZlogin_page_reqZ
login_pager;   r<   Z
login_formrequestZlogin_resultsrE   rI   rK   Z
check_formZ	check_reqZcheck_responseerrr6   r6   r7   _login:  s    
           zFacebookIE._loginc                 C   s   |    d S N)rZ   rV   r6   r6   r7   _real_initializey  s    zFacebookIE._real_initializec           .         s   |ddd }dd jjddgddd	d
d}|rV|dg }fdd}fddfddfddfdd}|sЈjdj dj gddd	td
}||}|s\|d}|r\g   fdddzfdd	}|d pg }	|d!p&i }
|	s>|
r>|	|
 |	D ]}
t|
d"d# t	pZi }t|d$d# d%d# gt
pxg }|D ]H}t|d&d# t	}t|d'd# t
pg }|D ]}|| q|| q~qBt|d(d# t
pg }|D ]}||d!d) q|d*pi }|rPt|d+d# d,d# gt
p,g }|D ]}|| q2 sP|  S |std-}|d k	rtd.|d/ d0d1n tfd2d3d4D r  |sd5|krd6td7id8}|d9}|rt|d:d# t	}|r|d; ||d< < d=}|d>pg D ]}|d? d@kr |dA _q jt|jdB t|dCdD dE }g  t|dFd# p~g D ]\}t|dGd# pi }|dH}|sqt|} j|  ||d< q S |sjj j td/jdIdJdKd	jdLdMdNd	jdOdPdQd	dRdSdTidU}jjdV|dWdd	d
d}||}|sxtdXt|d/krg  |D ]F}|d? dY}|sq t|| |d? dZ q S |d? }g }i } |D ]}!|!d[ }"|!rt|!t	r|!g}!|!rt|!t
s0qd\D ]n}#d]D ]b}$|!d? d^|#|$f }%|%r<|"d_krjd`nd?}&|#dakr|&db7 }&|dc|"|#|$f |%|&dd q<q4|!d? | |!d? de}'|'r| dfg dg|'i q|stdh| j didjd d	}(|(s j dkdld d	}(|(s8j!dmdjd d	}(|(rJt"|(dn}(ndo }(t#t$dppjdqdrd d	pj%d
d})t&jdsdtd d	}*!dudvg}+t'jdwdxd d	},|(||)|*|+|,| dy}-|-S ){Nz://m.facebook.com/z://www.facebook.com/c                 S   sH   g }| D ]:}t |dd dkr|d d }|dr||d  q|S )Nc                 S   s   | d d S )Nr   r   r6   xr6   r6   r7   <lambda>      zJFacebookIE._extract_from_url.<locals>.extract_video_data.<locals>.<lambda>ZVideoConfigr   r   video_idZ	videoData)r   getr2   )	instances
video_dataitemZ
video_itemr6   r6   r7   extract_video_data  s    
z8FacebookIE._extract_from_url.<locals>.extract_video_dataz handleServerJS\(({.+})(?:\);|,")z\bs\.handle\(({.+?})\);zserver js data{}rJ   Ffatalrd   c                    s    | r t | dd tpg S d S )Nc                 S   s   | d d S )NZjsmodsrd   r6   r^   r6   r6   r7   r`     ra   zUFacebookIE._extract_from_url.<locals>.extract_from_jsmods_instances.<locals>.<lambda>)r   list)Zjs_data)rg   r6   r7   extract_from_jsmods_instances  s      zCFacebookIE._extract_from_url.<locals>.extract_from_jsmods_instancesc                    s*   |  d}|r&| tt| d S )Ndash_manifest)rc   extendZ_parse_mpd_formatsr   r   )videoformatsrm   r\   r6   r7   extract_dash_manifest  s
    

z;FacebookIE._extract_from_url.<locals>.extract_dash_manifestc                    s(   | D ]}d| di d< q |  d S )Nzfacebookexternalhit/1.1Zhttp_headersz
User-Agent)
setdefaultZ_sort_formats)rp   fr\   r6   r7   process_formats  s    z5FacebookIE._extract_from_url.<locals>.process_formatsc                    s&    j  jd|  dddddp$i S )Nz3handleWithCustomApplyEach\([^,]+,\s*({.*?%s.*?})\);zreplay datarh   rJ   Fri   )_parse_jsonrN   )_filter)rV   rb   r4   r6   r7   extract_relay_data  s       z8FacebookIE._extract_from_url.<locals>.extract_relay_datac                    sD    | }| dpg D ](}|d dkrt|dd tp8i   S qd S )Nrequirer   ZRelayPrefetchedStreamCachec                 S   s   | d d d d d S )Nr-   r   Z__bboxresultdatar6   r^   r6   r6   r7   r`     ra   zUFacebookIE._extract_from_url.<locals>.extract_relay_prefetched_data.<locals>.<lambda>)rc   r   dict)rv   Zreplay_datarx   )rw   r6   r7   extract_relay_prefetched_data  s    zCFacebookIE._extract_from_url.<locals>.extract_relay_prefetched_datazNbigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+z4bigPipe\.onPageletArrive\(({.*?id\s*:\s*"%s".*?})\);zjs dataz>"(?:dash_manifest|playable_url(?:_quality_hd)?)"\s*:\s*"[^"]+"c           
   	      s   g }t ddg}dD ]2\}}| d| }|s0q|||||d q| | | | dpp| dpp}||t| dd	 t| d
d	 t| dt| ddd}t| dd	 }| d}	|	r||	|d n|pd| |d<  | d S )Nsdhd)) r}   )Z_quality_hdr~   playable_url)	format_idqualityr#   ZvideoIdr   c                 S   s   | d d S )NZthumbnailImageurir6   r^   r6   r6   r7   r`     ra   zKFacebookIE._extract_from_url.<locals>.parse_graphql_video.<locals>.<lambda>c                 S   s   | d d S )Nownerr   r6   r^   r6   r6   r7   r`     ra   Zpublish_timeZplayable_duration_in_msi  )r   rp   r'   r.   r"   durationc                 S   s   | d d S )NZsavable_descriptiontextr6   r^   r6   r6   r7   r`     ra   name)r   descriptionFacebook video #%sr   )r   rc   r2   r   r   r   update)
ro   rp   qsuffixr   r   v_idinfor   r   )entriesrq   rt   rb   r6   r7   parse_graphql_video  s<    



z9FacebookIE._extract_from_url.<locals>.parse_graphql_videomediac                    s(   |  |pi }| ddkr$ |S d S )NZ
__typenameZVideo)rc   )
attachmentkeyr   )r   r6   r7   parse_attachment  s    z6FacebookIE._extract_from_url.<locals>.parse_attachmentnodesnodec                 S   s   | d d d S )NZcomet_sectionscontentstoryr6   r^   r6   r6   r7   r`     ra   z.FacebookIE._extract_from_url.<locals>.<lambda>c                 S   s   | d d S )NZattached_storyattachmentsr6   r^   r6   r6   r7   r`     ra   c                 S   s   | d S )Nr   r6   r^   r6   r6   r7   r`     ra   c                 S   s   | d d S )NZstyle_type_rendererr   r6   r^   r6   r6   r7   r`     ra   c                 S   s   | d d S )NZall_subattachmentsr   r6   r^   r6   r6   r7   r`     ra   c                 S   s   | d d d S )NZmediasetZ	currMediaedgesr6   r^   r6   r6   r7   r`     ra   )r   ro   c                 S   s   | d d S )Nr   r   r6   r^   r6   r6   r7   r`     ra   c                 S   s   | d d S )NZcreation_storyr   r6   r^   r6   r6   r7   r`     ra   z8class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>z/The video is not available, Facebook said: "%s"r   TrG   c                 3   s   | ]}| kV  qd S r[   r6   ).0p)r4   r6   r7   	<genexpr>  s     z/FacebookIE._extract_from_url.<locals>.<genexpr>)z>You must log in to continuezid="login_form"zid="loginbutton"z/watchparty/l   Ws/j ZlivingRoomID)Zdoc_id	variablesz"login_data"\s*:\s*{c                 S   s   | d d S )NZ
login_datar;   r6   r^   r6   r6   r7   r`     ra   valuer   z!\[\s*"RelayAPIConfigDefaults"\s*,definer   ZRelayAPIConfigDefaultsr   r/   )rz   rz   living_roomc                 S   s   | d d d S )NZrecapZwatched_contentr   r6   r^   r6   r6   r7   r`   *  ra   c                 S   s   | d d S )Nr   ro   r6   r^   r6   r6   r7   r`   +  ra   r   z%pkg_cohort["\']\s*:\s*["\'](.+?)["\']z
pkg cohortzPHASED:DEFAULTz!client_revision["\']\s*:\s*(\d+),zclient revisionZ3944515z>"DTSGInitialData"\s*,\s*\[\]\s*,\s*{\s*"token"\s*:\s*"([^"]+)"z
dtsg tokenr   )Z__aZ__pcZ__revrI   rC   rD   )rz   headerszfor\s+\(\s*;\s*;\s*\)\s*;(.+)ztahoe js datazCannot parse data	video_urlrb   Zstream_type)r}   r~   )srcZsrc_no_ratelimitz%s_%sZprogressiveir~      z%s_%s_%s)r   r#   
preferencesubtitles_srcenr#   zCannot find video formatsz2<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>r   zf(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>zalternative titler   P   r   ZfbPhotoPageAuthorNamezownerName\s*:\s*"([^"]+)"r    z<abbr[^>]+data-utime=["\'](\d+)r"   zog:imageztwitter:imagez \bviewCount\s*:\s*["\']([\d,.]+)z
view count)r   r   rp   r    r"   r'   r)   	subtitles)r   )(rM   replaceru   rN   rc   _SUPPORTED_PAGLETS_REGEXr   r2   r   r{   rk   Zplaylist_resultr0   rP   r   r3   anyZraise_login_requiredjsondumps_api_configZ_download_jsonr   r   r   
url_result_VIDEO_PAGE_TEMPLATEie_key_VIDEO_PAGE_TAHOE_TEMPLATElen
isinstancerr   rQ   Z_html_search_metar   r   r   Z_og_search_titler   r   ).rV   r#   rb   re   Zserver_js_datarl   r|   rz   r   r   r   r   r   r   nsnr   Zedgero   Zm_msgZ	post_dataZprefetched_datar;   Z
relay_datar   r   r   Z
tahoe_dataZtahoe_js_datavr   rp   r   rs   r   r   Zsrc_typer   r   r   Zvideo_titler    r"   r'   r)   r%   r6   )	r   rq   rw   rg   r   rt   rV   rb   r4   r7   _extract_from_url|  s"   
 	       "

  
  
           
  

              zFacebookIE._extract_from_urlc                 C   s.   |  |}|dr| j| n|}| ||S )Nz	facebook:)	_match_id
startswithr   r   )rV   r#   rb   Zreal_urlr6   r6   r7   _real_extract  s    
zFacebookIE._real_extractN)__name__
__module____qualname__
_VALID_URLrL   rR   Z_NETRC_MACHINEZIE_NAMEr   r   int_TESTSr   r   staticmethodr8   rZ   r]   r   r   r6   r6   r6   r7   r   #   s  
	 
 	   	 		   d 
?  *r   c                	   @   sL   e Zd ZdZdddddddd	d
e gdddddddgZdd ZdS )FacebookPluginsVideoIEzQhttps?://(?:[\w-]+\.)?facebook\.com/plugins/video\.php\?.*?\bhref=(?P<id>https.+)zhttps://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fgov.sg%2Fvideos%2F10154383743583686%2F&show_text=0&width=560Z 5954e92cdfe51fe5782ae9bda7058a07Z10154383743583686r   zWhat to do during the haze?zGov.sgZ20160826iWr   )r#   r$   r%   Zadd_iezrhttps://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fvideo.php%3Fv%3D10204634152394104Tr+   zhttps://www.facebook.com/plugins/video.php?href=https://www.facebook.com/gov.sg/videos/10154383743583686/&show_text=0&width=560c                 C   s   |  t| |t S r[   )r   r
   r   r   r   )rV   r#   r6   r6   r7   r     s    z$FacebookPluginsVideoIE._real_extractN)r   r   r   r   r   r   r   r   r6   r6   r6   r7   r     s(   r   )
__future__r   r   r0   rU   commonr   compatr   r   r   r	   r
   r   utilsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r6   r6   r6   r7   <module>   s    @     