U -e~@sddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddlm Z ddl mZddlmZmZddlmZmZmZmZmZmZmZmZmZddlmZddlZddl m!Z!d d l"m#Z#d d l$m%Z%d d l&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,d d l-m.Z.d dl/m0Z0ddl1m2Z2e.e3Z4dddddddddddddg Z5dde%Dd d iZ6d!d"e%DZ7e8d#Z9e:;d$d e:;d%d e:;d&d e:;d'd(e:;d)d*e:;d+d,e:;d-d.e:;d/d0iZd3d4eed?ZFd@dAZGdBdCZHdee2eIdDdEdFZJdee2eKdDdGdHZLdee2eIdDdIdJZMddKdLZNdMdNZOeDeDdOdPdQZPeeDdRdSdTZQdeDee2eeDdUdVdWZRdeDee2eeDeeDeeDefffdUdXdYZSdeDee2eeDeeDeeDefffdUdZd[ZTddd]eDee2d^d_d`ZUdeDee2eeDdadbdcZVddddeee2d]dfdgZWdee2d]dhdiZXGdjdkdkeYeZZeeDeeZfdldmdnZ[dd]ee2d]dodpZ\dd]ee2d]dqdrZ]dee2d]dsdtZ^dee2d]dudvZ_dee2d]dwdxZ`dee2d]dydzZadee2d]d{d|ZbGd}d~d~eZcGdddecZdGdddecZeGdddZfdS)N) TimeoutError)BytesIO)chain)Path PurePosixPath) AnyCallableDict GeneratorIterableListOptionalTupleUnion) ElementTree) ClientError)config)COMPRESSION_FILESYSTEMS)"get_authentication_headers_for_urlget_datasets_user_agent http_head is_local_pathis_relative_pathurl_or_path_join) get_logger) map_nested)DownloadConfigtxtcsvjsonZjsonlZtsvZconllZconlluorigZparquetZpklpicklerelxmlcCsi|]}|jd|jqS).) extensionlstripprotocol.0Zfs_classr,m/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/download/streaming_download_manager.py 6s r.zipcCsh|] }|jqSr,)r)r*r,r,r- :sr0z(?Ksr7c@s eZdZdS)NonStreamableDatasetErrorN)__name__ __module__ __qualname__r,r,r,r-r8Qsr8cGsPt|d^}}t|r,tjj|f|Stj|f|}d|g|SdS)u# This function extends os.path.join to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xjoin function allows you to apply the join on the first path of the chain. Example:: >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt") zip://folder1/file.txt::https://host.com/archive.zip ::N)strsplitrospathjoin posixpath)apbr,r,r-xjoinUs rFcCs\t|d^}}t|r0tjt|}n t|}| drL|d7}d |g|S)u# This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xdirname function allows you to apply the dirname on the first path of the chain. Example:: >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip") zip://folder1::https://host.com/archive.zip r<://) r=r>rr?r@dirnameras_posixrBendswithrArCrEr,r,r-xdirnameps  rM)urlpathdownload_configcCsVt|d^}}t|r&tj|St||d\}}tj||d^}}||SdS)aExtend `os.path.exists` function to support both local and remote files. Args: urlpath (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` r<rOstorage_optionsN) _as_strr>rr?r@exists!_prepare_path_and_storage_optionsfsspecget_fs_token_paths)rNrOmain_hop rest_hopsrRfs_r,r,r-xexistss  r\cCs<t|d^}}t|r.tjt|St|SdS)u This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xbasename function allows you to apply the basename on the first path of the chain. Example:: >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip") file.txt r<N) r=r>rr?r@basenamerrJrBrLr,r,r- xbasenamesr^cCsft|d^}}t|r.tjt|St|\}}d| drR|dn|g||fSdS)u, This function extends os.path.split to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xsplit function allows you to apply the xsplit on the first path of the chain. Example:: >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip") ('zip://folder1::https://host.com/archive.zip', 'file.txt') r<rGrHN) r=r>rr?r@rrJrBrArK)rCrEtailr,r,r-xsplits r`cCsTt|d^}}t|r.tjt|St|\}}d |g||fSdS)u8 This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xsplitext function allows you to apply the splitext on the first path of the chain. Example:: >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip") ('zip://folder1/file::https://host.com/archive.zip', '.txt') r<N) r=r>rr?r@splitextrrJrBrA)rCrEextr,r,r- xsplitexts rcrOreturncCsVt|d^}}t|r&tj|St||d\}}tj||d^}}||SdS)zExtend `os.path.isfile` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` r<rPrQN) r=r>rr?r@isfilerUrVrW)r@rOrXrYrRrZr[r,r,r-xisfiles  rgc Cst|d^}}t|r&tj|St||d\}}tj||d^}}| |}|dkr~t ||d}t | }W5QRX|SdS)zExtend `os.path.getsize` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `int`: optional r<rPrQN) r=r>rr?r@getsizerUrVrWsizexopenr5read) r@rOrXrYrRrZr[rifr,r,r-xgetsizes   rmcCsrt|d^}}t|r&tj|St||d\}}tj||d^}}|dd}| dsddS||SdS) zExtend `os.path.isdir` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` r<rPrQ://r/TN) r=r>rr?r@isdirrUrVrWstrip)r@rOrXrYrRrZr[ inner_pathr,r,r-xisdirs   rscCsjt|d^}}t|r:|r.tjj||dStj|S|rZtj|t|dddStj|SdS)zExtend `os.path.relpath` function to support remote files. Args: path (`str`): URL path. start (`str`): Start URL directory path. Returns: `str` r<)startrN)r=r>rr?r@relpathrB)r@rtrXrYr,r,r-xrelpath4s  rvcs$|jtjfdd}||_dS)Nc sd}tddD]p}z||}WqWqttfk r}z4|}tdtjd|ddttjW5d}~XYqXqt d||S)Nrz4Got disconnected from remote data host. Retrying in zsec [ro]zServer Disconnected) rangerrloggerwarningrZSTREAMING_READ_RETRY_INTERVALtimesleepConnectionError)argskwargsZdisconnect_errretryouterr max_retriesrkr,r-read_with_retriesIs    z?_add_retries_to_file_obj_read_method..read_with_retries)rkrZSTREAMING_READ_MAX_RETRIES)file_objrr,rr-$_add_retries_to_file_obj_read_methodEsr)r@recCs*|dd}dD]}||d}q|S)Nr&z?-_r)r>)r@r'Zsymbr,r,r-_get_path_extension\srrec Csz|dWnttjfk r*YdSX|t}|dttD]X}t|dt|}|dk rr|St |dt|}|dk rHt d|dqHdS)zQread the magic number from a file-like object and return the compression protocolrNzCompression protocol 'z' not implemented.) seekAttributeErrorioUnsupportedOperationrkMAGIC_NUMBER_MAX_LENGTHrx$MAGIC_NUMBER_TO_COMPRESSION_PROTOCOLget0MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOLNotImplementedError)rlr6i compressionr,r,r-*_get_extraction_protocol_with_magic_numberfs   rrNrOrec Cst|}|dd}t|}|tks8|dks8|drrBASE_KNOWN_EXTENSIONSrK!COMPRESSION_EXTENSION_TO_PROTOCOLrUrVopenrFileNotFoundError startswithr HF_ENDPOINT)rNrOr@r'rRrlr,r,r-_get_extraction_protocolxs0  rcCsJg}i}|dD](}t||d\}}||||qd||fS)Nr<rP)r>,_prepare_single_hop_path_and_storage_optionsappendupdaterA)rNrOZprepared_urlpathZprepared_storage_optionsZhoprRr,r,r-rUs  rUc Cs|dkr dn|j}d|kr(|ddnd}|dk rJ||jkrJ|j|}n,|dk rr||jkrrdd|jD}ni}|r||i}|dkrTt||dd tid d id ||i||<d |krt|}d}|jD]<\}}| dr|d|7}|j}d|i||i||<qd |kr6d|kr6|d7}| drzd||dd<n&|dkrz|t j d||i||<||fS)a\ Prepare the URL and the kwargs that must be passed to the HttpFileSystem or to requests.get/head In particular it resolves google drive URLs It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths. Storage options are formatted in the form {protocol: storage_options_for_protocol} NrnrfilecSs"i|]\}}|tkr||qSr,)rVZavailable_protocols)r+Z option_nameZ option_valuer,r,r-r.s z@_prepare_single_hop_path_and_storage_options..)httphttps)tokenz user-agent trust_envT)headersZ client_kwargszdrive.google.comZdownload_warningz &confirm=cookieszconfirm=z &confirm=tz"https://raw.githubusercontent.com/identityrzAccept-EncodingZhf)rZendpoint) rr>rRitemsrrrrrrrr) rNrOrr)rRresponserkvr,r,r-rsN           rrrP)rrOc Ost|}|d^}}t|r0t||f||St||d\}}||pHi}z"tj|f|d|i|} Wnltk r} zt| dkrtd| nW5d} ~ XYn2t k r| t j rt |ddnYnXt | | S)aExtend `open` function to support remote files using `fsspec`. It also has a retry mechanism in case connection fails. The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co Args: file (`str`): Path name of the file to be opened. mode (`str`, *optional*, default "r"): Mode in which the file is opened. *args: Arguments to be passed to `fsspec.open`. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs: Keyword arguments to be passed to `fsspec.open`. Returns: file object r<rPmodezCannot seek streaming HTTP filezStreaming is not possible for this dataset because data host server doesn't support HTTP range requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)Nr)rSr>rrrUrV ValueErrorr=r8rrrrr) rrrOr~rZfile_strrXrYrRrer,r,r-rjs4 "  rj)r@rOrec Cst|d^}}t|r$t|St||d\}}tj||d^}}|dd}|drv| |svt d|||}dd |DSd S) zExtend `os.listdir` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `list` of `str` r<rPrQrnrrozDirectory doesn't exist: cSsg|]}tj|dqS)name)r?r@r])r+objr,r,r- szxlistdir..N) rSr>rr?listdirrUrVrWrqrpr) r@rOrXrYrRrZr[rrobjectsr,r,r-xlistdirs   rF) recursiverOc st|d^}t|r(tj||dSt||d\}}tj||d^}}|dd}||}t|jt rt|jn|jdfdd |DSd S) aExtend `glob.glob` function to support remote files. Args: urlpath (`str`): URL path with shell-style wildcard patterns. recursive (`bool`, default `False`): Whether to match the "**" pattern recursively to zero or more directories or subdirectories. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `list` of `str` r<)rrPrQrnrrcs&g|]}dd|gqS)r<rn)rA)r+ globbed_pathr)rYr,r-r6szxglob..N) rSr>rglobrUrVrW isinstancer)r=) rNrrOrXrRrZr[rr globbed_pathsr,rr-xglobs  rc kst|d^}}t|r0tj|f|EdHnt||d\}}tj||d^}}|dd}|drx| |sxgSt |j t r|j n|j d} |j|f|D],\} } } d | d| g|| | fVqdS) auExtend `os.walk` function to support remote files. Args: urlpath (`str`): URL root path. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs: Additional keyword arguments forwarded to the underlying filesystem. Yields: `tuple`: 3-tuple (dirpath, dirnames, filenames). r<NrPrQrnrror)rSr>rr?walkrUrVrWrqrprr)r=rA) rNrOrrXrYrRrZr[rrr)dirpathdirnames filenamesr,r,r-xwalk9s rcseZdZdZfddZdeedddZd eeddd Zd d Z e dd d dZ e e d ddZ e e d ddZe e d ddZddZee dfddddZe ddddZfddZZS)!xPathzHExtension of `pathlib.Path` to support both local paths and remote URLs.csVt}|d^}}t|r$|S|dd}td|}||drLdnd7}|S)Nr<\rornrGrH)super__str__r>rreplace#SINGLE_SLASH_AFTER_PROTOCOL_PATTERNsubrK)selfZpath_strrXrYZ path_as_posix __class__r,r-rWs   z xPath.__str__NrPcCstt||dS)zExtend `pathlib.Path.exists` method to support both local and remote files. Args: download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` rP)r\r=)rrOr,r,r-rTas z xPath.existsc cs|}|d^}}t|r4t||EdHn|r~|d}t||d\}}|dd|i}d||f|dd}nd}tjt |||d^}} |t ||} | D]*} t |d|j d| g|VqdS)a]Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Args: pattern (`str`): Pattern that resulting paths must match. download_config : mainly use token or storage_options to support different platforms and auth types. Yields: [`xPath`] r<NrrPrnrrQ) rJr>rrrrUrArVrWrFtyper)) rpatternrOZ posix_pathrXrYrNrRrZr[rrr,r,r-rls z xPath.globcKs|jd|f|S)zRglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Args: pattern (`str`): Pattern that resulting paths must match. Yields: [`xPath`] z**/)r)rrrr,r,r-rglobs z xPath.rglobrcCst|t|S)zName function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: [`xPath`] )rrMrJrr,r,r-parentsz xPath.parentcCst|ddjS)zName function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r<r)rrJr>rrr,r,r-rsz xPath.namecCst|ddjS)zStem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r<r)rrJr>stemrr,r,r-rsz xPath.stemcCst|ddjS)zSuffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r<r)rrJr>suffixrr,r,r-rsz xPath.suffixcOstt|f||S)aExtend :func:`xopen` to support argument of type :obj:`~pathlib.Path`. Args: **args: Arguments passed to :func:`fsspec.open`. **kwargs: Keyword arguments passed to :func:`fsspec.open`. Returns: `io.FileIO`: File-like object. )rjr=)rr~rr,r,r-rs z xPath.open.)rDrecGst|t|f|S)zExtend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`. Args: *p (`tuple` of `str`): Other path components. Returns: [`xPath`] )rrFrJrrDr,r,r-joinpaths zxPath.joinpathcCs ||Sr4)rrr,r,r- __truediv__szxPath.__truediv__cs`t|d^}}t|r2t|tt|St|dt|t||g|S)Nr<) r=r>rrr with_suffixrArrJ)rrrXrYrr,r-rszxPath.with_suffix)N)N)r9r:r;__doc__rr rrTrrpropertyrr=rrrrrrrr __classcell__r,r,rr-rTs"      rr@cCs"t|trt|Sttt|Sr4)rrr=rr,r,r-rSsrScOsLddl}t|dr$|j|f||St|}|jt|d|df||SdSNrrkrbrP)r2hasattrrr=rj)filepath_or_bufferrOr~rr2r,r,r- xgzip_opens  rcOsLddl}t|dr$|j|f||St|}|jt|d|df||SdSr)numpyrloadr=rj)rrOr~rnpr,r,r- xnumpy_loads  rcKsdddl}t|dr |j|f|St|}|dddkrHt||d|d<|jt|d|df|SdS)NrrkrZinferrPr)pandasrZread_csvr=rrrjrrOrpdr,r,r-xpandas_read_csvs rc Ksddl}t|drPz|j|f|WStk rL|jt|f|YSXnXt|}z|jt|d|df|WStk r|jtt|d|df|YSXdSr)rrZ read_excelrrrkr=rjrr,r,r-xpandas_read_excels rcKs@ddlm}t|dr$|j|f|S|jt|d|df|SdSr)Zscipy.iorrZloadmatrj)rrOrsior,r,r- xsio_loadmats  rc CsLt|drtj||dSt|d|d}tj||dW5QRSQRXdS)aExtend `xml.etree.ElementTree.parse` function to support remote files. Args: source: File path or file object. parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `xml.etree.ElementTree.Element`: Root element of the given source document. rk)parserrrPN)rETparserj)sourcerrOrlr,r,r- xet_parses rc KsTt|drtjjj|f|St|d|d }tjjj|f|W5QRSQRXdS)aExtend `xml.dom.minidom.parse` function to support remote files. Args: filename_or_file (`str` or file): File path or file object. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`. Returns: :obj:`xml.dom.minidom.Document`: Parsed document. rkrrPN)rr%domZminidomrrj)Zfilename_or_filerOrrlr,r,r-xxml_dom_minidom_parse-s rc@s&eZdZdZedddZddZdS)_IterableFromGeneratorzkUtility class to create an iterable from a generator function, in order to reset the generator when needed.) generatorcOs||_||_||_dSr4rr~r)rrr~rr,r,r-__init__Bsz_IterableFromGenerator.__init__ccs|j|j|jEdHdSr4rrr,r,r-__iter__Gsz_IterableFromGenerator.__iter__N)r9r:r;rrrrr,r,r,r-r?src@seZdZdZeddZeddZeee ddfddd Z ede e e ee ddfd d d Zeddd dZede e ddddZdS)ArchiveIterablezIAn iterable of (path, fileobj) from a TAR archive, used by `iter_archive`ccsftj|dd}|D]L}|j}|s&q|dkr0qtj|drDq||}||fVg|_ q~dS)Nzr|*)fileobjrr&__) tarfilerrisregr?r@r]r extractfilemembers)rlstreamtarinfo file_pathrr,r,r- _iter_tarNs  zArchiveIterable._iter_tarccs^t|}|D]F}|j}|r&q|dkr0qtj|drDq| |}||fVqdS)Nr) zipfileZipFileinfolistfilenameis_dirr?r@r]rr)rlZzipfmemberr rr,r,r- _iter_zip_s   zArchiveIterable._iter_zipNrccs6t|}|dkr"||EdHn||EdHdS)Nr/)rrr )clsrlrr,r,r-_iter_from_fileobjnsz"ArchiveIterable._iter_from_fileobjrc csTt||d}t|d|d0}|dkr6||EdHn||EdHW5QRXdS)NrPrr/)rrjrr )rrNrOrrlr,r,r-_iter_from_urlpathvs  z"ArchiveIterable._iter_from_urlpathcCs ||j|Sr4)r)rrr,r,r-from_bufszArchiveIterable.from_bufrdcCs||j||Sr4)r)rurlpath_or_bufrOr,r,r- from_urlpathszArchiveIterable.from_urlpath)N)N)r9r:r;r staticmethodr r classmethodr rrr=r rrrrr,r,r,r-rKs$    rc@s\eZdZdZed eeeefee e eddfdddZ ed ee ddddZ dS) FilesIterablez8An iterable of paths from a list of directories or filesN)urlpathsrOreccst|ts|g}|D]}t||dr.) rlistrgr^rrsrsortedrFr)rrrOrNrrrrr,r,r-_iter_from_urlpathss"     z!FilesIterable._iter_from_urlpathsrdcCs||j||Sr4)r)rrrOr,r,r- from_urlpathsszFilesIterable.from_urlpaths)N)N) r9r:r;rrrr=r r rr rrr,r,r,r-rs rc@seZdZdZdZdeeeeeeeedddZe ddZ d d Z eed d d Z ddZ eed ddZddZeeejfeedddZeeeefeedddZdS)StreamingDownloadManagera Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives. Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract data, but they rather return the path or url that could be opened using the `xopen` function which extends the built-in `open` function to stream data from remote files. TN) dataset_namedata_dirrO base_pathcCs.||_||_|ptjd|_|p&t|_dS)Nr&)Z _dataset_name _data_dirr?r@abspath _base_pathrrO)rr!r"rOr#r,r,r-rsz!StreamingDownloadManager.__init__cCs|jSr4)r$rr,r,r- manual_dirsz#StreamingDownloadManager.manual_dircCst|j|dd}|S)aUNormalize URL(s) of files to stream data from. This is the lazy version of `DownloadManager.download` for streaming. Args: url_or_urls (`str` or `list` or `dict`): URL(s) of files to stream data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') ``` TZ map_tuple)r _downloadr url_or_urlsr,r,r-downloadsz!StreamingDownloadManager.download)rNrecCs t|}t|rt|j|}|Sr4)r=rrr&)rrNr,r,r-r)s z"StreamingDownloadManager._downloadcCst|j|dd}|S)aAdd extraction protocol for given url(s) for streaming. This is the lazy version of `DownloadManager.extract` for streaming. Args: url_or_urls (`str` or `list` or `dict`): URL(s) of files to stream data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> extracted_files = dl_manager.extract(downloaded_files) ``` Tr()r_extract)rr+rr,r,r-extractsz StreamingDownloadManager.extractcCst|}t||jd}|dd}t|}|dks>|drNtd|d|dkrZ|S|tkrtj |dd}d|kr|d| dn|}|d |d|S|d |SdS) NrPr<rrrz+Extraction protocol for TAR archives like 'z' is not implemented in streaming mode. Please use `dl_manager.iter_archive` instead. Example usage: url = dl_manager.download(url) tar_archive_iterator = dl_manager.iter_archive(url) for filename, file in tar_archive_iterator: ...r&rnz://::) r=rrOr>rrKr!SINGLE_FILE_COMPRESSION_PROTOCOLSr?r@r]rindex)rrNr)r@r'Z inner_filer,r,r-r-s  z!StreamingDownloadManager._extractcCs|||S)a0Prepare given `url_or_urls` for streaming (add extraction protocol). This is the lazy version of `DownloadManager.download_and_extract` for streaming. Is equivalent to: ``` urls = dl_manager.extract(dl_manager.download(url_or_urls)) ``` Args: url_or_urls (`str` or `list` or `dict`): URL(s) to stream from data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. )r.r,r*r,r,r-download_and_extractsz-StreamingDownloadManager.download_and_extract)rrecCs(t|drt|Stj||jdSdS)aNIterate over files within an archive. Args: urlpath_or_buf (`str` or `io.BufferedReader`): Archive path or archive binary file object. Yields: `tuple[str, io.BufferedReader]`: 2-tuple (path_within_archive, file_object). File object is opened in binary mode. Example: ```py >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> files = dl_manager.iter_archive(archive) ``` rkrPN)rrrrrO)rrr,r,r- iter_archive)s  z%StreamingDownloadManager.iter_archive)rrecCstj||jdS)aIterate over files. Args: urlpaths (`str` or `list` of `str`): Root paths. Yields: str: File URL path. Example: ```py >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') >>> files = dl_manager.iter_files(files) ``` rP)rrrO)rrr,r,r- iter_filesBsz#StreamingDownloadManager.iter_files)NNNN)r9r:r;rZ is_streamingr r=rrrr'r,r)r.r-r1rrBufferedReaderr rr2r r3r,r,r,r-r s*  r )N)N)N)N)N)N)N)N)r)N)N)N)N)N)NN)N)grrr?rBrerr{Zxml.dom.minidomr%r asynciorr itertoolsrpathlibrrtypingrrr r r r r rrZ xml.etreerrrVZaiohttp.client_exceptionsrrrZ filesystemsrZutils.file_utilsrrrrrrZ utils.loggingrZutils.py_utilsrrOrr9ryrrr/compilerbytesfromhexrrmaxr Exceptionr8rFrMr=r\r^r`rcboolrgintrmrsrvrrrrrUrrjrrrrrrSrrrrrrrrrrr r,r,r,r-s   ,               9-      ?%