U -e[ @sdZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z m Z ddlZddlmZddlmZddlmZdd lmZdd lmZmZmZmZmZdd lmZdd lm Z dd l!m"Z"ddl#m$Z$e"e%Z&Gddde'Z(edd-ddZ)edd.ddZ*edd/e+e+e edddZ,edd0e+e+e edddZ-d1e+e e ee e+fe ee e ee+fe e e+e$fe e e.e+fd!d"d#Z/d2e+e e e+e$fe ee e ee+fe e+e e ee e+fd$d%d&Z0d3e+e e+e e e+e e+e e+e e+e e+fffe ee e ee+fe e e+e$fe e e.e+fed'd(d)Z1d4e+e e+e e e+e e+e e+e e+e e+fffe ee e ee+fe e e+e$fe e e.e+fd*d+d,Z2dS)5z List and inspect datasets.N)PurePath)DictListMappingOptionalSequenceUnion)DownloadConfig) DownloadMode)StreamingDownloadManager) DatasetInfo)dataset_module_factoryget_dataset_builder_classimport_main_classload_dataset_buildermetric_module_factory) deprecated)relative_to_absolute_path) get_logger)Versionc@s eZdZdS)SplitsNotFoundErrorN)__name__ __module__ __qualname__rrQ/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/inspect.pyr/srz,Use 'huggingface_hub.list_datasets' instead.TFcCs8tj|d}|sdd|D}|s0dd|D}t|S)a]List all the datasets scripts available on the Hugging Face Hub. Args: with_community_datasets (`bool`, *optional*, defaults to `True`): Include the community provided datasets. with_details (`bool`, *optional*, defaults to `False`): Return the full details on the datasets instead of only the short name. Example: ```py >>> from datasets import list_datasets >>> list_datasets() ['acronym_identification', 'ade_corpus_v2', 'adversarial_qa', 'aeslc', 'afrikaans_ner_corpus', 'ag_news', ... ] ``` )fullcSsg|]}d|jkr|qS/id.0datasetrrr Ns z!list_datasets..cSsg|] }|jqSrr r"rrrr%Ps)huggingface_hub list_datasetslist)Zwith_community_datasets with_detailsZdatasetsrrrr'3s  r'uxUse 'evaluate.list_evaluation_modules' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluatecCs0t}|sdd|D}|s,dd|D}|S)uList all the metrics script available on the Hugging Face Hub. Use `evaluate.list_evaluation_modules` instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate Args: with_community_metrics (:obj:`bool`, optional, default ``True``): Include the community provided metrics. with_details (:obj:`bool`, optional, default ``False``): Return the full details on the metrics instead of only the short name. Example: ```py >>> from datasets import list_metrics >>> list_metrics() ['accuracy', 'bertscore', 'bleu', 'bleurt', 'cer', 'chrf', ... ] ``` cSsg|]}d|jkr|qSrr r#Zmetricrrrr%us z list_metrics..cSsg|] }|jqSrr r*rrrr%ws)r& list_metrics)Zwith_community_metricsr)Zmetricsrrrr+Ts r+z?Clone the dataset repository from the Hugging Face Hub instead.)path local_pathdownload_configc Kst|fd|i|}t|}t|}tj|}t|D]|\}} } tj|tj ||} tj | dddd| D| dd<| D]$} t tj|| tj| | qt || q.Nz"The processing script for dataset  can be inspected at . The main class is in zP. You can modify this processing script and use it with `datasets.load_dataset("")`.)rrinspect getsourcefileosr,r7walkjoinrelpathmakedirsshutilcopy2copystatrprintras_posix) r,r-r.download_kwargsdataset_module builder_clsmodule_source_pathmodule_source_dirpathdirpathdirnames filenames dst_dirpathfilenamerrrinspect_dataset{s  "$rQuzUse 'evaluate.inspect_evaluation_module' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluatec Kst|fd|i|}t|jdd}t|}tj|}t|D]|\}} } tj |tj ||} tj | dddd| D| dd<| D]$} t tj || tj | | qt || qBt|}td |d |d |d t|d dS)u Allow inspection/modification of a metric script by copying it on local drive at local_path. Use `evaluate.inspect_evaluation_module` instead, from the new library 🤗 Evaluate instead: https://huggingface.co/docs/evaluate Args: path (``str``): path to the dataset processing script with the dataset builder. Can be either: - a local path to processing script or the directory containing the script (if the script has the same name as the directory), e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'`` - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with ``datasets.list_datasets()``) e.g. ``'squad'``, ``'glue'`` or ``'openai/webtext'`` local_path (``str``): path to the local folder to copy the datset script to. download_config (Optional ``datasets.DownloadConfig``): specific download configuration parameters. **download_kwargs (additional keyword arguments): optional attributes for DownloadConfig() which will override the attributes in download_config if supplied. r.F)r$Tr/cSsg|]}|ds|qSr1r4r6rrrr%s z"inspect_metric..Nz"The processing scripts for metric r8r9zP. You can modify this processing scripts and use it with `datasets.load_metric("r:)rr module_pathr;r<r=r,r7r>r?r@rArBrCrDrrErrF) r,r-r.rGZ metric_moduleZ metric_clsrJrKrLrMrNrOrPrrrinspect_metrics  "$rSr)r, data_filesr. download_moderevisiontokenc  sR|dkr td|dt|td}fdd|DS)a! Get the meta information about a dataset, returned as a dict mapping config name to DatasetInfoDict. Args: path (`str`): path to the dataset processing script with the dataset builder. Can be either: - a local path to processing script or the directory containing the script (if the script has the same name as the directory), e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'` - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`datasets.list_datasets`]) e.g. `'squad'`, `'glue'` or``'openai/webtext'` revision (`Union[str, datasets.Version]`, *optional*): If specified, the dataset module will be loaded from the datasets repository at this version. By default: - it is set to the local version of the lib. - it will also try to load it from the main branch if it's not available at the local version of the lib. Specifying a version that is different from your local version of the lib might cause compatibility issues. download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): Download/generate mode. data_files (`Union[Dict, List, str]`, *optional*): Defining the data_files of the dataset configuration. token (`str` or `bool`, *optional*): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. If `True`, or not specified, will get token from `"~/.huggingface"`. use_auth_token (`str` or `bool`, *optional*): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. If `True`, or not specified, will get token from `"~/.huggingface"`. `use_auth_token` was deprecated in favor of `token` in version 2.14.0 and will be removed in 3.0.0. **config_kwargs (additional keyword arguments): Optional attributes for builder class which will override the attributes if supplied. Example: ```py >>> from datasets import get_dataset_infos >>> get_dataset_infos('rotten_tomatoes') {'default': DatasetInfo(description="Movie Review Dataset. This is a dataset of containing 5,331 positive and 5,331 negative processed sentences from Rotten Tomatoes movie reviews...), ...} ``` r'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0. You can remove this warning by passing 'token= ' instead.)r,rVr.rUrTrWc s,i|]$}|tf|dqS)r, config_namerTr.rUrVrW)get_dataset_config_info)r#r[ config_kwargsrTr.rUr,rVrWrr s z%get_dataset_infos..)warningswarn FutureWarningget_dataset_config_names) r,rTr.rUrVrWuse_auth_tokenr^Z config_namesrr]rget_dataset_infoss"7  re)r,rVr.rUdynamic_modules_pathrTc KsNt|f|||||d|}t|tj|d}t|jpL|j ddgS)aGet the list of available config names for a particular dataset. Args: path (`str`): path to the dataset processing script with the dataset builder. Can be either: - a local path to processing script or the directory containing the script (if the script has the same name as the directory), e.g. `'./dataset/squad'` or `'./dataset/squad/squad.py'` - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with [`datasets.list_datasets`]) e.g. `'squad'`, `'glue'` or `'openai/webtext'` revision (`Union[str, datasets.Version]`, *optional*): If specified, the dataset module will be loaded from the datasets repository at this version. By default: - it is set to the local version of the lib. - it will also try to load it from the main branch if it's not available at the local version of the lib. Specifying a version that is different from your local version of the lib might cause compatibility issues. download_config ([`DownloadConfig`], *optional*): Specific download configuration parameters. download_mode ([`DownloadMode`] or `str`, defaults to `REUSE_DATASET_IF_EXISTS`): Download/generate mode. dynamic_modules_path (`str`, defaults to `~/.cache/huggingface/modules/datasets_modules`): Optional path to the directory in which the dynamic modules are saved. It must have been initialized with `init_dynamic_modules`. By default the datasets and metrics are stored inside the `datasets_modules` module. data_files (`Union[Dict, List, str]`, *optional*): Defining the data_files of the dataset configuration. **download_kwargs (additional keyword arguments): Optional attributes for [`DownloadConfig`] which will override the attributes in `download_config` if supplied, for example `token`. Example: ```py >>> from datasets import get_dataset_config_names >>> get_dataset_config_names("glue") ['cola', 'sst2', 'mrpc', 'qqp', 'stsb', 'mnli', 'mnli_mismatched', 'mnli_matched', 'qnli', 'rte', 'wnli', 'ax'] ``` )rVr.rUrfrT) dataset_namer[default) rrr=r,basenamer(Zbuilder_configskeysZbuilder_kwargsget) r,rVr.rUrfrTrGrHrIrrrrc's8 rc)r,r[rTr.rUrVrWreturnc s|dkr td|dt|}tf||||||d|} | j} | jdkr|rZ|nt}|dk rn||_| t | j |dz(fdd| t | j |dD| _Wn,t k r} ztd | W5d} ~ XYnX| S) aGet the meta information (DatasetInfo) about a dataset for a particular config Args: path (``str``): path to the dataset processing script with the dataset builder. Can be either: - a local path to processing script or the directory containing the script (if the script has the same name as the directory), e.g. ``'./dataset/squad'`` or ``'./dataset/squad/squad.py'`` - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with ``datasets.list_datasets()``) e.g. ``'squad'``, ``'glue'`` or ``'openai/webtext'`` config_name (:obj:`str`, optional): Defining the name of the dataset configuration. data_files (:obj:`str` or :obj:`Sequence` or :obj:`Mapping`, optional): Path(s) to source data file(s). download_config (:class:`~download.DownloadConfig`, optional): Specific download configuration parameters. download_mode (:class:`DownloadMode` or :obj:`str`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. revision (:class:`~utils.Version` or :obj:`str`, optional): Version of the dataset script to load. As datasets have their own git repository on the Datasets Hub, the default version "main" corresponds to their "main" branch. You can specify a different version than the default "main" by using a commit SHA or a git tag of the dataset repository. token (``str`` or :obj:`bool`, optional): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. If True, or not specified, will get token from `"~/.huggingface"`. use_auth_token (``str`` or :obj:`bool`, optional): Optional string or boolean to use as Bearer token for remote files on the Datasets Hub. If True, or not specified, will get token from `"~/.huggingface"`. `use_auth_token` was deprecated in favor of `token` in version 2.14.0 and will be removed in 3.0.0. **config_kwargs (additional keyword arguments): optional attributes for builder class which will override the attributes if supplied. rrXrY)namerTr.rUrVrWN) base_pathr.csi|]}|j|jdqS))rmrg)rm)r#Zsplit_generatorr,rrr_sz+get_dataset_config_info..z `use_auth_token` was deprecated in favor of `token` in version 2.14.0 and will be removed in 3.0.0. **config_kwargs (additional keyword arguments): Optional attributes for builder class which will override the attributes if supplied. Example: ```py >>> from datasets import get_dataset_split_names >>> get_dataset_split_names('rotten_tomatoes') ['train', 'validation', 'test'] ``` rrXrY)r[rTr.rUrVrW)r`rarbr\r(rqrj) r,r[rTr.rUrVrWrdr^rprrrget_dataset_split_namess&8  ru)TF)TF)N)N)NNNNNr)NNNNN)NNNNNNr)NNNNNNr)3__doc__r;r=rBr`pathlibrtypingrrrrrrr&Zdownload.download_configr Zdownload.download_managerr Z#download.streaming_download_managerr rpr loadrrrrrZutils.deprecation_utilsrZutils.file_utilsrZ utils.loggingrZ utils.versionrrlogger ValueErrorrr'r+strrQrSboolrercr\rurrrrs            $)+ X G( Q(