U
    -e:!                     @   s   d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
Zd dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZmZ eeZdd ZG dd deZdS )    N)ArgumentParser)Path)copyfilermtree)	Generator)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)dataset_module_factoryimport_main_class)logger)VerificationMode)ERROR
get_loggerc              
   C   s0   t | j| j| j| j| j| jp | j| j| j	| j
	S )N)TestCommanddatasetname	cache_dirdata_dirall_configsZ	save_info
save_infosignore_verificationsforce_redownloadclear_cache)args r   W/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/commands/test.py_test_command_factory   s    
r   c                
   @   sH   e Zd ZdZeedddZeeeeeeeeed	ddZ	dd	 Z
d
S )r   F)parserc                 C   s   | j ddd}|jdtd dd |jdtd dd |jd	td d
d |jdddd |jdddd |jdddd |jdddd |jdddd |jdddd |jdtdd |jtd d S )NtestzTest dataset implementation.)helpz--namezDataset processing name)typedefaultr    z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr    z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz--save_infoszalias to save_infor   zName of the dataset to download)r!   r    )func)
add_parseradd_argumentstrset_defaultsr   )r   Ztest_parserr   r   r   register_subcommand%   sD      zTestCommand.register_subcommand)	r   r   r   r   r   r   r   r   r   c
           
      C   s\   || _ || _|| _|| _|| _|| _|| _|| _|	| _|	rN|sNt	d t
d |rXd| _d S )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset script.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cacheprintexit)
selfr   r   r   r   r   r   r   r   r   r   r   r   __init__I   s     zTestCommand.__init__c              	      sZ  t  t jd k	r,jr,td td jj }t|t	j
 jrb jrbt jnd}ttd d f d fdd}t| D ]\}}td|jj d|d  d| d	 tjtj| tjj|_|jjstjntjjrt j!nt j"d
d |#  j$r&|$  j$rtj  d}t%|jd }tj||}tj&|rxtj'|}	n:tj&|r|}	n&tj(|r|}	nd }	td|  |	d k	rtj|	d}
t)||
 td|
  j*rtj(|j+rt,-d|j+  t.|j+ tjj+tjj/}tj(|rt,-d|  t.| qtd d S )NzABoth parameters `config` and `all_configs` can't be used at once.r+   )returnc                  3   s   j rj jrjt jD ]P\} }djkrF f jjdjV  q f |jjjdjV  qnFdjkr f jjdjV  n f jjdjV  d S )Nconfig_name)r   r   )r:   r   r   )r0   BUILDER_CONFIGS	enumerateZbuilder_kwargsr.   r/   r   )iconfigZbuilder_clsr:   moduler7   r   r   get_buildersr   s2    


z%TestCommand.run.<locals>.get_builderszTesting builder 'z' (/)F)Zdownload_modeZverification_modeZtry_from_hf_gcsz	README.mdz.pyzDataset card saved at zClearing cache at zTest successful.)0	fl_loggersetLevelr   r-   r0   r5   r6   r,   r
   r   module_pathr;   lenr   r   r<   r>   r   ospathexistsjoinZget_imported_module_dirdatasetsZDATASETDICT_INFOS_FILENAMEZ_record_infosZdownload_and_preparer3   r	   ZREUSE_CACHE_IF_EXISTSZFORCE_REDOWNLOADr2   r   Z	NO_CHECKSZ
ALL_CHECKSZ
as_datasetr1   r   isfiledirnameisdirr   r4   r.   r   warningr   ZDOWNLOADED_DATASETS_DIR)r7   rI   Z
n_buildersrA   jZbuilderZdataset_readme_pathr   Zcombined_pathZdataset_dirZuser_dataset_readme_pathdownload_dirr   r?   r   runh   sf    
"$
	



zTestCommand.runN)__name__
__module____qualname__Z__test__staticmethodr   r*   r(   boolr8   rS   r   r   r   r   r   "   s   %r   ) rH   argparser   pathlibr   shutilr   r   typingr   Zdatasets.configrL   Zdatasets.builderr   Zdatasets.commandsr   Z"datasets.download.download_managerr	   Zdatasets.loadr
   r   Zdatasets.utils.filelockr   rD   Zdatasets.utils.info_utilsr   Zdatasets.utils.loggingr   r   rT   r   r   r   r   r   r   <module>   s   