U -eBZ@sddlZddlZddlZddlZddlZddlmmZddl m Z ddl m Z ddl mZddlmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZdd lmZm Z dd l!m"Z"ee#Z$dZ%ddZ&GdddeZ'edGdddeZ(dS)N)ArgumentParser)Path)Optional)config)BaseDatasetsCLICommand)DownloadConfig)DownloadManager)MockDownloadManager)dataset_module_factoryimport_main_class) deprecated) get_loggerset_verbosity_warning) map_nestedzutf-8c Cs*t|j|j|j|j|j|j|j|j|j SN) DummyDataCommandpath_to_dataset auto_generaten_lines json_fieldxml_tagmatch_text_fileskeep_uncompressed cache_direncoding)argsr]/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/commands/dummy_data.pydummy_data_command_factorysrc seZdZfddZfddZfddZdeeeeeeeeee d d d Z deeeeeeeeeeeed d dZ e de fddZddZZS)!DummyDataGeneratorDownloadManagercs$tj||||_g|_g|_dSr)super__init__mock_download_managerdownloaded_dummy_pathsexpected_dummy_paths)selfr"rkwargs __class__rrr!*sz*DummyDataGeneratorDownloadManager.__init__cs@t|}|j|}t|jj|ddt|jj|dd|SNT)Z map_tuple)r downloadr"rr#appendr$r%Z url_or_urlsoutputZ dummy_outputr'rrr*0s   z*DummyDataGeneratorDownloadManager.downloadcsHtt|}|j|}t|jj|ddt|jj|dd|Sr))r extractr*r"rr#r+r$r,r'rrdownload_and_extract7s  z6DummyDataGeneratorDownloadManager.download_and_extractN)rrrrrreturnc Cstjtj|jj|jj|jjdddd}d|j_t |j |j D]B\}}tj|jj|jj|jj|} ||j || |||||d7}qB|dkrt d|dkS)N dummy_dataTexist_okrFrrrrrzDummy data generation failed: no dummy files were created. Make sure the data files format is supported by the auto-generation.)osmakedirspathjoinr"datasets_scripts_dir dataset_namedummy_data_folderload_existing_dummy_datazipr#r$_create_dummy_dataloggererror) r%rrrrrtotalsrc_pathZrelative_dst_pathdst_pathrrrauto_generate_dummy_data_folder>s@  zADummyDataGeneratorDownloadManager.auto_generate_dummy_data_folder)rCrDrrrrrr1c s|pt}tj|r`td|t|jddddg}tfdd|D} |dk rtj |} | dD]} | t | | O} qp| rt|j j d d d t||d `} t|d |d F} g}t| D]\}}|krq||q| d |W5QRXW5QRXdSdkrt||d } t| }|dk rJ||}t|trtdd|Dstdt|dfdd|D}n |d}|dk r||i}t|j j d d d t|d |d } t|| W5QRXW5QRXdStfdddDrJ|dkr2tdn|j ||||ddStd|ddStj!|rd}t"|D]f\}}}|D]T}|#dstj||}tj|t|$|}||j%||||||d7}qq||SdS)Nz#Trying to generate dummy data file z.txtz.csvz.jsonlz.tsvc3s|]}|kVqdSrr.0 extensiondst_path_extensionsrr wszGDummyDataGeneratorDownloadManager._create_dummy_data..,Tr4parentsrwz.jsoncss|]}t|tVqdSr) isinstancelist)rGvrrrrKszCouldn't parse columns z\. Maybe specify which json field must be used to read the data with --json_field .csi|]\}}||dqSrr)rGkrU)rrr szHDummyDataGeneratorDownloadManager._create_dummy_data..c3s|]}|kVqdSrrrFrIrrrKs)z.xmlz.txmzEFound xml file but 'xml_tag' is set to None. Please provide --xml_tag)rrzCouldn't generate dummy file 'z9'. Ignore that if this file is not useful for dummy data.r.r5)&DEFAULT_ENCODINGr6r8isfiler@debugrsuffixesanybasenamesplitfnmatchparentmkdiropen enumerater+writer9stripjsonloadrSdictallvalues ValueErrorrTkeysitemsdumpwarning_create_xml_dummy_dataisdirwalk startswith relative_tor?)r%rCrDrrrrrZline_by_line_extensionsZis_line_by_line_text_file file_namepatternsrc_fileZdst_fileZ first_linesilineZ json_dataZfirst_json_datarBr8_filesnameZ src_file_pathZ dst_file_pathr)rJrrr?hs     (            z4DummyDataGeneratorDownloadManager._create_dummy_datac Cst|jjdddt||d}d}g}tj|ddD]R\}} |dkrT|| q8|} | j|kr8||krx|d7}q8|r8|d  | q8tj | d j ||dW5QRXdS) NTrMrOr)startend)eventsr~rR)element) rrarbrcETZ iterparser+poptagremove ElementTreere) rCrDrrrrxZn_linerNeventelemr{rrrrqs   z8DummyDataGeneratorDownloadManager._create_xml_dummy_datacCsTtj||jj}tj|d}d}td|dt|d||t |dS)Nr2z"Compressing dummy data folder to 'z.zip'r>) r6r8r9r"r<r@infoshutil make_archivermtree)r%rroot_dir base_namebase_dirrrr!compress_autogenerated_dummy_datas zCDummyDataGeneratorDownloadManager.compress_autogenerated_dummy_data)r0NNNN)NNNN)__name__ __module__ __qualname__r!r*r/intrstrboolrEr? staticmethodrYrqr __classcell__rrr'rr)sB    / TrzThe `datasets` repository does not host the dataset scripts anymore. Therefore, dummy data is no longer needed to test their loading with CI.c @sreZdZeedddZeeee ee ee eee ee ed ddZ ddZ e ed d d Z d d Z dS)r)parsercCs|jddd}|jdddd|jdtd d d |jd tdd d |jdtddd |jdtddd |jdddd|jdtddd |jdtddtd |jdtdd|jtddS)Nr2zGenerate dummy data.)helpz--auto_generate store_truez!Automatically generate dummy data)actionrz --n_linesr0zBNumber of lines or samples to keep when auto-generating dummy data)typedefaultrz --json_fieldzOptional, json field to read the data from when auto-generating dummy data. In the json data files, this field must point to a list of samples as json objects (ex: the 'data' field for squad-like files)z --xml_tagz[Optional, xml tag name of the samples inside the xml files when auto-generating dummy data.z--match_text_fileszOptional, a comma separated list of file patterns that looks for line-by-line text files other than *.txt or *.csv. Example: --match_text_files *.labelz--keep_uncompressedzWhether to leave the dummy data folders uncompressed when auto-generating dummy data. Useful for debugging for to do manual adjustements before compressing.z --cache_dirzKCache directory to download and cache files when auto-generating dummy dataz --encodingz=Encoding to use when auto-generating dummy data. Defaults to rz/Path to the dataset (example: ./datasets/squad))rr)func) add_parser add_argumentrrrY set_defaultsr)rZ test_parserrrrregister_subcommandsZz$DummyDataCommand.register_subcommand) rrrrrrrrrc Cs||_tj|r.|tjddd|_n|tjddd|_tj|pVt j }||_ ||_ ||_ ||_||_||_||_| |_dS)N/r)_path_to_datasetr6r8rrreplacesepr_ _dataset_name expanduserrZHF_DATASETS_CACHE_auto_generate_n_lines _json_field_xml_tag_match_text_files_keep_uncompressed _cache_dir _encoding) r%rrrrrrrrrrrrr! s  zDummyDataCommand.__init__c Cstt|j}t|j}|jp$dg}g}t}|D]v}|rF|jnd}|||j |d}|rd|j n|j j } t |j || ddd} |jr||j|| |jdq8|j|| dq8|jr|jst|rtd|jdntd |jdW5QRXdS) N) config_namehashrTF)r;rversionZuse_local_dummy_datar=)dataset_buildermock_dl_managerr)rrz>Automatic dummy data generation succeeded for all configs of ''z.zEDummy data generation done and dummy data test succeeded for config 'z''.csg|]}|dkr|qS)rr)rGZ split_nameZn_examples_per_splitrr ws z=DummyDataCommand._autogenerate_dummy_data..zCDummy data generation done but dummy data test failed since splits z have 0 examples for config 'z#Dummy data generated in directory 'zg' but kept uncompressed. Please compress this directory into a zip file to use it for dummy data tests.)$rr6r8r9rZDOWNLOADED_DATASETS_DIRZDOWNLOADED_DATASETS_PATHrrr_split_generatorsr=rErrrrrr:r;rr7Z_prepare_splitZ split_infoZ num_examplesr}OSErrorr@rArrjrkrprr<r) r%rrrZ dl_cache_dirrZ dl_managerZpath_do_datasetZsplit_generatorsZsplit_generatoreZ empty_splitsZgenerated_dummy_data_dirrrrrKsj      z)DummyDataCommand._autogenerate_dummy_datac Cstj|j|j}td|dtj|ddz||}WnDt k r}z&t d|j d|j d|j dW5d}~XYnXt}g}|j}|D]} td | j|| j| j} |jf| } zjd } |j dk rd |j jd nd } | d| |j d|d|d7} | D] \}}q| d|d7} Wqt k rh}z||j W5d}~XYqXqd|}t|dkrt|dkrtt||kr| dtt|d|d|d7} |}n0d|}| d|d|d7} | d|d7} | d|d 7} t|dkr~tt||kr~| d!|d"|d#|d$7} | d%|d&|d$7} | d'|d(|d)|d*7} nN| d+|d,|d#|d-7} | d.|d/|d$7} | d0|d(|d)|d*7} | d1|d2|d37} | d47} t | dS)5Nz$Creating dummy folder structure for z... Tr3zDataset z with config a seems to already open files in the method `_split_generators(...)`. You might consider to instead only open files in the method `_generate_examples(...)` instead. If this is not possible the dummy data has to be created with less guidance. Make sure you create the file rXz/Collecting dummy data file paths to create for zU ==============================DUMMY DATA INSTRUCTIONS============================== zconfig z of rQz(- In order to create the dummy data for z, please go into the folder 'z ' with `cd z` . za- It appears that the function `_generate_examples(...)` expects one or more files in the folder z using the function `glob.glob(...)`. In this case, please refer to the `_generate_examples(...)` method to see under which filename the dummy data files should be created. z, rrRz1- Please create a single dummy data file called 'z' from the folder 'zV'. Make sure that the dummy data file provides at least one example for the split(s) 'z' z0- Please create the following dummy data files 'z' z- For each of the splits 'zU', make sure that one or more of the dummy data files provide at least one example z- If the method `_generate_examples(...)` includes multiple `open()` statements, you might have to create other files in addition to 'zG'. In this case please refer to the `_generate_examples(...)` method z@- After the dummy data file is created, it should be zipped to 'z.zip' with the command `zip z.zip z` z- You can now delete the file 'z' with the command `rm z- To get the file 'z;' back for further changes to the dummy data, simply unzip z.zip with the command `unzip z.zip` zP- After all dummy data files are created, they should be zipped recursively to 'z.zip' with the command `zip -r z/` z!- You can now delete the folder 'z' with the command `rm -r z- To get the folder 'z'- Make sure you have created the file 'z .zip' in 'z' zT=================================================================================== )r6r8r9rr<r@rr7rFileNotFoundErrorrrrfilenamesetdummy_file_namer}r+ gen_kwargsZ_generate_examplesaddlennextiter)r%rrr<Zgenerator_splitsrZfiles_to_createZ split_namesrr_r generatorZdummy_data_guidance_printZ config_stringkeyrecordZ files_stringrrrrst      $  z/DummyDataCommand._print_dummy_data_instructionsN)rrrrrrrrrrr!rrrrrrrrs. ':r))r`rgr6rrZxml.etree.ElementTreeetreerrargparserpathlibrtypingrZdatasetsrZdatasets.commandsrZ!datasets.download.download_configrZ"datasets.download.download_managerrZ'datasets.download.mock_download_managerr Z datasets.loadr r Z datasets.utils.deprecation_utilsr Zdatasets.utils.loggingr rZdatasets.utils.py_utilsrrr@rYrrrrrrrs4          0