U
    -e(                     @   s   d Z ddlZddlZddlZddlmZ ddlmZm	Z	m
Z
mZ ddlmZ ddlmZmZ ddlmZ dd	lmZ eeZG d
d dZdS )z Mock download manager interface.    N)Path)CallableListOptionalUnion)ZipFile   )cached_pathhf_github_url)
get_logger)Versionc                
   @   s   e Zd ZdZdZdZd/eeeeef e	e e
e
e	ee  dddZed	d
 Zedd Zedd Zdd Zedd Zedd Zedd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Z d-d. Z!dS )0MockDownloadManagerZ
dummy_dataZdatasetsFNT)dataset_nameconfigversion	cache_diruse_local_dummy_dataload_existing_dummy_datadownload_callbacksc                 C   sH   d| _ || _|| _|| _|| _|p$g | _|| _t|| _d | _	d | _
d S )Nr   )Zdownloaded_sizer   r   r   r   r   r   strversion_name_dummy_file_bucket_url)selfr   r   r   r   r   r   r    r   h/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/datasets/download/mock_download_manager.py__init__&   s    


zMockDownloadManager.__init__c                 C   s   | j d kr|  | _ | j S N)r   download_dummy_datar   r   r   r   
dummy_fileA   s    

zMockDownloadManager.dummy_filec                 C   s0   | j d k	r tjd| j j| jS tjd| jS )Ndummy)r   ospathjoinnamer   r   r   r   r   dummy_data_folderG   s    
z%MockDownloadManager.dummy_data_folderc                 C   s   t j| jdS )Nzdummy_data.zip)r"   r#   r$   r&   r   r   r   r   dummy_zip_fileO   s    z"MockDownloadManager.dummy_zip_filec                 C   s8   | j dkr| jn| j}t|| jddd}tj|| jS )NT)r   Zextract_compressed_fileZforce_extract)	r   local_path_to_dummy_datagithub_path_to_dummy_datar	   r   r"   r#   r$   dummy_file_name)r   Zpath_to_dummy_data_dirZ
local_pathr   r   r   r   S   s       z'MockDownloadManager.download_dummy_datac                 C   s   t j| j| j| jS r   )r"   r#   r$   datasets_scripts_dirr   r'   r   r   r   r   r(   ^   s    z,MockDownloadManager.local_path_to_dummy_datac                 C   s*   | j d kr$t| j| jtjd| _ | j S )N/)r   r
   r   r'   replacer"   sepr   r   r   r   r)   b   s    
z-MockDownloadManager.github_path_to_dummy_datac                 C   s8   t j| jr| jS d| jt jddd d S Nr,   )r"   r#   isdirr    r$   r-   r.   splitr   r   r   r   
manual_dirh   s    zMockDownloadManager.manual_dirc                 G   sT   | j r| j}n| j}t|tr*| ||S t|ttfrD| ||S | 	||S d S r   )
r   r    r*   
isinstancedictcreate_dummy_data_dictlisttuplecreate_dummy_data_listcreate_dummy_data_single)r   data_urlargsr    r   r   r   download_and_extractq   s    
z(MockDownloadManager.download_and_extractc                 G   s
   |  |S r   r=   )r   r;   r<   r   r   r   download   s    zMockDownloadManager.downloadc                 C   s
   |  |S r   r>   )r   r;   Zcustom_downloadr   r   r   download_custom   s    z#MockDownloadManager.download_customc                 O   s   |S r   r   )r   r#   r<   kwargsr   r   r   extract   s    zMockDownloadManager.extractc                 C   s   i S r   r   r   r   r   r   get_recorded_sizes_checksums   s    z0MockDownloadManager.get_recorded_sizes_checksumsc           	         s   i }|  D ]\}}| jD ].}t|tr<|D ]}|| q,q|}|| qt|trh fdd|D }n |}tj tj	t
|j}|||< qtdd | D rtt| t| k rdd |  D }|S )Nc              	      s(   g | ] }t j tjt|jqS r   )r"   r#   r$   urllibparse
quote_plusr   r%   ).0xpath_to_dummy_datar   r   
<listcomp>   s     z>MockDownloadManager.create_dummy_data_dict.<locals>.<listcomp>c                 s   s   | ]}t |tV  qd S r   )r4   r   )rG   ir   r   r   	<genexpr>   s     z=MockDownloadManager.create_dummy_data_dict.<locals>.<genexpr>c                 S   s   i | ]\}}||| qS r   r   )rG   keyvaluer   r   r   
<dictcomp>   s      z>MockDownloadManager.create_dummy_data_dict.<locals>.<dictcomp>)itemsr   r4   r7   r"   r#   r$   rD   rE   rF   r   r%   allvalueslenset)	r   rJ   r;   Zdummy_data_dictrN   Zsingle_urlsdownload_callback
single_urlrO   r   rI   r   r6      s$    




&z*MockDownloadManager.create_dummy_data_dictc           	   	   C   s   g }t dd |D }t dd |D }|rF|s4|rF|d gt| }|D ]B}| jD ]}|| qTtj|tj|	dd }|
| qJ|S )Nc                 s   s   | ]}t td |V  qdS )z[0-9]{3,}-of-[0-9]{3,}N)boolrefindallrG   urlr   r   r   rM      s     z=MockDownloadManager.create_dummy_data_list.<locals>.<genexpr>c                 s   s   | ]}| d V  qdS )z3https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmedN)
startswithr[   r   r   r   rM      s    r   r,   r0   )rR   rT   r   r"   r#   r$   rD   rE   rF   r2   append)	r   rJ   r;   Zdummy_data_listZis_tf_recordsZis_pubmed_recordsrW   rV   rO   r   r   r   r9      s    

 z*MockDownloadManager.create_dummy_data_listc                 C   sR   | j D ]}|| qtj|tj|dd }tj|sF| j	sJ|S |S d S r/   )
r   r"   r#   r$   rD   rE   rF   r2   existsr   )r   rJ   r;   rV   rO   r   r   r   r:      s    

 z,MockDownloadManager.create_dummy_data_singlec                 C   s   d S r   r   r   r   r   r   delete_extracted_files   s    z*MockDownloadManager.delete_extracted_filesc                 C   s   d S r   r   r   r   r   r   manage_extracted_files   s    z*MockDownloadManager.manage_extracted_filesc                 #   sh    fdd}t |} jr"||n|d}|D ]2}| r0|jds0|| |dfV  q0d S )Nc              	   3   s\   t  jj}| |}t j}| }W 5 Q R X |D ]}|| r8|	|V  q8d S r   )
r   r    parentrelative_tor   r(   namelistr]   as_posixjoinpath)r#   Zdummy_parent_pathrelative_pathzip_filemembersmemberr   r   r   _iter_archive_members   s    
z?MockDownloadManager.iter_archive.<locals>._iter_archive_members*.__rb)	r   r   rglobis_filer%   r]   rc   re   open)r   r#   rk   
file_paths	file_pathr   r   r   iter_archive   s    
z MockDownloadManager.iter_archivec                 c   s   t |ts|g}|D ]}tj|rDtj|dr< d S |V  qt|D ]P\}}}tj|drlqN|  t	|D ] }|drq|tj
||V  q|qNqd S )Nrm   )r4   r7   r"   r#   isfilebasenamer]   walksortsortedr$   )r   pathsr#   dirpathdirnames	filenamesfilenamer   r   r   
iter_files   s    

zMockDownloadManager.iter_files)NFTN)"__name__
__module____qualname__r*   r+   Zis_streamingr   r   r   r   rX   r   r   r   propertyr    r&   r'   r   r(   r)   r3   r=   r?   r@   rB   rC   r6   r9   r:   r`   ra   rv   r   r   r   r   r   r   !   sR       







r   )__doc__r"   rY   urllib.parserD   pathlibr   typingr   r   r   r   zipfiler   Zutils.file_utilsr	   r
   Zutils.loggingr   Zutils.versionr   r   loggerr   r   r   r   r   <module>   s   