U
    -e*                     @   s  d dl Z d dlZd dlZd dlmZ d dlZd dlZzd dlm	Z
 d dlmZ W n ek
rh   dZ
Y nX z$d dlZd dlmZ d dlmZ W n ek
r   d ZZY nX ejj	Zejjdd Zejjdd Zejjd	d
 Zejjdd Zejjdd Zejjdd Zdd Zdd Zejjejddddgdd Zejjejddgdd Z ejjejddd gd!d" Z!d#d$ Z"ejj#ejd%d&d'd( Z$dS ))    N)mock)_write_table)alltypes_samplec                  C   sr   t dd} tj| }t }t||ddd |d t	|}|d tj
||d}t| |   d S )N'  sizesnappy2.6)compressionversionr   )metadata)r   paTablefrom_pandasioBytesIOr   seekpqZread_metadataParquetFiletmassert_frame_equalread	to_pandas)dfa_tablebufr   Zfileh r   h/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/parquet/test_parquet_file.pytest_pass_separate_metadata/   s    



r   c                     s   d\} }t | d}tj|}t }t||| | ddd |d t	|  j
|ks^t fddt|D }t|}t||  d S )	Nr      r   r   r	   Zrow_group_sizer
   r   r   c                    s   g | ]}  |qS r   Zread_row_group.0ipfr   r   
<listcomp>U   s     z.test_read_single_row_group.<locals>.<listcomp>)r   r   r   r   r   r   r   r   r   r   num_row_groupsAssertionErrorrangeconcat_tablesr   r   r   NKr   r   r   
row_groupsresultr   r&   r   test_read_single_row_groupC   s    
 


r2   c                     s   d\} }t | d}tj|}t }t||| | ddd |d t	|t
|jd d   fdd	t|D }t|}t|  |   fd
d	t|D }t|}t|  |  d S )Nr   r   r   r	   r!   r      c                    s   g | ]}j | d qS columnsr"   r#   colsr'   r   r   r(   h   s     zAtest_read_single_row_group_with_column_subset.<locals>.<listcomp>c                    s   g | ]}j |   d qS r4   r"   r#   r7   r   r   r(   n   s     )r   r   r   r   r   r   r   r   r   r   listr6   r+   r,   r   r   r   r-   r   r7   r   -test_read_single_row_group_with_column_subsetZ   s"    
 



r:   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}|j
|ks^t|t|}t||  d S )Nr   r   r   r	   r!   r   )r   r   r   r   r   r   r   r   r   r   r)   r*   read_row_groupsr+   r   r   r   )r.   r/   r   r   r   r'   r1   r   r   r   test_read_multiple_row_groupss   s    
 

r<   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}t
|jd d }|jt||d}t|| |  |jt||| d}t|| |  d S )	Nr   r   r   r	   r!   r   r3   r5   )r   r   r   r   r   r   r   r   r   r   r9   r6   r;   r+   r   r   r   )r.   r/   r   r   r   r'   r8   r1   r   r   r   0test_read_multiple_row_groups_with_column_subset   s    
 

r=   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}|
 dks`t|
|jd d dks|td S )	Nr   r   r   r	   r!   r   r   r    )r   r   r   r   r   r   r   r   r   r   Zscan_contentsr*   r6   )r.   r/   r   r   r   r'   r   r   r   test_scan_contents   s    
 

r>   c              	   C   sl   | d }t t| dt| d}tt}t| W 5 Q R X |t	r^t
jdkr^d S || d S )N	directoryzCannot open for reading: path 'z' is a directorywin32)osmkdirstrpytestraisesIOErrorr   r   ZerrisinstancePermissionErrorsysplatformmatch)tempdirpathmsgexcr   r   r   0test_parquet_file_pass_directory_instead_of_file   s    rO   c               
   C   s   t jt ddgt ddggddgd} t  }t| | t| }|j	d
 ddgksht|j	d	
 ddgkstd
D ]*}tttf |j	| W 5 Q R X qd S )Nr       ZfoobarZintsstrs)namesr      )r3   )r   tablearrayZBufferOutputStreamr   write_tabler   getvaluereaderZread_columnZ	to_pylistr*   rD   rE   
ValueError
IndexError)rV   Zbiofindexr   r   r   test_read_column_invalid_index   s    r_   
batch_sizei,    i  c              	   C   s   d}d}t |d}| d }tj|}t||d|d t|}|jd d |jdd  fD ]|}|j||d}	t	d	|| |}
t
|	|
D ]N\}}t||| }t| |j||d d f jd d |f jd
d qq\d S )Ni  ra   r   pandas_roundtrip.parquetr	   r   
chunk_size
   )r`   r6   r   TZdrop)r   r   r   r   r   r   r   r6   iter_batchesr+   zipminr   r   r   iloclocreset_index)rK   r`   
total_sizerd   r   filenamearrow_tablefile_r6   batchesZbatch_startsbatchstartendr   r   r    test_iter_batches_columns_reader   s$    

 *ru   rd   c           
      C   s   t ddd}| d }tj|}|jjd k	s0tt||d|d t	|}dd }t
||}d	}t|jD ]t}	t||  ||	g d
 |d7 }t||  jdd||	g jd
d  jdd |d7 }qld S )Nr   T)r   Zcategoricalrb   r	   rc   c                 s   s4   t | jD ]$}| jd|gd}|D ]
}|V  q"q
d S )N  )r`   r0   )r+   r)   rg   )r]   Z	row_grouprq   rr   r   r   r   get_all_batches   s    z1test_iter_batches_reader.<locals>.get_all_batchesr   rv   rT   rf   )r   r   r   r   ZschemaZpandas_metadatar*   r   r   r   r9   r+   r)   r   r   r   r;   headrl   rj   )
rK   rd   r   rn   ro   rp   rw   rq   Zbatch_nor%   r   r   r   test_iter_batches_reader   s0    


ry   
pre_bufferFTc                 C   sj   d\}}t |d}tj|}t }t|||| ddd |d tj	|| d}|
 j|ksftd S )Nr   r   r   r	   r!   r   )rz   )r   r   r   r   r   r   r   r   r   r   r   Znum_rowsr*   )rz   r.   r/   r   r   r   r'   r   r   r   test_pre_buffer  s    
 
r{   c              
   C   s  |  d}tddgddgd}t|| t|d}tj|dd |jrRtW 5 Q R X t	j
tjd}tj|dd |  W 5 Q R X t	j
tjd }tj|dd  |  W 5 Q R X t|d<}tjtd	d
" tj|dd  |jrtW 5 Q R X W 5 Q R X t|dT}t|&}|  |jr<t|jrHtW 5 Q R X |jr^t|jrjtW 5 Q R X |jst|jstt|}|  |jrtW 5 Q R X |jstdS )z
    Unopened files should be closed explicitly after use,
    and previously opened files should be left open.
    Applies to read_table, ParquetDataset, and ParquetFile
    zfile.parquetr   rT   )Zcol1Zcol2rbT)Zuse_legacy_datasetcloseznot a path-like object)rJ   N)joinpathr   rV   r   rX   openZ
read_tableclosedr*   r   patchobjectr   Zassert_calledZParquetDatasetr   rD   rE   	TypeError)rK   fnrV   r]   Z
mock_closepr   r   r   #test_parquet_file_explicitly_closed   s:    
r   use_uri)TFc              	   C   s   |\}}}|r|n|f}|r i nt |d}tdtdi}tj|||d tj||}	|	 |ksht|	j	rrt|	
  |	j	sttj|| }
|
 |kst|
j	rtW 5 Q R X |
j	std S )N)
filesystemare   )dictr   rV   r+   r   rX   r   r   r*   r   r}   )rK   Zs3_example_fsr   Zs3_fsZs3_uriZs3_pathargskwargsrV   Zparquet_filer]   r   r   r   !test_parquet_file_with_filesystemS  s    


r   )%r   rA   rH   Zunittestr   rD   Zpyarrowr   Zpyarrow.parquetZparquetr   Zpyarrow.tests.parquet.commonr   ImportErrorZpandaspdZpandas.testingtestingr   r   markZ
pytestmarkr   r2   r:   r<   r=   r>   rO   r_   Zparametrizeru   ry   r{   r   Zs3r   r   r   r   r   <module>   sX   






+3