U
    -e2                     @   s  d dl Z d dlZd dlmZ d dlmZmZ d dlmZ z$d dl	m
Z d dlmZmZmZ W n ek
rt   dZY nX zd dlZd dlmZ W n ek
r   d ZZY nX e jj
Ze jjedd Zdd	 Zd
d Ze jjedd Ze jjedd Ze jje jdde e gdd Ze jje jjdd Z e jje jdde e gdd Z!e jje jj"dd Z#e jje jj"dd Z$e jje jj"dd Z%e jjdd Z&e jjedd  Z'd!d" Z(dS )#    N)fs)
FileSystemLocalFileSystem)parametrize_legacy_dataset)_read_table_test_dataframe_range_integersc                 C   s   t d}d|d< tjj|dd}t }tj||jdd}g }tdD ]4}||d< tjj|dd}|	| |
|  qF|  | }tt||d	}	tj|d
d}
t|	 |
 d S Nd   r   	unique_idFZpreserve_index2.6version
   use_legacy_datasetTZignore_index)r   paTablefrom_pandasBufferOutputStreampqParquetWriterschemarangewrite_tableappendcopyclosegetvaluer   BufferReaderpdconcattmassert_frame_equal	to_pandastempdirr   dfarrow_tableoutwriterframesibufresultexpected r2   j/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/parquet/test_parquet_writer.py#test_parquet_incremental_file_build.   s&    
 r4   c              
   C   s   t dt  t dt  g}t |}t dgt dgg}t j|ddg}| d }tj	||dddd	&}t
t || W 5 Q R X W 5 Q R X d S )
NZPOSdesc   Zblazsimple_validate_schema.parquetr   ZsnappyZspark)r   compressionZflavor)r   fieldZuint32stringr   arrayr   from_arraysr   r   pytestraises
ValueErrorr   )r(   Zsimple_fieldsZsimple_schemaZsimple_from_arrayZsimple_tablepathwr2   r2   r3    test_validate_schema_write_tableK   s    
 rA   c               	   C   s`   t t* ttdt g} td |  W 5 Q R X t t tdd  W 5 Q R X d S )NxZ	some_path)	r<   r=   	TypeErrorr   r   r8   Zint32r   r   )Zsome_schemar2   r2   r3   test_parquet_invalid_writera   s
    rD   c              	   C   s   t d}d|d< tjj|dd}t }tj||jddH}g }tdD ]4}||d< tjj|dd}|	| |
|  qHW 5 Q R X | }tt||d	}	tj|d
d}
t|	 |
 d S r	   )r   r   r   r   r   r   r   r   r   r   r   r   r    r   r!   r"   r#   r$   r%   r&   r'   r2   r2   r3   test_parquet_writer_context_objk   s$    
 rE   c              
   C   s  t d}d|d< tjj|dd}t }d}zttj||jddX}g }td	D ]D}||d< tjj|dd}|	| |
|  |d
krNt|qNW 5 Q R X W n2 tk
r }	 zt|	|kstW 5 d }	~	X Y nX | }
tt|
|d}tj|dd}t| | d S )Nr
   r   r   Fr   zArtificial Errorr   r   r      r   Tr   )r   r   r   r   r   r   r   r   r   r   r   r   r>   	ExceptionstrAssertionErrorr    r   r!   r"   r#   r$   r%   r&   )r(   r   r)   r*   r+   Z
error_textr,   r-   r.   er/   r0   r1   r2   r2   r3   .test_parquet_writer_context_obj_with_exception   s8    
" rK   
filesystemc           	   	   C   sL  t d}tjj|dd}tjj|dd}t| d }t| d }tj||j|dd}|	| W 5 Q R X t
| }t|| tj||j|dd}|| W 5 Q R X t
| }t|| tj||j|dd}|| W 5 Q R X t
| }t|| tj||j|dd}|| W 5 Q R X t
| }t|| d S )Nr
   Fr   zdata_table.parquetzdata_batch.parquetr   rL   r   )r   r   r   r   ZRecordBatchrH   r   r   r   r   r   r&   r$   r%   Zwrite_batchwrite)	r(   rL   r)   tablebatchZ
path_tableZ
path_batchr,   r0   r2   r2   r3   "test_parquet_writer_write_wrappers   sZ                rQ   c                    s   dd  fdd}|d d d |d d |d d d |d d d | d  d d |d d |d d d d S )	Ni   i   c           	         s   t jjt| dgdgd}|d kr4t|d  ntj|d |d td }|d krbn|}|j|ksttt	| }t
|d D ]}||j|kstq| ||d   }|dkr||d j|kstn||d j|kstd S )NbrB   )namesztest.parquet)Zrow_group_sizer6   r   )r   r   r;   r   r   r   read_metadataZnum_row_groupsrI   minr   Z	row_groupZnum_rows)		data_size
chunk_sizeZexpect_num_chunksrO   metadataZexpected_chunk_sizeZlatched_chunk_sizeZ	chunk_idx	remainderZabs_max_chunk_sizeZdefault_chunk_sizer(   r2   r3   check_chunk_size   s,    

z8test_parquet_writer_chunk_size.<locals>.check_chunk_size   r
      r6   r2   )r(   r[   r2   rZ   r3   test_parquet_writer_chunk_size   s    r^   c              	   C   sj   t d}tjj|dd}t| d }tj||j|dd}|| W 5 Q R X t	|
 }t|| d S )Nr
   Fr   zdata.parquetr   rM   )r   r   r   r   rH   r   r   r   r   r   r&   r$   r%   )r(   rL   r)   rO   r?   r,   r0   r2   r2   r3   $test_parquet_writer_filesystem_local  s       r_   c              	   C   sh   t d}tjj|dd}| \}}}tj||j|dd}|| W 5 Q R X t|	 }t
|| d S )Nr
   Fr   r   rM   r   r   r   r   r   r   r   r   r   r&   r$   r%   Zs3_example_fsr)   rO   r   urir?   r,   r0   r2   r2   r3   !test_parquet_writer_filesystem_s3  s    
   rc   c              	   C   sj   t d}tjj|dd}| \}}}tj||jdd}|| W 5 Q R X t||d	 }t
|| d S )Nr
   Fr   r   r   rL   r`   ra   r2   r2   r3   %test_parquet_writer_filesystem_s3_uri(  s    
re   c              	   C   sr   t d}tjj|dd}| \}}|d }tj||j|dd}|| W 5 Q R X t||d	 }t
|| d S )Nr
   Fr   z/test.parquetr   rM   rd   r`   )Zs3_example_s3fsr)   rO   r   	directoryr?   r,   r0   r2   r2   r3   #test_parquet_writer_filesystem_s3fs7  s       rg   c               	   C   sT   t d} tjj| dd}t }tjtdd t	j
t |j|d W 5 Q R X d S )Nr
   Fr   zspecified path is file-like)matchrd   )r   r   r   r   r   r   r<   r=   r>   r   r   r   r   )r)   rO   rL   r2   r2   r3   ,test_parquet_writer_filesystem_buffer_raisesI  s      ri   c              	      s   t   G  fdddt}| }d}td}t jj|dd}tj||j|dd}|	| W 5 Q R X |j
|ksvt|jd	kst jst  }tt || d
}| }	t|	| tt0}
tjt  |j|d d}t|
|kstW 5 Q R X d S )Nc                       s"   e Zd Zdd Zd fdd	ZdS )zEtest_parquet_writer_with_caller_provided_filesystem.<locals>.CustomFSc                 S   s   d | _ d | _d S Nr?   mode)selfr2   r2   r3   __init__\  s    zNtest_parquet_writer_with_caller_provided_filesystem.<locals>.CustomFS.__init__rbc                    s   || _ || _ S rj   rk   )rm   r?   rl   r+   r2   r3   open`  s    zJtest_parquet_writer_with_caller_provided_filesystem.<locals>.CustomFS.openN)ro   )__name__
__module____qualname__rn   rq   r2   rp   r2   r3   CustomFS[  s   ru   zexpected_fname.parquetr
   Fr   r   rM   wbr   rd   zVfilesystem passed but where is file-like, so there is nothing to open with filesystem.)r   r   r   r   r   r   r   r   r   r   r?   rI   rl   closedr    r   r!   r&   r$   r%   r<   r=   r>   rH   )r   ru   r   fnamer)   rO   r,   r/   Z
table_readZdf_readZerr_infoZexpected_msgr2   rp   r3   3test_parquet_writer_with_caller_provided_filesystemV  s.    

 ry   c              	   C   s   t ddddgi}| d }t||j}|| W 5 Q R X t|}d|jksXt|jd sft| d }tj||jdd	}|| W 5 Q R X t|}|jd kstd S )
Nar6   r\   r]   ztest_with_schema.parquets   ARROW:schemaztest_without_schema.parquetF)Zstore_schema)	r   rO   r   r   r   r   rT   rX   rI   )r(   rO   Zpath1r,   metaZpath2r2   r2   r3    test_parquet_writer_store_schema  s    

r|   ))r<   Zpyarrowr   r   Zpyarrow.filesystemr   r   Zpyarrow.tests.parquet.commonr   Zpyarrow.parquetZparquetr   r   r   r   ImportErrorZpandasr"   Zpandas.testingtestingr$   markZ
pytestmarkr4   rA   rD   rE   rK   ZparametrizeZ_get_instancerQ   Zlarge_memoryr^   r_   Zs3rc   re   rg   ri   ry   r|   r2   r2   r2   r3   <module>   st   

"()
(