U
    -e|                  	   @   s  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZ d dlmZ d dlmZmZmZmZ z d dlmZ d dlmZmZ W n ek
r   dZY nX z0d dlZd dlmZ d dlm Z  d d	lm!Z! W n ek
r   d ZZY nX ej"jZ#d
d Z$edd Z%ej"jedd Z&ej"jedd Z'ej"jedd Z(ej"jedd Z)ej"jedd Z*edd Z+edd Z,ej"j-dd Z.ej"jedd Z/ej"jed d! Z0ed"d# Z1ej"jed$d% Z2ed&d' Z3ej"j4eej"5d(de	 e6 gej"5d)d*d+d, Z7d-d. Z8d/d0 Z9ed1d2 Z:ed3d4 Z;ed5d6 Z<ed7d8 Z=d9d: Z>ej"jed;d< Z?ej"jed=d> Z@ej"jd?d@ ZAedAdB ZBedCdD ZCej"jedEdF ZDedGdH ZEej"jej"jFej"GdIej"GdJdKdL ZHeej"5dMdNdO dPdO dQdO dRdO gej"5dSdTdUgej"5dVdTdUgdWdX ZIdYdZ ZJd[d\ ZKd]d^ ZLej"j4d_d` ZMdadb ZNdcdd ZOdS )e    )OrderedDictN)fs)LocalFileSystem
FileSystem)util)_check_roundtrip_roundtrip_tableparametrize_legacy_dataset_test_dataframe)_read_table_write_table)dataframe_with_lists)alltypes_samplec              	   C   sp   t ddddgi}tjtdd t|| d dd	 W 5 Q R X tjtd
d t|| d dd W 5 Q R X d S )Na         z"Unsupported Parquet format versionmatchztest_version.parquetz2.2versionz%Unsupported Parquet data page version)data_page_version)patablepytestraises
ValueErrorr   )tempdirr    r   a/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/parquet/test_basic.pytest_parquet_invalid_version7   s    
r    c                 C   sL   t dddgd }t jj|gdgd}ddg}|D ]}t||| d	 q4d S )
Nr   r   r   i f0namesi   i   )data_page_sizeuse_legacy_dataset)r   arrayTablefrom_arraysr   )r%   arrtZ
page_sizesZtarget_page_sizer   r   r   test_set_data_page_sizeA   s    r+   c                 C   s,   t d}tjj|dd}t|dddd d S )Nd   FZpreserve_index
   r   2.4)r$   Zwrite_batch_sizer   )r
   r   r'   from_pandasr   r%   dfr   r   r   r   test_set_write_batch_sizeM   s       r3   c              	   C   sR   t d}tjj|dd}t|dddd tt t|dddd W 5 Q R X d S )	Nr,   Fr-   r   r.   r/   )Zdictionary_pagesize_limitr$   r   r   )r
   r   r'   r0   r   r   r   	TypeErrorr1   r   r   r   "test_set_dictionary_pagesize_limitX   s      r5   c              
   C   s   g }t jtdd}|t j|gd  t \}}t j|}|t j|gd  dD ]*}dD ] }|D ]}t|d| ||d qpqhq`d S )Nr.   sizer   )z1.0z2.0)TF2.6)r   r%   r   use_dictionary)	r   RecordBatchr0   r   appendr'   from_batchesr   r   )r%   Ztablesbatchr2   _r   r9   r   r   r   r   test_chunked_table_writef   s     
 r?   c              	   C   s|   t dd}tj|}t|ddid|d t| d }t|d}t||dd	 W 5 Q R X tj	|d|d
}|
|sxtd S )Nr.   r6   
memory_mapTr8   Zread_table_kwargsr   r%   tmp_filewbr   )r@   r%   r   r   r'   r0   r   stropenr   pqread_pandasequalsAssertionErrorr   r%   r2   r   filenamef
table_readr   r   r   test_memory_map{   s    

 rO   c              	   C   s|   t dd}tj|}t|ddid|d t| d }t|d}t||dd	 W 5 Q R X tj	|d
|d}|
|sxtd S )Nr.   r6   buffer_sizei  r8   rA   rB   rC   r   i   )rP   r%   rD   rK   r   r   r   test_enable_buffered_stream   s    

 rQ   c                 C   sn   t jt dggdg}d}| | }| r2tt|t| | sLttt||d}|	|sjtd S )N*   intsz	foo # barr%   )
r   r'   r(   r&   existsrJ   r   rE   r   rI   )r   r%   r   rL   pathrN   r   r   r   test_special_chars_filename   s    rW   c              	   C   sP   t jtdd tjd | d W 5 Q R X t jtdd td  W 5 Q R X d S )NNoner   rT   )r   r   r4   rG   
read_tableParquetFilerT   r   r   r   test_invalid_source   s    r[   c                  C   s*   t jttdgdgd} t| dd d S )Ni@  r!   r"   r   )Zrow_group_size)r   r   listranger   )r*   r   r   r   (test_file_with_over_int16_max_row_groups   s    r^   c                 C   s   t dd}tj|}tjjdd | D |jjd}|jdj	t
 ksRt|jdj	tt
 ksrtt|d| d	 d S )
Nr.   r6   c                 S   s   g | ]}| d dd  qS )r   N)chunk).0colr   r   r   
<listcomp>   s     z.test_empty_table_roundtrip.<locals>.<listcomp>r"   nullZ	null_listr8   )r   r%   )r   r   r'   r0   r(   Zitercolumnsschemar#   fieldtyperc   rJ   list_r   r1   r   r   r   test_empty_table_roundtrip   s    
   rh   c                 C   s(   t  }tjj|dd}t|| d d S )NFr-   rT   )pd	DataFramer   r'   r0   r   )r%   r2   emptyr   r   r   test_empty_table_no_columns   s    rl   c                    st   t t tt d g t dddgg} fdd|D } fdd|D }tj|t }t|| d d S )N)int32Zlist_stringr   )Gc                    s$   g | ]}t j|t  d  qS )rf   )r   r&   structflattenr`   r=   colsr   r   rb      s   zEtest_write_nested_zero_length_array_chunk_failure.<locals>.<listcomp>c                    s"   g | ]}t jj|t  d qS ))rd   )r   r:   r(   rd   rr   rs   r   r   rb      s   rT   )	r   r   rm   rg   stringr'   r<   rd   r   )r%   dataZ	my_arraysZ
my_batchestblr   rs   r   1test_write_nested_zero_length_array_chunk_failure   s    

rx   c                 C   s   | d }t dtjdtjdi}t|| t||d}| }t	|| t
| d }t dtjdtjdi}t|| t||d}| }t	|| d S )Nzzzz.parquetxr.   ZdtyperT   )ri   rj   nparangeZint64r   r   	to_pandastmassert_frame_equalrE   )r   r%   rV   r2   rN   Zdf_readr   r   r   test_multiple_path_types   s    

r   c              	   C   sr   | d }t ddddgi}t|| t|}t||d}||sJtt	t
 t|t d W 5 Q R X d S )Ntest.parquetr   r   r   r   rT   
filesystem)r   r   r   r   ZFSProtocolClassr   rI   rJ   r   r   r4   r   )r   r%   rV   r   Zfs_protocol_objresultr   r   r   test_fspath  s    

 r   r   name)data.parquetu   例.parquetc              	   C   s   |rt |tjrtd tddddgi}| | }t|t	| t
|  tj|||d}W 5 Q R X ||szt|  | rtt
|  tj|||d W 5 Q R X t|}||std S )Nz6Passing new filesystem not supported for legacy readerr   r   r   r   )r   r%   r   )
isinstancer   r   r   skipr   r   rG   write_tablerE   r   Z
change_cwdrY   rI   rJ   unlinkrU   )r   r%   r   r   r   rV   r   r   r   r   test_relative_paths  s     

r   c                	   C   s$   t t td W 5 Q R X d S )Nzi-am-not-existing.parquet)r   r   FileNotFoundErrorrG   rY   r   r   r   r   test_read_non_existing_file=  s    r   c               	   C   s>   G dd dt j} tjtdd t| d W 5 Q R X d S )Nc                   @   s   e Zd Zdd Zdd ZdS )z3test_file_error_python_exception.<locals>.BogusFilec                 W   s   t dd S NzorglubZeroDivisionErrorselfargsr   r   r   readE  s    z8test_file_error_python_exception.<locals>.BogusFile.readc                 W   s   t dd S r   r   r   r   r   r   seekH  s    z8test_file_error_python_exception.<locals>.BogusFile.seekN)__name__
__module____qualname__r   r   r   r   r   r   	BogusFileD  s   r   r   r       )ioBytesIOr   r   r   rG   rY   )r   r   r   r    test_file_error_python_exceptionC  s    r   c              	   C   s   t ddddgi}t|t| d  tt| d d}tj||d}W 5 Q R X ||sbttt| d d}tjt 	||d}W 5 Q R X ||std S )Nr   r   r   r   r   rbrT   )
r   r   rG   r   rE   rF   rY   rI   rJ   Z
PythonFile)r   r%   r   rM   r   r   r   r   test_parquet_read_from_bufferP  s    r   c              	   C   s   t ttttd}t ttttd}||g}t jj|ddgd}t	||dddd t	||ddgdgd t	||dddgddgd t jj||gddgd}t	||dgdgd	 t jj|gd
gd}t
t t	||dd| d W 5 Q R X d S )Nr,   r   br"   gzipFT)expectedcompressionr9   use_byte_stream_split)r   r9   r   tmp)r   r   r9   r%   )r   r&   r\   mapfloatr]   intr'   r(   r   r   r   IOError)r%   	arr_floatarr_intZ
data_floatr   mixed_tabler   r   r   test_byte_stream_split`  s:     r   c              
   C   sf  t ttttd}t ttttd}t dd tdD }t jj|||gdddgd}t	||dd	d
d
d| d t	||dd
| d t	||dd
dd
d| d t	||dd
ddd| d t
jtdd  t	||dd
d	d
d| d W 5 Q R X t
t  t	||ddd
d
d| d W 5 Q R X t
t t	||dd| d W 5 Q R X t
t t	||dddi| d W 5 Q R X t
t t	||dgdd
i| d W 5 Q R X t
t t	||dd
i| d W 5 Q R X t
t$ t	||ddgdd	d
d| d W 5 Q R X t
t" t	||dddd	d
d| d W 5 Q R X t
t t	||dd| d W 5 Q R X d S )Nr,   c                 S   s   g | ]}t |qS r   )rE   )r`   ry   r   r   r   rb     s     z(test_column_encoding.<locals>.<listcomp>r   r   cr"   FZBYTE_STREAM_SPLITZPLAIN)r   r   r   )r   r9   column_encodingr%   ZDELTA_BINARY_PACKEDZDELTA_LENGTH_BYTE_ARRAYz0BYTE_STREAM_SPLIT only supports FLOAT and DOUBLEr   ZRLE_DICTIONARYZMADE_UP_ENCODING)r   r   r%   ZRLE)r   r9   r   r   r%   T)r   r&   r\   r   r   r]   r   r'   r(   r   r   r   r   OSErrorr   r4   )r%   r   r   Zarr_binr   r   r   r   test_column_encoding  s    		


r   c              
   C   s   t ttttd}||g}t jj|ddgd}t||dd| d t||dd| d t||dd	d
| d t||dddd
| d t||dd| d t||dd| d ddddg}t	
 }|D ]2\}}tttf t||||d W 5 Q R X qd S )N  r   r   r"   r   r   )r   r   compression_levelr%      snappy)r   r   )r   r   r%   r   r   Zlz4	   )r      )r   i)rX   i  )Zlzo   )r   r   )r   r&   r\   r   r   r]   r'   r(   r   r   r   r   r   r   r   r   )r%   r)   rv   r   Zinvalid_combinationsbufcodeclevelr   r   r   test_compression_level  sJ    
 r   c                  C   sV   t dddddg} d}t j| g|g}t|ddid	}d
}|jd j|ksRtd S )Nr   r   r   r   r   zprohib; ,	{}ZflavorZspark)Zwrite_table_kwargsZprohib______)r   r&   r'   r(   r   rd   r   rJ   )Za0r   r   r   Zexpected_namer   r   r    test_sanitized_spark_field_names5  s    r   c                 C   sp   t dd}tj|}t }t||ddd |d t|d| d}|d t|d	| d}|	|slt
d S )
Ni'  r6   ZSNAPPYr8   )r   r   r   T)use_threadsr%   F)r   r   r'   r0   r   r   r   r   r   rI   rJ   )r%   r2   r   r   Ztable1table2r   r   r   test_multithreaded_read@  s"    

  
  r   c              	   C   s   t jtdgddddgd}tj| }t	 }t
||dd |d	 t|| d
}||shttt t
||d	d W 5 Q R X d S )Nr   ABCD)columns)
chunk_sizer   rT   )ri   rj   r{   r|   r   r'   r0   Zreset_indexr   r   r   r   r   rI   rJ   r   r   r   )r%   rv   r   r   r   r   r   r   test_min_chunksizeU  s    
r   c                 C   s   t tdttddtdddtjddd	d
dddgt tdt jdddt jddddt jddddd	}t	j
|}| d }zt||dd W n t	jk
r   Y nX | rtd S )Nabcr   r   r      u1      @      @float64rz   TF20130101periodsz
US/Eastern)r   tzns)r   freq)	r   r   r   derM   ghirB   r/   r   )ri   rj   r\   r]   r{   r|   astypeCategorical
date_ranger   r'   r0   r   ZArrowExceptionrU   rJ   )r   r2   ZpdfrL   r   r   r   (test_write_error_deletes_incomplete_fileg  s&    
r   c              
   C   sP   d}zt j||d W n4 tk
rJ } z||jd ks:tW 5 d }~X Y nX d S )Nznon-existent-file.parquetrT   r   )rG   rY   	Exceptionr   rJ   )r   r%   rV   r   r   r   r   test_read_non_existent_file  s
    r   c              	   C   sl   |r6d}t jt|d tj| d |d W 5 Q R X n2t $ tjdd tj| d |d W 5 Q R X d S )N!Passing 'use_legacy_dataset=True'r   zv0.7.1.parquetrT   error)action)r   warnsFutureWarningrG   rY   warningscatch_warningssimplefilter)Zdatadirr%   msgr   r   r   test_read_table_doesnt_warn  s    


r   c                 C   sd   t jt ddggdg}t }tj||dd |d tj	|| d}t
| |  d S )Nr   defZsome_colr   r   r   rT   )r   r'   r(   r&   r   r   rG   r   r   rY   r~   r   r}   )r%   r   rM   Z	roundtripr   r   r   test_zlib_compression_bug  s    
r   c              
   C   s   t | d }tjtjtfdd* t|d}W 5 Q R X tj||d W 5 Q R X tjtjtfdd4 t|d}|	d W 5 Q R X tj||d W 5 Q R X d S )Nr   zsize is 0 bytesr   rC   rT   zsize is 4 bytess   ffff)
rE   r   r   r   ZArrowInvalidr   rF   rG   rY   write)r   r%   rV   rM   r   r   r   test_parquet_file_too_small  s    
r   zignore:RangeIndex:FutureWarningz.ignore:tostring:DeprecationWarning:fastparquetc           	      C   s   t d}ttdttddtjddddd	d
d	gtjdddt	dddgd}t
|}t| d }tj||d d ||}| }t|| t| d }||| t|}|d t|d< t| | d S )Nfastparquetr   r   r   r   r   r   rz   TFr   r   r   r   r   )r   r   r   r   r   rM   zcross_compat_arrow.parquetr   z cross_compat_fastparquet.parquetrM   )r   Zimportorskipri   rj   r\   r]   r{   r|   r   r   r   r   rE   rG   r   rZ   r}   r~   r   r   rH   r   object)	r   fpr2   r   Z
file_arrowZfp_fileZdf_fpZfile_fastparquetZtable_fpr   r   r   $test_fastparquet_cross_compatibility  s*    



r   array_factoryc                   C   s   t dd gd S Nr   r.   r   r&   r   r   r   r   <lambda>  r   r   c                   C   s   t dd gd  S r   r   r&   Zdictionary_encoder   r   r   r   r     r   c                   C   s   t dd gd S N r.   r   r   r   r   r   r     r   c                   C   s   t dd gd  S r   r   r   r   r   r   r     r   r9   FTread_dictionaryc           
      C   s   t jd|  i}t }tj||dd |d |r>dgnd }tj|d||d}|j	D ].}|j
\}| d }	|	 |	jd ksZtqZd S )	Nra   T)r9   r   F)r   r   r%   r       )r   r'   Zfrom_pydictr   r   rG   r   r   rY   r   chunksbuffersZ
to_pybytesr7   rJ   )
r   r9   r   r%   Z
orig_tableZbior   ra   r_   r   r   r   r   test_buffer_contents  s    

r   c                 C   sP   t jt tdgdgd}| d }tj||dd t|}||sLtd S )Nr   rS   r"   zarrow-10480.pyarrow.gzZGZIPr   )	r   r   r&   r]   rG   r   rY   rI   rJ   )r   r   rV   r   r   r   r   "test_parquet_compression_roundtrip  s
    
r  c              	   C   s   t jt jg ddgdg}| d }d}t||j}t|D ]}|| q@W 5 Q R X t	|}|j
j|ksttt|D ]}|||s|tq|d S )Nrm   ro   r!   zempty_row_groups.parquetr   )r   r'   r(   r&   rG   ZParquetWriterrd   r]   r   rZ   metadataZnum_row_groupsrJ   Zread_row_grouprI   )r   r   rV   Z
num_groupswriterr   readerr   r   r   test_empty_row_groups	  s    
r  c                 C   sV   d gd }| dg tj|gdg}| d }t|| t|}||ksRtd S )Ni   r   columnzarrow-11607.parquet)r;   r   r'   r(   rG   r   rY   rJ   )r   rv   r   rV   r   r   r   r   test_reads_over_batch  s    

r  c                 C   s   | d }|j dd tjdddgddd	ggd
dgd}t||d  tjdddgdddggdd
gd}t||d  tt|}tjddddddgddd	dddggd
dgd}||kstd S )NZ dataset_column_order_permutationT)exist_okr   r   r   g?g?g333333?r   r   r"   zdata1.parquetg?g      ?g333333?r   r   r   zdata2.parquet)mkdirr   r   rG   r   rY   rE   rJ   )r   caseZdata1Zdata2r   r   r   r   r    test_permutation_of_column_order)  s    ""r  c              	   C   sT   t ddddgi}| d }t|| tjtdd tj|dd	 W 5 Q R X d S )
Nr   r   r   r   r   r   r   TrT   )r   r   rG   r   r   r   r   rY   )r   r   rV   r   r   r   !test_read_table_legacy_deprecated=  s     r  c              	   C   s   | d }t ttd}d}t j|g| dd t|D d}t|| tjt	dd tj
|d	| d
 W 5 Q R X tjt	dd tj
||d W 5 Q R X tj
|d| d
}||ksttj
|d| d}||kstt
|}||kstd S )Nzlargethrift.parquetr.   r   c                 S   s   g | ]}d | qS )Zsome_long_column_name_r   )r`   r   r   r   r   rb   P  s     z+test_thrift_size_limits.<locals>.<listcomp>r"   z1Couldn't deserialize thrift:.*Exceeded size limitr   2   )Zthrift_string_size_limit)Zthrift_container_size_limitr,   r   )r   r&   r\   r]   r   rG   r   r   r   r   rY   rJ   )r   rV   r&   Znum_colsr   gotr   r   r   test_thrift_size_limitsI  s0    
r  )Pcollectionsr   r   r   numpyr{   r   Zpyarrowr   r   Zpyarrow.filesystemr   r   Zpyarrow.testsr   Zpyarrow.tests.parquet.commonr   r   r	   r
   Zpyarrow.parquetZparquetrG   r   r   ImportErrorZpandasri   Zpandas.testingtestingr~   Zpyarrow.tests.pandas_examplesr   r   markZ
pytestmarkr    r+   r3   r5   r?   rO   rQ   rW   r[   Zslowr^   rh   rl   rx   r   r   ZdatasetZparametrizeZ_get_instancer   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   filterwarningsr   r   r  r  r  r  r  r  r   r   r   r   <module>   s   


	





  

&
y
3





$
