U
    âÇ-eyK  ã                   @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ e jj	Z
zd dlmZ d dlZW n ek
rl   Y nX e jdddd„ ƒZd	d
„ Zd*dd„Zd+dd„Ze jje j ddddg¡dd„ ƒƒZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Ze jjd d!„ ƒZd"d#„ Zd$d%„ Zd&d'„ Z d(d)„ Z!dS ),é    N)Úfs)Úutil)Úassert_frame_equalÚmodule)Úscopec                 C   s   | d S )NÚorc© )Zbase_datadirr   r   úW/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/test_orc.pyÚdatadir'   s    r
   c                 C   s*  |D ]}|| }| | }|dkrjdd„ |D ƒrj||   ¡ }t|ƒD ]\}}dd„ |D ƒ||< qD|||< q|d j}t|tjƒrŒt |¡}nt|tjƒr¢|jj}nz|t	j
krdgt|ƒ }	tt||ƒƒD ]F\}\}
}t |¡sÊ|
 ¡ j}d|  }t	 
t|| ƒ¡ |¡|	|< qÊt |	¡}|||< qdS )z_
    Fix type of expected values (as read from JSON) according to
    actual ORC datatype.
    Úmapc                 S   s&   g | ]}|D ]}|  ¡ d dhk‘qqS ©ÚkeyÚvalue)Úkeys)Ú.0ÚmÚdr   r   r	   Ú
<listcomp>5   s       z&fix_example_values.<locals>.<listcomp>c                 S   s   g | ]}|d  |d f‘qS r   r   )r   r   r   r   r	   r   9   s     r   Né
   )ÚcopyÚ	enumerateÚ	__class__Ú
issubclassÚdatetimeÚpdÚto_datetimeÚdateÚdtÚdecimalÚDecimalÚlenÚzipZisnullÚas_tupleÚexponentÚroundÚscalebZSeries)Zactual_colsZexpected_colsÚnameÚexpectedÚactualÚcolÚir   ÚtypZconverted_decimalsr   ÚvÚexpÚfactorr   r   r	   Úfix_example_values,   s4    
ÿ




ÿ
r/   c                 C   s6   |d k	s|d k	r$|||… j dd}t| |dd d S )NT)ZdropF)Zcheck_dtype)Zreset_indexr   )Úorc_dfÚexpected_dfÚstartÚstopr   r   r	   Úcheck_example_valuesR   s    r4   Fc           
      C   sî   ddl m} | | ¡}| ¡ }t|tjƒs.t‚| ¡  t	 
| ¡ ¡}t|jƒt|jƒks\t‚|j |j¡sx|j|jd}|r†t||ƒ t||ƒ d}t|jƒD ]<}| |¡}	tt	 
|	 ¡ ¡|||t|	ƒ d |t|	ƒ7 }qž||jksêt‚dS )zC
    Check a ORC file against the expected columns dictionary.
    r   ©r   ©Úcolumns)r2   r3   N)Úpyarrowr   ÚORCFileÚreadÚ
isinstanceÚpaÚTableÚAssertionErrorÚvalidater   Z	DataFrameZ	to_pydictÚsetr7   ÚequalsZreindexr/   r4   ÚrangeZnstripesZread_striper    Znrows)
Zorc_pathr1   Úneed_fixr   Úorc_fileÚtabler0   Zjson_posr*   Úbatchr   r   r	   Úcheck_example_fileX   s,    




ýrG   ÚfilenamezTestOrcFile.test1.orczTestOrcFile.testDate1900.orczdecimal.orcc                 C   s2   ||  }t jt| d¡ƒdd}t||dd dS )zÊ
    Check a ORC file example against the equivalent JSON file, as given
    in the Apache ORC repository (the JSON file has one JSON object per
    line, corresponding to one row in the ORC file).
    z.jsn.gzT)Úlines)rC   N)r   Ú	read_jsonÚstrÚwith_suffixrG   )rH   r
   ÚpathrE   r   r   r	   Útest_example_using_json}   s    rN   c                 C   s(  ddl m} | | d ¡ ¡ }|jdks,t‚t dt ¡ fdt 	¡ fdt 
¡ fdt ¡ fdt ¡ fd	t ¡ fd
t ¡ fdt ¡ fdt ¡ fdt dt t dt ¡ fdt ¡ fg¡¡fg¡fdt t dt ¡ fdt ¡ fg¡¡fdt t ¡ t dt ¡ fdt ¡ fg¡¡fg¡}|j|ks$t‚d S )Nr   r5   zTestOrcFile.emptyFile.orcZboolean1Zbyte1Zshort1Zint1Zlong1Zfloat1Zdouble1Zbytes1Zstring1ÚmiddleÚlistr   )r8   r   r9   r:   Únum_rowsr>   r<   ÚschemaZbool_Zint8Zint16Úint32Úint64Zfloat32Zfloat64ÚbinaryÚstringÚstructÚlist_Zmap_)r
   r   rE   Zexpected_schemar   r   r	   Útest_orcfile_empty   s@    









ÿÿÿ
ÿÿ
ÿÿírY   c                 C   s¬   ddl m} t ddddgi¡}| d }| ¡  |d }| |t|ƒ¡ |j|t 	¡ d	}| 
|¡sht‚|jd
t | ¡d	}| 
|¡sŠt‚| t |¡¡}| 
|¡s¨t‚d S )Nr   r5   Úaé   é   é   Údata_dirzdata.orc)Ú
filesystemzdata_dir/data.orc)r8   r   r<   rE   ÚmkdirÚwrite_tablerK   Ú
read_tabler   ZLocalFileSystemrA   r>   r   Z_filesystem_uri)Útmpdirr   rE   Ú	directoryrM   Úresultr   r   r	   Útest_filesystem_uri°   s"     ÿÿrf   c                 C   sÆ   ddl m} t dd dd g¡}t d dd dg¡}t ||dœ¡}|  d¡}| ||¡ | |¡}| |¡snt	‚| |g ¡}d	|j
ksˆt	‚d|jks–t	‚|j|d
gd}d	|j
ks´t	‚d|jksÂt	‚d S )Nr   r5   r[   r]   ÚArrowÚORC©rT   Úutf8útest.orcé   rT   r6   )r8   r   r<   ÚarrayrE   Újoinra   rb   rA   r>   rQ   Znum_columns)rc   r   rZ   ÚbrE   ÚfileÚoutput_tabler   r   r	   Útest_orcfile_readwriteÈ   s    

rr   c                  C   sŒ   ddl m}  ddlm} |ƒ }t dd dd g¡}t d dd dg¡}t ||dœ¡}|  ||¡ | d¡ |  	|¡}| 
¡ }| |¡sˆt‚d S )	Nr   r5   )ÚBytesIOr[   r]   rg   rh   ri   )r8   r   Úiors   r<   rm   rE   ra   Úseekr9   r:   rA   r>   )r   rs   ÚbufrZ   ro   rE   rD   rq   r   r   r	   Útest_bytesio_readwriteÛ   s    

rw   c               	   C   sX  ddl m}  t ¡ }t dd dd g¡}t d dd dg¡}t ||dœ¡}|  ||¡ t | ¡ ¡}|  	|¡}| 
¡ }| |¡s‚t‚|jdkst‚|jd	ksžt‚|jd
ks¬t‚|jdksºt‚t ¡ }t t¡ |  ||¡ W 5 Q R X t | ¡ ¡}|  	|¡}| 
¡ }| |¡st‚|jdks$t‚|jd	ks4t‚|jd
ksDt‚|jdksTt‚d S )Nr   r5   r[   r]   rg   rh   ri   ÚUNCOMPRESSEDz0.12i'  i   ©r8   r   r<   ÚBufferOutputStreamrm   rE   ra   ZBufferReaderÚgetvaluer9   r:   rA   r>   ÚcompressionÚfile_versionÚrow_index_strideZcompression_sizeÚpytestZwarnsÚFutureWarning©r   Úbuffer_output_streamrZ   ro   rE   Zbuffer_readerrD   rq   r   r   r	   Útest_buffer_readwriteê   s2    

rƒ   c               	   C   sl  ddl m}  t ¡ }t dd dd g¡}t d dd dg¡}t ||dœ¡}| j||dd	d
dd t | ¡ ¡}|  	|¡}| 
¡ }| |¡sŒt‚|jdksšt‚|jd	ks¨t‚|jd
ks¶t‚|jdksÄt‚t ¡ }t t¡ | j||dd	ddd W 5 Q R X t | ¡ ¡}|  	|¡}| 
¡ }| |¡s(t‚|jdks8t‚|jd	ksHt‚|jdksXt‚|jdksht‚d S )Nr   r5   r[   r]   rg   rh   ri   Úsnappyz0.11iˆ  i €  )r|   r}   r~   Úcompression_block_sizeZSNAPPYZuncompressedi N  i @  rx   ry   r   r   r   r	   Ú'test_buffer_readwrite_with_writeoptions  sN    ú
ú
r†   c               	   C   s¶  ddl m}  t ¡ }t dd dd g¡}t d|i¡}t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||d	d
 W 5 Q R X t t¡ | j	||dd
 W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t
¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t
¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||dd W 5 Q R X t t¡ | j	||d d W 5 Q R X t t¡ | j	||d!d" W 5 Q R X t t¡ | j	||dd#gd" W 5 Q R X t t¡ | j	||dd$d%hd" W 5 Q R X t t¡ | j	||dd& W 5 Q R X t t¡ | j	||d'd& W 5 Q R X t t¡ | j	||d(d& W 5 Q R X d S ))Nr   r5   r[   r]   rT   )Z
batch_sizeiœÿÿÿgR¸…ë @g¤p=
×£À?)r}   z1.1)Zstripe_sizeipþÿÿg®Gáº °@)r|   ÚnoneZzlid)r…   i8ÿÿÿgR¸…ë"‘@)Zcompression_strategyÚnoZlarge)r~   iàüÿÿg®Gáz”0¨@Úcat)Zpadding_toleranceÚarrow)Zdictionary_key_size_thresholdg333333ó?gš™™™™™	ÀrV   )Zbloom_filter_columnsgffffffö?r\   éÿÿÿÿ)Zbloom_filter_fppgš™™™™™ñ?gš™™™™™¹¿)r8   r   r<   rz   rm   rE   r   ÚraisesÚ
ValueErrorra   Ú	TypeError)r   r‚   rZ   rE   r   r   r	   Ú+test_buffer_readwrite_with_bad_writeoptions;  sp   ýýýýýýýýýýýýýýýýýýýýýýýýýýýýýýr   c                 C   s  ddl m} t dt ¡ ¡}t dt |g¡¡}t dt ¡ ¡t dt t dt ¡ ¡¡¡t dt |t d	t ¡ ¡g¡¡t d
t t dt t dt ¡ ¡t d	t ¡ ¡g¡¡¡¡t dt ¡ ¡g}dgddggddiddœgdddœdddœggdgg}tj|t 	|¡d}t
| d ƒ}| ||¡ | |¡}| ¡ }	|	 |¡sJt‚|jddgd}
|
 | ddg¡¡stt‚|jdddgd}| | dddg¡¡s¢t‚|jdgd}t ddddiigi¡}| |¡sØt‚|jdgd}t dd	digi¡}| |¡s
t‚|jdddgd}| | ddg¡¡s6t‚|jdgd}t d
ddiddiggi¡}| |¡spt‚|jddgd}
|
 | ddg¡¡sšt‚|jdddgd}| | ddd
g¡¡sÈt‚t t¡ |jdgd W 5 Q R X t t¡ |jdgd W 5 Q R X d S )Nr   r5   ÚinnerrO   ÚbasicrP   ÚitemrW   Úinner2zlist-structÚinner1Zbasic2r[   r\   r]   rl   )rO   r“   é   é   )r”   r“   é   é   é	   )rR   rk   r6   zstruct.middle.innerzstruct.inner2zlist-struct.inner1Zwrong)r8   r   r<   ÚfieldrT   rW   rS   rX   rE   rR   rK   ra   r9   r:   rA   r>   Úselectr   rŒ   ÚIOErrorr   )Útempdirr   r   rO   ÚfieldsZarrsrE   rM   rD   Zresult1Zresult2Zresult3Zresult4Z	expected4Zresult5Z	expected5Zresult6Zresult7Z	expected7r   r   r	   Útest_column_selection  sz     ÿ ÿ  þÿÿð   þ
ÿrŸ   c              
   C   sP   ddl m} t| d ƒ}| |¡$}t t¡ | ¡  W 5 Q R X W 5 Q R X d S )Nr   r5   rk   )r8   r   rK   Z	ORCWriterr   rŒ   ÚAttributeErrorÚtest)r   r   rM   Úwriterr   r   r	   Útest_wrong_usage_orc_writerl  s
    r£   c              	   C   st   ddl m} t| d ƒ}t dd dd g¡}t d d d d g¡}t ||dœ¡}t tj¡ | 	||¡ W 5 Q R X d S )Nr   r5   rk   r[   r]   ri   )
r8   r   rK   r<   rm   rE   r   rŒ   ZArrowNotImplementedErrorra   )r   r   rM   rZ   ro   rE   r   r   r	   Ú test_orc_writer_with_null_arraysu  s    r¤   )NN)F)"r   r   r   r8   r<   r   Zpyarrow.testsr   Úmarkr   Z
pytestmarkZpandas.testingr   Zpandasr   ÚImportErrorZfixturer
   r/   r4   rG   ZparametrizerN   rY   rf   rr   rw   rƒ   r„   r†   r   rŸ   r£   r¤   r   r   r   r	   Ú<module>   sF   

&

%ý!!
/ cO	