U
    âÇ-e7“  ã                   @   sœ  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZ	d dl
Zd dlmZmZ zd dlmZ d dlZW n ek
rˆ   Y nX G dd„ dƒZG dd„ deƒZG d	d
„ d
eƒZG dd„ deƒZejdd„ ƒZejdd„ ƒZejdd„ ƒZejeje d¡ddeje d¡ddgddd„ ƒZdd„ Zdd„ Z dd„ Z!ej" #dd d!„ d"d!„ g¡d#d$„ ƒZ$d%d&„ Z%ej"jd'd(„ ƒZ&d)d*„ Z'd+d,„ Z(ej"jd-d.„ ƒZ)ej"jd/d0„ ƒZ*d1d2„ Z+ej" #d3ej, -¡ ej,j-d4d5g¡d6d7„ ƒZ.d8d9„ Z/ej" #d3ej, -¡ ej,j-d4d5g¡d:d;„ ƒZ0d<d=„ Z1ej"jd>d?„ ƒZ2ej"jd@dA„ ƒZ3ej" #dBd4dCg¡dDdE„ ƒZ4ej"j5dFdG„ ƒZ6dHdI„ Z7dJdK„ Z8ej" #d3ej, 9¡ ej,j9dCdLej,j9dCdMej,j9ej,j:j;dNej,j9dCej,j:j;dOg¡dPdQ„ ƒZ<dRdS„ Z=dTdU„ Z>dVdW„ Z?dXdY„ Z@dZd[„ ZAej"jd\d]„ ƒZBejd^d_„ ƒZCd`da„ ZDdbdc„ ZEddde„ ZFej"jGdfdg„ ƒZHdhdi„ ZIdjdk„ ZJdldm„ ZKG dndo„ doejLƒZMG dpdq„ dqeƒZNejdrds„ ƒZOdtdu„ ZPdvdw„ ZQej"jdxdy„ ƒZRej"jdzd{„ ƒZSej"jej" #d|d}d~g¡dd€„ ƒƒZTdd‚„ ZUej"jdƒd„„ ƒZVej"jdŸd…d†„ƒZWej"jd‡dˆ„ ƒZXej"jd‰dŠ„ ƒZYej"jd‹dŒ„ ƒZZej"jddŽ„ ƒZ[ej"jdd„ ƒZ\ej"jd‘d’„ ƒZ]ej"jd“d”„ ƒZ^d•d–„ Z_d—d˜„ Z`d™dš„ Zad›dœ„ Zbddž„ ZcdS ) é    )ÚUserListN)Úchanged_environÚinvoke_script)Úassert_frame_equalc                   @   s:   e Zd ZdZdd„ fdd„Zdd„ Zdd	„ Zddd„ZdS )Ú
IpcFixtureNc                   C   s   t  ¡ S ©N©ÚioÚBytesIO© r   r   úW/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/test_ipc.pyÚ<lambda>*   ó    zIpcFixture.<lambda>c                 C   s   || _ |  ¡ | _d S r   )Ú_sink_factoryÚget_sinkÚsink)ÚselfÚsink_factoryr   r   r   Ú__init__*   s    zIpcFixture.__init__c                 C   s   |   ¡ S r   )r   ©r   r   r   r   r   .   s    zIpcFixture.get_sinkc                 C   s
   | j  ¡ S r   )r   Úgetvaluer   r   r   r   Ú
get_source1   s    zIpcFixture.get_sourceé   Fc           
      C   s´   d}t  dt  ¡ fdt  ¡ fg¡}|  | j|¡}g }t|ƒD ]2}t jtj	 
|¡dd dddgg|d}| |¡ q<|rŒt j |¡}	| |	¡ n|D ]}| |¡ q|j| _| ¡  |S )	Nr   ÚoneÚtwoÚfooÚbarÚbazbazÚqux©Úschema)Úpar    Zfloat64Úutf8Ú_get_writerr   ÚrangeÚrecord_batchÚnpÚrandomÚrandnÚappendÚTableÚfrom_batchesÚwrite_tableÚwrite_batchÚstatsÚwrite_statsÚclose)
r   Znum_batchesÚas_tableÚnrowsr    ÚwriterÚbatchesÚiÚbatchÚtabler   r   r   Úwrite_batches4   s(    
ÿýzIpcFixture.write_batches)r   F)Ú__name__Ú
__module__Ú__qualname__r/   r   r   r   r8   r   r   r   r   r   '   s
   r   c                   @   s&   e Zd ZdZdZdd„ Zddd„ZdS )	ÚFileFormatFixtureTNc                 C   s   t jj||| jdS )N©Úoptions)r!   ÚipcÚnew_filer>   ©r   r   r    r   r   r   r#   S   s    zFileFormatFixture._get_writerFc                 C   sÀ   | j |d}t |  ¡ ¡}tj |¡}|jt|ƒks8t‚t	|ƒD ]:\}}| 
|¡}||  |¡sdt‚|j |d j¡s@t‚q@t|jtjjƒst‚t| jtjjƒs¤t‚t|jƒt| jƒks¼t‚d S )N©r1   r   )r8   r!   ÚBufferReaderr   r?   Ú	open_fileÚnum_record_batchesÚlenÚAssertionErrorÚ	enumerateÚ	get_batchÚequalsr    Ú
isinstancer.   Z	ReadStatsr/   Z
WriteStatsÚtuple)r   r1   r4   Úfile_contentsÚreaderr5   r6   r   r   r   Ú_check_roundtripV   s    
z"FileFormatFixture._check_roundtrip)F)r9   r:   r;   Úis_filer>   r#   rO   r   r   r   r   r<   N   s   r<   c                   @   s    e Zd ZdZdZdZdd„ ZdS )ÚStreamFormatFixtureFNc                 C   s   t jj||| j| jdS )N)Úuse_legacy_formatr>   )r!   r?   Ú
new_streamÚuse_legacy_ipc_formatr>   rA   r   r   r   r#   q   s    üzStreamFormatFixture._get_writer)r9   r:   r;   rT   r>   rP   r#   r   r   r   r   rQ   i   s   rQ   c                   @   s   e Zd Zdd„ ZdS )ÚMessageFixturec                 C   s   t  ||¡S r   ©r!   ZRecordBatchStreamWriterrA   r   r   r   r#   |   s    zMessageFixture._get_writerN)r9   r:   r;   r#   r   r   r   r   rU   z   s   rU   c                   C   s   t ƒ S r   )r   r   r   r   r   Úipc_fixture€   s    rW   c                   C   s   t ƒ S r   )r<   r   r   r   r   Úfile_fixture…   s    rX   c                   C   s   t ƒ S r   )rQ   r   r   r   r   Ústream_fixtureŠ   s    rY   zFile Format)ÚidzStream Format)Úparamsc                 C   s   | j S r   )Úparam)Úrequestr   r   r   Úformat_fixture   s    r^   c               	   C   s2   d} t  tj¡ tj t | ¡¡ W 5 Q R X d S ©Nr   )ÚpytestÚraisesr!   ÚArrowInvalidr?   rD   rC   ©Úbufr   r   r   Útest_empty_file   s    re   c                 C   s   | j dd d S )NFrB   ©rO   ©rX   r   r   r   Útest_file_simple_roundtrip£   s    rh   c                 C   s   | j dd d S )NTrB   rf   rg   r   r   r   Útest_file_write_table§   s    ri   r   c                   C   s   t  ¡ S r   r   r   r   r   r   r   ¬   r   r   c                   C   s   t  ¡ S r   )r!   ÚBufferOutputStreamr   r   r   r   r   ­   r   c                 C   sP   t | ƒ}| ¡ }t | ¡ ¡}tj |¡}| ¡ }tj 	|¡}| 
|¡sLt‚d S r   )r<   r8   r!   rC   r   r?   rD   Úread_allr*   r+   rJ   rG   )r   Úfixturer4   rM   rN   ÚresultÚexpectedr   r   r   Útest_file_read_all«   s    ro   c           	      C   s®   |   ¡  |  ¡ }tj |¡}tj t |¡¡}t |¡}| ¡ }| ¡ }| ¡ }| |¡s^t	‚| |¡slt	‚|j
}|jdks€t	‚|jdksŽt	‚|j
|ksœt	‚|j
|ksªt	‚d S ©Né   r   )r8   r   r!   r?   rD   rC   ÚRecordBatchFileReaderrk   rJ   rG   r.   Únum_messagesrE   )	rX   ÚsourceÚreader1Úreader2Úreader3Úresult1Úresult2Úresult3Úst1r   r   r   Útest_open_file_from_buffer¼   s    
r|   c                 C   sT   dd„ |   ¡ D ƒ}t |  ¡ ¡}tj |¡}| ¡ }t |¡j	dd}t
||ƒ d S )Nc                 S   s   g | ]}|  ¡ ‘qS r   ©Ú	to_pandas©Ú.0r6   r   r   r   Ú
<listcomp>Õ   s     z)test_file_read_pandas.<locals>.<listcomp>T©Zdrop)r8   r!   rC   r   r?   rD   Úread_pandasÚpdÚconcatÚreset_indexr   )rX   ÚframesrM   rN   rm   rn   r   r   r   Útest_file_read_pandasÓ   s    rˆ   c              	   C   sz   |   ¡  |  ¡ }| d¡j}t|dƒ}| |¡ W 5 Q R X tj t	 
|¡¡ ¡ }tj t |¡¡ ¡ }| |¡svt‚d S )Nú
file.arrowÚwb)r8   r   ÚjoinÚstrpathÚopenÚwriter!   r?   rD   ÚpathlibÚPathrk   ÚOSFilerJ   rG   )rX   Útmpdirrt   ÚpathÚfÚt1Út2r   r   r   Útest_file_pathlibß   s    r—   c               	   C   s2   t  d¡} t tj¡ tj | ¡ W 5 Q R X d S r_   )r	   r
   r`   ra   r!   rb   r?   Úopen_streamrc   r   r   r   Útest_empty_streamí   s    
r™   c              
   C   s   t  ¡ }t  t  d|¡g¡}|  d¡j}t  |d¡H}t j ||¡.}t j	dg|d}t  
|g|¡}| |¡ W 5 Q R X W 5 Q R X td|ƒ dS )zËARROW-15783: Verify to_pandas works for interval types.

    Interval types require static structures to be enabled. This test verifies
    that they are when no other library functions are invoked.
    Únumsr‰   rŠ   )é   é   é   ©Útypezread_record_batch.pyN)r!   Zmonth_day_nano_intervalr    Úfieldr‹   rŒ   r‘   r?   r@   Úarrayr%   rŽ   r   )r’   Zmdn_interval_typer    r“   r   r3   Zinterval_arrayr6   r   r   r   Ú"test_read_year_month_nano_intervaló   s    r¢   c              	   C   s’   t  tj d¡t jdtjdddgddgdddœ¡}tj 	|¡}|  
| j|j¡}| |¡ W 5 Q R X tj t |  ¡ ¡¡ ¡ }t| ¡ |ƒ d S )Nr   r   r   T©Ú
categoriesZordered©r   r   )r„   Ú	DataFramer&   r'   r(   ÚCategoricalÚnanr!   ÚRecordBatchÚfrom_pandasr#   r   r    r-   r?   r˜   rC   r   rk   r   r~   )rY   Údfr6   Úwrr7   r   r   r   Ú!test_stream_categorical_roundtrip  s    
þþr­   c           	      C   sÄ   |   ¡  |  ¡ }tj |¡}tj t |¡¡}t |¡}| ¡ }| ¡ }| ¡ }| |¡s^t	‚| |¡slt	‚|j
}|jdks€t	‚|jdksŽt	‚|j
|ksœt	‚|j
|ksªt	‚t|ƒt| jƒksÀt	‚d S rp   )r8   r   r!   r?   r˜   rC   ZRecordBatchStreamReaderrk   rJ   rG   r.   rs   rE   rL   r/   )	rY   rt   ru   rv   rw   rx   ry   rz   r{   r   r   r   Útest_open_stream_from_buffer  s     
r®   r>   F©Úuse_threadsc                 C   sd   |   ¡  |  ¡ }tjj||d}| ¡  |j}|jdks<t‚|j	dksJt‚t
|ƒt
| jƒks`t‚d S ©Nr=   rq   r   )r8   r   r!   r?   r˜   rk   r.   rs   rG   rE   rL   r/   )rY   r>   rt   rN   Ústr   r   r   Útest_open_stream_options0  s    r³   c              	   C   s:   |   ¡  |  ¡ }t t¡ tjj|dd W 5 Q R X d S ©NTr=   )r8   r   r`   ra   Ú	TypeErrorr!   r?   r˜   )rY   rt   r   r   r   Ú#test_open_stream_with_wrong_optionsB  s    r¶   c                 C   sN   |   ¡  |  ¡ }tjj||d}| ¡  |j}|jdks<t‚|j	dksJt‚d S r±   )
r8   r   r!   r?   rD   rk   r.   rs   rG   rE   )rX   r>   rt   rN   r²   r   r   r   Útest_open_file_optionsJ  s    r·   c              	   C   s:   |   ¡  |  ¡ }t t¡ tjj|dd W 5 Q R X d S r´   )r8   r   r`   ra   rµ   r!   r?   rD   )rX   rt   r   r   r   Ú!test_open_file_with_wrong_options[  s    r¸   c              	   C   s¾   t  tj d¡t jdtjdddgddgdddœ¡}tjj	|dd}tj
j	|dd}|  | j|j¡}| |¡ | |¡ W 5 Q R X tj t |  ¡ ¡¡ ¡ }t| ¡ t j||gdd	ƒ d S )
Nr   r   r   Tr£   r¥   F©Zpreserve_index©Zignore_index)r„   r¦   r&   r'   r(   r§   r¨   r!   r*   rª   r©   r#   r   r    rŽ   r?   r˜   rC   r   rk   r   r~   r…   )rY   r«   r7   r6   r¬   r   r   r   Útest_stream_write_dispatchc  s     
þþ
ÿr»   c              	   C   sê   t  dtj d¡i¡}tjj|d d… dd}tjj|dd}tj 	|||g¡}|  
| j|j¡}|j|dd W 5 Q R X ttj |  ¡ ¡ƒ}ttt|ƒƒddddgks¬t‚tj 	|¡}t| ¡ t j|d d… ||d d… gd	d
ƒ d S )Nr   é   é
   Fr¹   é   )Zmax_chunksizer   Trº   )r„   r¦   r&   r'   r(   r!   r©   rª   r*   r+   r#   r   r    r,   Úlistr?   r˜   r   ÚmaprF   rG   r   r~   r…   )rY   r«   Úb1Úb2r7   r¬   r4   Zresult_tabler   r   r   Útest_stream_write_table_batchesx  s"     
ÿÿÿrÃ   rT   Tc              	   C   s    || _ |  ¡ }t |  ¡ ¡}tj |¡}|j |d j¡s>t	‚d}t
|ƒD ]"\}}| || ¡sdt	‚|d7 }qJ|t|ƒks~t	‚t t¡ | ¡  W 5 Q R X d S ©Nr   r›   )rT   r8   r!   rC   r   r?   r˜   r    rJ   rG   rH   rF   r`   ra   ÚStopIterationÚread_next_batch)rY   rT   r4   rM   rN   Útotalr5   Ú
next_batchr   r   r   Útest_stream_simple_roundtrip  s    
rÉ   c               	   C   s  t  ¡ } tj ddd¡}tjj|gdgd}tjj	dd}tjj
| |j|d}| |¡ W 5 Q R X t|  ¡ ƒ}t  ¡ }tjdd	d
}tjj	|d}tjj
||j|d}| |¡ W 5 Q R X t| ¡ ƒ}||ksÖt‚tj | ¡ ¡ }	tj |¡ ¡ }
|	|
kst‚d S )Nr   r   i'  Úvalues©ÚnamesÚzstd©Úcompressionr=   r   )Zcompression_level)r	   r
   r&   r'   Úrandintr!   r*   Úfrom_arraysr?   ÚIpcWriteOptionsZRecordBatchFileWriterr    r,   rF   r   ÚCodecrG   rD   rk   )r   rÊ   r7   r>   r3   Zlen1Zsink2ÚcodecZlen2r•   r–   r   r   r   Útest_compression_roundtrip¤  s6      ÿ  ÿrÕ   c               
   C   sØ  t j ¡ } | jdkst‚| jdks&t‚| jt jjjks:t‚d| _| jdksNt‚d| _| jdksbt‚t jjj	| _| jt jjj	ks‚t‚dD ]$}t
 ttf¡ || _W 5 Q R X q†| jd ksºt‚dD ]<}t j |¡r¾|| _| j|ksât‚| ¡ | _| j|ks¾t‚q¾d | _| jd kst‚t
 t¡ d| _W 5 Q R X | jdks>t‚d| _| jdksTt‚t j d¡rÔt jjt jjj	ddddd} | jt jjj	ks”t‚| jdks¤t‚| jdks´t‚| jdksÄt‚| jdksÔt‚d S )NFT)ÚV5é*   )Úlz4rÍ   r   rØ   )Úmetadata_versionÚallow_64bitrR   rÏ   r°   )r!   r?   rÒ   rÚ   rG   rR   rÙ   ÚMetadataVersionrÖ   ÚV4r`   ra   rµ   Ú
ValueErrorrÏ   rÓ   Zis_availableÚupperr°   )r>   Úvaluer   r   r   Útest_write_optionsÃ  sP    

ûrà   c              	   C   s8   t jtdd  d| _tj ¡ | _|  ¡  W 5 Q R X d S )Nz4provide at most one of options and use_legacy_format©ÚmatchT)	r`   ra   rÝ   rT   r!   r?   rÒ   r>   r8   )rY   r   r   r   Ú#test_write_options_legacy_exclusiveô  s    þrã   )rÚ   )rR   )rÙ   )rR   rÙ   c           	   	   C   sÆ   d | _ || _|  ¡ }t |  ¡ ¡}tj |  ¡ ¡}|j|jksBt	‚tj 
|¡}|j |d j¡sdt	‚d}t|ƒD ]"\}}| || ¡sŠt	‚|d7 }qp|t|ƒks¤t	‚t t¡ | ¡  W 5 Q R X d S rÄ   )rT   r>   r8   r!   rC   r   r?   Úread_messagerÙ   rG   r˜   r    rJ   rH   rF   r`   ra   rÅ   rÆ   )	rY   r>   r4   rM   ÚmessagerN   rÇ   r5   rÈ   r   r   r   Útest_stream_options_roundtripý  s    	
ræ   c               	   C   sØ   t j ¡ } | jdkst‚| jdks&t‚| jg ks4t‚d| _| jdksHt‚d| _| jdks\t‚ddg| _| jddgksxt‚t t	¡ d | _W 5 Q R X t jjdddgd} | jdks¶t‚| jdksÄt‚| jdgksÔt‚d S )NTFr   r›   )r°   Úensure_native_endianÚincluded_fields)
r!   r?   ÚIpcReadOptionsr°   rG   rç   rè   r`   ra   rµ   r=   r   r   r   Útest_read_options  s(    

 þrê   c              	   C   s  t j ¡ }t jjdgd}t jjt  ddddg¡t  dddd	g¡gd
dgd}|  | j|j¡}| 	|¡ W 5 Q R X |  
¡ }t jj||d}t jj||t  ¡ d}| ¡ }| ¡ }	|jdksÀt‚|	jdksÎt‚t jjt  dddd	g¡gdgd}
|	|
ksüt‚||ks
t‚d S )Nr›   )rè   r   r   Úbazr   rœ   r   é   ÚaÚbrË   r=   )r>   Zmemory_pool)r!   r?   ré   r*   rÑ   r¡   r#   r   r    r,   r   r˜   Zsystem_memory_poolrk   Znum_columnsrG   )rY   Zoptions1Zoptions2r7   r¬   rt   ru   rv   rx   ry   rn   r   r   r   Ú!test_read_options_included_fields8  s.    
ÿþ  ÿ"rï   c              	      sj  t  t  ¡ t  ¡ ¡‰ddd gdddgddgdd ddgddgg}‡fdd„|D ƒ}|d d… }|d j‰d‡ ‡fd	d
„	}ˆ jr¾t t j¡ ||ƒ W 5 Q R X t t j¡ ||ƒ W 5 Q R X nB||ƒ}|j	dksÔt
‚|jdksât
‚|jdksðt
‚|jdks t
‚d ˆ _t jjddˆ _ˆ jr@t t j¡ ||ƒ W 5 Q R X nH||ƒ}|j	dksXt
‚|jdksht
‚|jdksxt
‚|jdksˆt
‚||ƒ}|j	dks t
‚|jdks°t
‚|jdksÀt
‚|jdksÐt
‚t jjddˆ _||dd}|j	dksüt
‚ˆ jr6|jdkst
‚|jdks$t
‚|jdksft
‚n0|jdksFt
‚|jdksVt
‚|jdksft
‚d S )Nr   r   Zquuxc                    s*   g | ]"}t jjt j|ˆ d gdgd‘qS )rž   ZdictsrË   )r!   r©   rÑ   r¡   )r€   Úv)Útyr   r   r   Y  s   ÿz)test_dictionary_delta.<locals>.<listcomp>rì   r   Fc              
      s^   ˆ   t ¡ ˆ¡D}|r.tj | ¡}| |¡ n| D ]}| |¡ q2|jW  5 Q R £ S Q R X d S r   )r#   r!   ZMockOutputStreamr*   r+   r,   r-   r.   )r4   r1   r3   r7   r6   )r^   r    r   r   r8   _  s    
ÿz,test_dictionary_delta.<locals>.write_batchesr   r   T)Zemit_dictionary_deltasr›   rœ   )Zunify_dictionariesrB   )F)r!   Ú
dictionaryÚint8r"   r    rP   r`   ra   rb   rE   rG   Znum_dictionary_batchesZnum_replaced_dictionariesZnum_dictionary_deltasrT   r?   rÒ   r>   )r^   Údatar4   Zbatches_delta_onlyr8   r²   r   )r^   r    rñ   r   Útest_dictionary_deltaQ  sh    
ü
þ
ÿÿrõ   c               
   C   s  t  t  dt  ¡ ¡g¡} t j t  ¡ | ¡}|jr4t‚|j	t jj
jksHt‚t j t  ¡ | ¡}|jrdt‚|j	t jj
jksxt‚tddƒf t j t  ¡ | ¡}|js t‚|j	t jj
jks´t‚t j t  ¡ | ¡}|jsÐt‚|j	t jj
jksät‚W 5 Q R X tddƒn t j t  ¡ | ¡}|jrt‚|j	t jj
jks.t‚t j t  ¡ | ¡}|jrLt‚|j	t jj
jksbt‚W 5 Q R X tddƒ„ tddƒn t j t  ¡ | ¡}|js¢t‚|j	t jj
jks¸t‚t j t  ¡ | ¡}|jsÖt‚|j	t jj
jksìt‚W 5 Q R X W 5 Q R X d S )Nr   ZARROW_PRE_0_15_IPC_FORMATÚ1ZARROW_PRE_1_0_METADATA_VERSION)r!   r    r    Zint32r?   rS   rj   Z_use_legacy_formatrG   Z_metadata_versionrÛ   rÖ   r@   r   rÜ   )r    r3   r   r   r   Ú!test_envvar_set_legacy_ipc_formatœ  s:    



 r÷   c                 C   sH   |   ¡ }t |  ¡ ¡}tj |¡}| ¡ }tj |¡}| 	|¡sDt
‚d S r   )r8   r!   rC   r   r?   r˜   rk   r*   r+   rJ   rG   )rY   r4   rM   rN   rm   rn   r   r   r   Útest_stream_read_allÀ  s    rø   c                 C   sN   dd„ |   ¡ D ƒ}|  ¡ }tj |¡}| ¡ }t |¡jdd}t	||ƒ d S )Nc                 S   s   g | ]}|  ¡ ‘qS r   r}   r   r   r   r   r   Ì  s     z+test_stream_read_pandas.<locals>.<listcomp>Tr‚   )
r8   r   r!   r?   r˜   rƒ   r„   r…   r†   r   )rY   r‡   rM   rN   rm   rn   r   r   r   Útest_stream_read_pandasÊ  s    rù   c                 C   s2   |   ¡ }|  ¡ }t |¡}tj |¡}|t|ƒfS r   )r8   r   r!   rC   ÚMessageReaderr˜   r¿   )rY   r4   rM   Z
buf_readerrN   r   r   r   Úexample_messagesÕ  s
    
rû   c                	   C   sH   t  t¡ tt ¡ ƒ W 5 Q R X t  t¡ tt ¡ ƒ W 5 Q R X d S r   )r`   ra   rµ   Úreprr!   ÚMessagerú   r   r   r   r   Útest_message_ctors_no_segfaultÞ  s    rþ   c                 C   sÆ   | \}}t |ƒdkst‚|d jdks*t‚t|d jtjƒs@t‚t|d jtjƒsVt‚|d jtj	j
kslt‚|dd … D ]H}|jdksŠt‚t|jtjƒsœt‚t|jtjƒs®t‚|jtj	j
ksxt‚qxd S )Nrq   r   r    r›   zrecord batch)rF   rG   rŸ   rK   Úmetadatar!   ÚBufferÚbodyrÙ   rÛ   rÖ   )rû   Ú_ÚmessagesÚmsgr   r   r   Útest_message_readeræ  s    r  c           
   	   C   sê   | \}}|d }|  ¡ }t | ¡ d ¡}tj |¡}tj |¡}tj | ¡ ¡}tj |¡}	| |¡slt‚| |¡szt‚| |¡sˆt‚| |	¡s–t‚tj	tj
dd tj t d¡¡ W 5 Q R X t 	t¡ tj |¡ W 5 Q R X d S )Nr   rœ   zCorrupted messagerá   s   ab)Ú	serializer!   rC   Z
to_pybytesr?   rä   rJ   rG   r`   ra   rb   ÚEOFError)
rû   r  r  r  rd   rN   ZrestoredZ	restored2Z	restored3Z	restored4r   r   r   Ú#test_message_serialize_read_messageö  s     r  c              
   C   sn   | \}}|D ]\}t  ¡ }t j|dd}| |¡ W 5 Q R X | ¡ }t j t j|dd¡}| |¡st	‚qd S )NÚgziprÎ   )
r!   rj   Zoutput_streamZserialize_tor   r?   rä   Zinput_streamrJ   rG   )rû   r  r  rå   Zraw_outZcompressed_outZcompressed_bufrm   r   r   r   Ú!test_message_read_from_compressed  s    ÿr
  c                 C   s0   | \}}t j |d ¡}| |d j¡s,t‚d S rÄ   )r!   r?   Úread_schemarJ   r    rG   )rû   r4   r  r    r   r   r   Útest_message_read_schema  s    r  c                 C   sF   | \}}t ||dd … ƒD ]&\}}tj ||j¡}| |¡st‚qd S )Nr›   )Úzipr!   r?   Úread_record_batchr    rJ   rG   )rû   r4   r  r6   rå   Z
read_batchr   r   r   Útest_message_read_record_batch$  s    r  c               	   C   s†   t jt jdgt  ¡ dgdgd} t  ¡ }t j || j¡}| | ¡ W 5 Q R X | 	¡ }t
jtdd t j || j¡ W 5 Q R X d S )Nó   foorž   ÚstrsrË   z type record batch but got schemará   )r!   r%   r¡   r"   rj   r?   rS   r    r-   r   r`   ra   ÚIOErrorr  )r6   Ústreamr3   rd   r   r   r   Ú.test_read_record_batch_on_stream_error_message,  s    ÿÿr  c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚStreamReaderServerc                 C   sT   t   t jt j¡| _| j d¡ | j d¡ | j ¡ \}}|| _d | _g | _	d | _
|S )N)ú	127.0.0.1r   r›   )ÚsocketÚAF_INETÚSOCK_STREAMÚ_sockÚbindÚlistenÚgetsocknameÚ_do_read_allÚ_schemaÚ_batchesÚ_table)r   Údo_read_allÚhostÚportr   r   r   Úinit?  s    zStreamReaderServer.initc                 C   s|   | j  ¡ \}}zT|jdd}tj |¡}|j| _| j	rB| 
¡ | _nt|ƒD ]\}}| j |¡ qJW 5 | ¡  | j  ¡  X d S )NÚrb©Úmode)r  Úacceptr0   Úmakefiler!   r?   r˜   r    r  r  rk   r!  rH   r   r)   )r   Ú
connectionÚclient_addressrt   rN   r5   r6   r   r   r   ÚrunJ  s    zStreamReaderServer.runc                 C   s   | j | jr| jn| jfS r   )r  r  r!  r   r   r   r   r   Ú
get_resultY  s    ÿzStreamReaderServer.get_resultN)r9   r:   r;   r%  r-  r.  r   r   r   r   r  =  s   r  c                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚSocketStreamFixturec                 C   s   d S r   r   r   r   r   r   r   `  s    zSocketStreamFixture.__init__c                 C   sN   t ƒ | _| j |¡}| j ¡  t tjtj¡| _| j d|f¡ |  	¡ | _
d S )Nr  )r  Ú_serverr%  Ústartr  r  r  r  Úconnectr   r   )r   r"  r$  r   r   r   Ústart_servere  s    
z SocketStreamFixture.start_serverc                 C   sD   dd l }| j | dd¡¡ | j ¡  | j ¡  | j ¡  | j 	¡ S )Nr   ÚQ)
Ústructr   rŽ   ÚpackÚflushr  r0   r0  r‹   r.  )r   r5  r   r   r   Ústop_and_get_resultm  s    


z'SocketStreamFixture.stop_and_get_resultc                 C   s   | j jddS )NrŠ   r'  )r  r*  r   r   r   r   r   u  s    zSocketStreamFixture.get_sinkc                 C   s   t  ||¡S r   rV   rA   r   r   r   r#   x  s    zSocketStreamFixture._get_writerN)r9   r:   r;   r   r3  r8  r   r#   r   r   r   r   r/  ^  s
   r/  c                   C   s   t ƒ S r   )r/  r   r   r   r   Úsocket_fixture|  s    r9  c                 C   sp   | j dd |  ¡ }|  ¡ \}}| |d j¡s4t‚t|ƒt|ƒksHt‚t|ƒD ]\}}||  |¡sPt‚qPd S )NF©r"  r   )r3  r8   r8  rJ   r    rG   rF   rH   )r9  Úwriter_batchesZreader_schemaZreader_batchesr5   r6   r   r   r   Útest_socket_simple_roundtrip  s    r<  c                 C   s>   | j dd |  ¡ }|  ¡ \}}tj |¡}| |¡s:t‚d S )NTr:  )r3  r8   r8  r!   r*   r+   rJ   rG   )r9  r;  r  rm   rn   r   r   r   Útest_socket_read_allŒ  s
    r=  c                  C   s`   t  ddgi¡} tj | ¡}t ¡ }t||ƒ | ¡ }tj 	|dd … ¡}| 
¡ }t| |ƒ d S )Nr   ç      ø?é   )r„   r¦   r!   r©   rª   rj   Ú
write_filer   r?   r˜   rƒ   r   )r«   r6   r   ÚbufferrN   Úrdfr   r   r   Útest_ipc_file_stream_has_eos˜  s    
rC  c                  C   sl   t  ddgi¡} tj | ¡}t ¡ }t||ƒ | ¡ }t |¡}t	|ƒ}|d  
¡ }t  |¡}t| |ƒ d S )Nr   r>  r   )r„   r¦   r!   r©   rª   rj   r@  r   rC   Ú	read_filer~   r   )r«   r6   r   rA  rN   r4   rô   rB  r   r   r   Útest_ipc_zero_copy_numpyª  s    


rE  Úipc_typer  Úfilec           
   	      sn  t  ddgi¡}tj |¡}t ¡ }d}tjjtjjdœ|  }|||j	ƒ4}t
|ƒD ]}|j|dt|ƒid qT| |¡ W 5 Q R X | ¡ }| dkrºtj |¡‰ tˆ  ¡ ƒ}	W 5 Q R X n0tj |¡‰ ‡ fdd	„t
ˆ jƒD ƒ}	W 5 Q R X t
|ƒD ]L}|	| jjd
kst‚t|	| jtjƒs$t‚|	| jdt|ƒiksòt‚qò|	| jjd
ksVt‚|	| jd ksjt‚d S )Nr   r>  rœ   )r  rG  Zbatch_id)Úcustom_metadatar  c                    s   g | ]}ˆ   |¡‘qS r   )Zget_batch_with_custom_metadata©r€   r5   ©rN   r   r   r   Ô  s   ÿz?test_batches_with_custom_metadata_roundtrip.<locals>.<listcomp>r›   )r„   r¦   r!   r©   rª   rj   r?   rS   r@   r    r$   r-   Ústrr   r˜   r¿   Z!iter_batches_with_custom_metadatarD   rE   r6   Znum_rowsrG   rK   rH  ZKeyValueMetadata)
rF  r«   r6   r   Zbatch_countZfile_factoryr3   r5   rA  Zbatch_with_metasr   rJ  r   Ú+test_batches_with_custom_metadata_roundtrip»  s<    ÿÿ
ÿ ÿ
rL  c               	   C   s¦   t jjt  ddddg¡t  ddddg¡gd	d
gd} t  ¡ }t j || j¡ W 5 Q R X | ¡ }t j 	|¡}| 
¡ }W 5 Q R X |j | j¡s’t‚t|ƒdks¢t‚d S )Nr›   rœ   r   rì   r   r   rë   r   rí   rî   rË   r   )r!   r*   rÑ   r¡   rj   r?   rS   r    r   r˜   rk   rJ   rG   rF   )r7   r   rt   rN   rm   r   r   r   Útest_ipc_stream_no_batchesâ  s    ÿþ
rM  c                  C   sF   d} d}t  dtj | ¡i¡}tj |¡}tj 	|¡| | ksBt
‚d S )Nr½   r?  r   )r„   r¦   r&   r'   r(   r!   r©   rª   r?   Zget_record_batch_sizerG   )ÚNÚitemsizer«   r6   r   r   r   Útest_get_record_batch_sizeô  s
    rP  c                 C   s2   t j| |rdndd}t j||d}t|| ƒ d S )Nrœ   r›   )Znthreadsr¯   )r!   Úserialize_pandasÚdeserialize_pandasr   )r«   r°   rd   rm   r   r   r   Ú"_check_serialize_pandas_round_tripþ  s    rS  c                  C   sH   t jdddgdd} ddg}t jdd	d
gtdƒdœ| |d}t|ƒ d S )Nr›   rœ   r   Úmy_index©Únamer   r   r>  çš™™™™™ù?ç333333û?Úabc©r   r   ©ÚindexÚcolumns©r„   ÚIndexr¦   r¿   rS  ©r\  r]  r«   r   r   r   Ú test_pandas_serialize_round_trip  s     þra  c                  C   sL   t jdddgdd} ddg}t jdd	d
gtdƒdœ| |d}t|dd d S )Nr›   rœ   r   rT  rU  r   r   r>  rW  rX  rY  rZ  r[  Tr¯   r^  r`  r   r   r   Ú)test_pandas_serialize_round_trip_nthreads  s     þrb  c                  C   sj   t jdddgdd} t jtdƒd d}t j | |g¡}ddg}t jd	d
dgtdƒdœ||d}t|ƒ d S )Nr›   rœ   r   Zlevel_1rU  Údefr   r   r>  rW  rX  rY  rZ  r[  )r„   r_  r¿   Z
MultiIndexrÑ   r¦   rS  )Zindex1Úindex2r\  r]  r«   r   r   r   Ú,test_pandas_serialize_round_trip_multi_index  s    ýre  c                  C   s   t  ¡ } t| ƒ d S r   )r„   r¦   rS  )r«   r   r   r   Ú%test_serialize_pandas_empty_dataframe*  s    rf  c                  C   s<   t  ttdddgdƒƒ¡} t | ¡}t |¡}t|| ƒ d S )Nr>  rW  rX  rY  )r„   r¦   r¿   r  r!   rQ  rR  r   )r«   rd   rm   r   r   r   Ú3test_pandas_serialize_round_trip_not_string_columns0  s    

rg  c                  C   sz   t jddddgidddgd} t  ddddgi¡}tj| dd}t |¡}t||ƒ tj| dd}t |¡}t|| ƒ d S )	Nrí   r›   rœ   r   )r\  Fr¹   T)r„   r¦   r!   rQ  rR  r   )r«   rn   rd   rm   r   r   r   Ú'test_serialize_pandas_no_preserve_index8  s    


rh  c                  C   sr   d} t  tj | ¡dtjdddgdœ¡}tj |¡}|j	 
¡ }| 
¡ }tj |¡}tj ||¡}| |¡snt‚d S )Nr   r   r   r   r   r¥   )r„   r¦   r&   r'   r(   r¨   r!   r©   rª   r    r  r?   r  r  rJ   rG   )r2   r«   r6   Ús_schemaZs_batchÚrecons_schemaZrecons_batchr   r   r   Ú#test_schema_batch_serialize_methodsF  s    
þ
rk  c                  C   s¢   dddœ} dddœ}t  dt  ¡ ¡}t jdt  ¡ | d}t j||g|d}| ¡ }t j |¡}| |¡slt	‚|j
|kszt	‚|d j
d ksŒt	‚|d	 j
| ksžt	‚d S )
Ns   bars   field)r  s   kinds   schemarí   rî   )rÿ   r   r›   )r!   r    ró   Ústringr    r  r?   r  rJ   rG   rÿ   )Zfield_metadataZschema_metadataZf0Úf1r    ri  rj  r   r   r   Ú'test_schema_serialization_with_metadataV  s    

rn  c              	   C   s*   t j || j¡}| | ¡ W 5 Q R X d S r   )r!   r?   r@   r    r-   )r6   r   r3   r   r   r   r@  h  s    r@  c              
      s<   t j | ¡&‰ ‡ fdd„tˆ jƒD ƒW  5 Q R £ S Q R X d S )Nc                    s   g | ]}ˆ   |¡‘qS r   )rI   rI  rJ  r   r   r   o  s     zread_file.<locals>.<listcomp>)r!   r?   rD   r$   rE   )rt   r   rJ  r   rD  m  s    rD  c               	   C   s†   t  dt  ¡ fg¡} t  ¡ }t j || ¡ W 5 Q R X | ¡ }t  t  |¡¡}| 	¡ }W 5 Q R X t
|ƒdksrt‚|j | ¡s‚t‚d S )Nr    r   )r!   r    Úint64rj   r?   r@   r   rr   rC   rk   rF   rG   rJ   )r    r   rd   rN   r7   r   r   r   Útest_write_empty_ipc_filer  s    
rp  c               	      s8  dd„ ‰ ‡ fdd„} t | ƒ ƒ}t |¡}tj ˆ ƒ |¡8}d }|ƒ d k	sLt‚t|ƒ| ƒ ks^t‚|ƒ d kslt‚W 5 Q R X tt | ƒ ƒƒ}t |¡}tj ˆ ƒ |¡8}d }|ƒ d k	s²t‚t|ƒ| ƒ ksÄt‚|ƒ d ksÒt‚W 5 Q R X | ƒ }t	 
t¡ tj dt ¡ fg|¡}W 5 Q R X t	 
t¡ tj d |¡}W 5 Q R X d S )Nc                   S   s   t  dt  ¡ fg¡S )Nr    )r!   r    ro  r   r   r   r   Úmake_schemaƒ  s    z0test_py_record_batch_reader.<locals>.make_schemac                     s8   ˆ ƒ } t jdddgg| d}t jddgg| d}||gS )Nr›   rœ   r   r   rì   r   )r!   r%   )r    Zbatch1Zbatch2©rq  r   r   Úmake_batches†  s    z1test_py_record_batch_reader.<locals>.make_batchesr    )r   ÚweakrefÚrefr!   ZRecordBatchReaderr+   rG   r¿   Úiterr`   ra   rµ   ro  )rs  r4   r¬   rN   r   rr  r   Útest_py_record_batch_reader‚  s@    


ÿ

ÿ ÿ
rw  )F)dÚcollectionsr   r	   r   r`   r  Ú	threadingrt  Únumpyr&   Zpyarrowr!   Zpyarrow.tests.utilr   r   Zpandas.testingr   Zpandasr„   ÚImportErrorr   r<   rQ   rU   rl   rW   rX   rY   r\   Zlazy_fixturer^   re   rh   ri   ÚmarkZparametrizero   r|   rˆ   r—   r™   r¢   r­   r®   r?   ré   r³   r¶   r·   r¸   r»   rÃ   rÉ   rÍ   rÕ   rà   rã   rÒ   rÛ   rÜ   ræ   rê   rï   rõ   r÷   rø   rù   rû   rþ   r  r  r	  r
  r  r  r  ÚThreadr  r/  r9  r<  r=  rC  rE  rL  rM  rP  rS  ra  rb  re  rf  rg  rh  rk  rn  r@  rD  rp  rw  r   r   r   r   Ú<module>   s  '


þþû

þ



þ
þ




1	ÿû
K$




!


%
	








