U
    -e                    @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
m	Z	mZmZmZ d dlZd dlmZ d dlZd dlmZ d dlZd dlmZmZ d dlmZmZmZ d dl m!  mZ" d dl#m$Z$ d dl%Z&zd dl%m'Z( W n e)k
r   Y nX z(d dl*Z+d dl,mZ- dd	l.m/Z/m0Z0 W n e)k
r6   Y nX ej1j*Z2dddZ3dddZ4dddZ5dddZ6d	ddZ7G dd dZ8G dd dZ9ej1:dddddd d!d"d#gd$d% Z;ej1:dddddd d!d"d#gd&d' Z<G d(d) d)Z=G d*d+ d+Z>G d,d- d-Z?G d.d/ d/Z@G d0d1 d1ZAG d2d3 d3ZBd4d5 ZCd6d7 ZDG d8d9 d9ZEd:d; ZFd<d= ZGej1:d>d?gd@gfdAdB ZHdCdD ZIdEdF ZJdGdH ZKeLe"Me"jNdIdJ ZOdKdL ZPdMdN ZQdOdP ZRdQdR ZSdSdT ZTdUdV ZUdWdX ZVdYdZ ZWd[d\ ZXd]d^ ZYd_d` ZZdadb Z[dcdd Z\dedf Z]dgdh Z^didj Z_dkdl Z`dmdn Zadodp Zbej1:dqe&c drfe&d dsfe&e dtfe&f dufe&g dvfe&h dwfe&i dxfe&j dyfe&k dzfe&l d{fe&m d|fe&n d}fe&o d~fe&p dfe&q dfe&r dfe&jrdddfe&s dfe&te&te&f dfe&udddfe&vddfe&vdddfe&wddfe&xddfgdd Zydd Zzdd Z{dd Z|dd Z}dd Z~dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd
ddZdd Zdd Zdd Zdd Zdd Zdd ZG dd de&jZdddZdd ZddĄ ZddƄ ZG ddȄ de&jZddʄ Zdd̄ Zdd΄ ZddЄ Zej1:ddddggddՄ Zddׄ Zej1j*ddل Zddۄ Zdd݄ Zdd߄ Zdd Zdd Zej1j'ej1ddd Zdd Zej1:ddddgej1:dddgej1:de	ddde	dddgdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zdd Zdd ZdS (      N)OrderedDict)datedatetimetime	timedeltatimezone)get_logical_type_pandas_api)invoke_scriptrandom_asciirands)Version)parquet   )dataframe_with_arraysdataframe_with_listsd   c                 C   sN  t tj| tjdtj| tjdtj| tjdtj| tjdtj| tjdtj| tjdtj| tj	dtj| tj
dtj| tjdtj| tjdtj| dktjd| ddtjd| ddtjd| ddtjd| ddtjd| ddtjd| d	dtjd| d
dtjd| dddd t| D d gdd t| d D  d g dg|  dS )Ndtyper   2016-01-01T00:00:00.001datetime64[s]datetime64[ms]datetime64[us]datetime64[ns]timedelta64[s]timedelta64[ms]timedelta64[us]timedelta64[ns]c                 S   s   g | ]}t |qS  str.0xr   r   Z/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/test_pandas.py
<listcomp>R   s     z%_alltypes_example.<locals>.<listcomp>c                 S   s   g | ]}t |qS r   r   r!   r   r   r$   r%   S   s         )uint8uint16uint32uint64int8int16int32int64float32float64boolzdatetime[s]zdatetime[ms]zdatetime[us]zdatetime[ns]r   r   r   r   r    Zstr_with_nullsZ	empty_str)pd	DataFramenparanger(   r)   r*   r+   r-   r.   r/   r0   r1   randomrandnrangesizer   r   r$   _alltypes_example9   s@     r<   FTc              	   C   s   |r
t jnt j}|j| |||r"dndd}	|	j|d}
|rJ|	j|sJt|d krV| }t	 . t
ddt tj|
|||rdndd	 W 5 Q R X d S )
Nr&   r   )schemapreserve_indexZnthreadsuse_threadsignoreelementwise comparison failedequivF)check_dtypeZcheck_index_type)paRecordBatchTablefrom_pandas	to_pandasr=   equalsAssertionErrorwarningscatch_warningsfilterwarningsDeprecationWarningtmassert_frame_equal)dfexpectedr@   expected_schemarD   r=   r>   as_batchklasstableresultr   r   r$   _check_pandas_roundtripX   s(    

  
rY   c                 C   s^   t j| d|d}|d k	r$|d kr$|}|d k	r:|j|ks:ttj| | jd}t	| | d S )NTrH   typename)
rE   arrayr[   rK   r3   SeriesrI   r]   rP   assert_series_equal)stype_expected_pa_typearrrX   r   r   r$   _check_series_roundtrips   s    re   c                 C   s   t j| d||d}| }t| }|d kr@|j| ksVtn|j||B  ksVt|d kr|d krrt| }nt| 	 }d ||	 < t
jt||dd d S )NT)rH   maskr[   FZcheck_names)rE   r^   rI   r3   isnull
null_countsumrK   r_   copyrP   r`   )valuesrS   rf   r[   rd   rX   Zvalues_nullsr   r   r$   _check_array_roundtrip   s    
rm   c                 C   s(   t j| d|d}| }t||  d S )NTrZ   rE   r^   rI   nptassert_array_equal)Znp_arrayr[   rd   rX   r   r   r$   "_check_array_from_pandas_roundtrip   s    rq   c                   @   s0  e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Zd4d5 Zd6d7 Zd8d9 Zd:d; Z d<d= Z!d>d? Z"d@dA Z#dBdC Z$dDdE Z%dFdG Z&dHdI Z'dJS )KTestConvertMetadataz9
    Conversion tests for Pandas metadata & indices.
    c                 C   s8   t ddddgi}tj|}|djdks4td S )Nr   r   r&      0)r3   r4   rE   rG   rH   fieldr]   rK   selfrR   rW   r   r   r$   test_non_string_columns   s    z+TestConvertMetadata.test_non_string_columnsc              	   C   s   t dddgdddgd}|d}tt& tj|}|	dj
d	ksRtW 5 Q R X | }t|jj
|j_
tt t||d
d W 5 Q R X d S )N      ?       @      @      @      @      @)r   r   r   1TrS   r>   )r3   r4   	set_indexpytestwarnsUserWarningrE   rG   rH   ru   r]   rK   rk   r    indexrY   )rw   rR   rW   rS   r   r   r$   "test_non_string_columns_with_index   s    
z6TestConvertMetadata.test_non_string_columns_with_indexc                 C   s   t jdddgdddgdddgdddgd}tjj|ddgd}tj|ddg }||sbttjj|ddgd}tj|ddg }||std S )	Nr   r&   rs         )r   r   r&   r   columns)r3   r4   rE   rG   rH   rJ   rK   rF   )rw   rR   rW   rS   Zrecord_batch_tableZrecord_batch_expectedr   r   r$   test_from_pandas_with_columns   s     z1TestConvertMetadata.test_from_pandas_with_columnsc                 C   s.   t ddddgi}dg|j_t|dd d S )Ndatar   r&   rs   aTr>   )r3   r4   r   namesrY   rw   rR   r   r   r$   %test_column_index_names_are_preserved   s    
z9TestConvertMetadata.test_column_index_names_are_preservedc                 C   s6   t jtjddt jdddddd}t|dd	 d S )
Nr   rs   z
2021-01-01Z50DZCET)periodsfreqtzr   Tr   )r3   r4   r5   r7   r8   
date_rangerY   r   r   r   r$   test_column_index_names_with_tz   s
    z3TestConvertMetadata.test_column_index_names_with_tzc                 C   s  d}t jdddddgit jddd|d	d
}t jdddddgit ddd
}tj|}tj|}t|jdksxt|	 }t
|| t|jt jstt|jddkst|jj|kst|	 }t
|| t|jt jstt|jddkst|jjd kstd S )Nfoor   r   r&   rs   r   r      stepr]   r   r         r   )r3   r4   
RangeIndexrE   rG   rH   lenr=   rK   rI   rP   rQ   
isinstancer   r	   Zget_rangeindex_attributer]   )rw   Z
index_namerR   df2rW   Ztable_no_index_namerX   result2r   r   r$   test_range_index_shortcut   s(    
z-TestConvertMetadata.test_range_index_shortcutc                 C   sl   t jdddddgit jddddd	d
}tjj|dd}|jdksFtd|jksTt|	 }t
|| d S )Nr   r   r&   rs   r   r   r   r   r   r   Tr   )r3   r4   r   rE   rG   rH   Znum_columnsrK   Zcolumn_namesrI   rP   rQ   )rw   rR   rW   restoredr   r   r$   $test_range_index_force_serialization   s    z8TestConvertMetadata.test_range_index_force_serializationc              	   C   sL   t jtjddddgd}t  tjdd t|dd	 W 5 Q R X d S )
Nr   r&   r   br   erroractionTr   )	r3   r4   r5   r7   r8   rL   rM   simplefilterrY   r   r   r   r$   test_rangeindex_doesnt_warn   s    
z/TestConvertMetadata.test_rangeindex_doesnt_warnc                 C   s<   t jddgddgg}t jdddg|d}t|d	d
 d S )NonetwoXYr   r   r&   r   rs   cr   Tr   r3   
MultiIndexfrom_arraysr4   rY   rw   r   rR   r   r   r$   test_multiindex_columns  s     z+TestConvertMetadata.test_multiindex_columnsc                 C   sJ   t jjddgt ddggddgd}t jdd	d
g|d}t|dd d S )Nr   r   z
2017-08-01z
2017-08-02Zlevel_1Zlevel_2r   r   r   r   r   Tr   )r3   r   r   DatetimeIndexr4   rY   r   r   r   r$   #test_multiindex_columns_with_dtypes	  s    z7TestConvertMetadata.test_multiindex_columns_with_dtypesc                 C   s   t jdgt jdgtdd}t|dd t jdgt jdgtdd}t|dd t jdgt jtdddgddd}t|dd d S )	Nr   r   r   Tr   皙?  object)r3   r4   Indexr   rY   r   r   r   r   r$   (test_multiindex_with_column_dtype_object  s     
z<TestConvertMetadata.test_multiindex_with_column_dtype_objectc                 C   s<   t jddgddgg}t jdddg|d}t|d	d
 d S )N   あ   いr   r   r   r   r   r   Tr   r   r   r   r   r$   test_multiindex_columns_unicode&  s    z3TestConvertMetadata.test_multiindex_columns_unicodec              	   C   s\   t jddgddgg}t jdddg|d}t  tjd	d
 t|dd W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   Tr   )r3   r   r   r4   rL   rM   r   rY   r   r   r   r$   test_multiindex_doesnt_warn+  s
    
z/TestConvertMetadata.test_multiindex_doesnt_warnc                 C   s    t dddg}t|dd d S )Nr   r   r   Tr   r3   r4   rY   r   r   r   r$   test_integer_index_column4  s    z-TestConvertMetadata.test_integer_index_columnc              	   C   sV  t jdddgt jjdddgddd	ggd d
gddd dgd}tt tjj	|dd}W 5 Q R X |j
j}|d \}}}}}|d dkst|d |d kst|d d kst|d dkst|d dkst|d |d kst|d \}	}
|	dkst|d |	kst|d d ks t|
d
ks.t|d |
ks@t|d |
ksRtd S )N)r   r   @)r&   r   g@)rs   r   g?r   r   r   rs   r&   r   r   r   __index_level_0__)r   r   Tr   r   r]   
field_nameNoneindex_columns)r3   r4   r   r   r   r   r   rE   rG   rH   r=   pandas_metadatarK   )rw   rR   tjscol1Zcol2Zcol3Zidx0r   Z
idx0_descrZ	foo_descrr   r   r$   test_index_metadata_field_name8  s2    z2TestConvertMetadata.test_index_metadata_field_namec                 C   s   t jdddgt jtdddd}tjj|dd	}|jj}|d
 \}|d d ksTt	|d dksdt	|d dkstt	|d }|d dkst	|d dkst	d S )Nr   r   rz   r&   r   r{   rs   r   r|   defcategoryr   r   Tr   column_indexesr]   pandas_typeZcategorical
numpy_typer,   metadataZnum_categoriesrs   orderedF)
r3   r4   r   listrE   rG   rH   r=   r   rK   rw   rR   r   r   r   mdr   r   r$   test_categorical_column_indexZ  s    
z1TestConvertMetadata.test_categorical_column_indexc                 C   s   t jdddgt jtdddd}tjj|dd	}|jj}|d
 \}|d dksTt	|d |d ksht	|d dksxt	|d dkst	|d }t
|dkst	|d dkst	d S )Nr   r   r   r   Zstringzr\   r   Tr   r   r]   r   r   r   r   unicoder   r   encodingUTF-8)r3   r4   r   r   rE   rG   rH   r=   r   rK   r   r   r   r   r$   test_string_column_indexk  s    
z,TestConvertMetadata.test_string_column_indexc                 C   s   t jdddgt jddddd}tjj|d	d
}|jj}|d \}|d d ksRt|d dksbt|d dksrt|d }|d dkstd S )Nr   r   r   
2017-01-01rs   America/New_Yorkstartr   r   r   Tr   r   r]   r   
datetimetzr   r   r   r   )	r3   r4   r   rE   rG   rH   r=   r   rK   r   r   r   r$   test_datetimetz_column_index~  s       
z0TestConvertMetadata.test_datetimetz_column_indexc                 C   s4   t dt jddddi}|d}t|dd d S )Nr   r   rs   r   r   Tr   )r3   r4   r   r   rY   r   r   r   r$   test_datetimetz_row_index  s       
z-TestConvertMetadata.test_datetimetz_row_indexc                 C   sF   t dddgdddgd}|jd|d< |d}t|dd d S )	Nr   r&   rs   r   r   r   r   Tr   )r3   r4   r   astyper   rY   r   r   r   r$   test_categorical_row_index  s    
z.TestConvertMetadata.test_categorical_row_indexc              	   C   s<   t jddgtdd}tt tj| W 5 Q R X d S )Nr   r   aar   )	r3   r4   r   r   raises
ValueErrorrE   rG   rH   r   r   r   r$   *test_duplicate_column_names_does_not_crash  s    z>TestConvertMetadata.test_duplicate_column_names_does_not_crashc              
   C   s   ddgddgg}|D ]x}t jj|dgdd}t j|gdg}t j|||g}tt |	  W 5 Q R X tt |	  W 5 Q R X qd S )Nr   r   r   Fsafer   )
rE   DictionaryArrayr   rF   rG   from_batchesr   r   
IndexErrorrI   )rw   indicesZindsrd   batchrW   r   r   r$   #test_dictionary_indices_boundscheck  s    z7TestConvertMetadata.test_dictionary_indices_boundscheckc                 C   s&   t jddgidgd}t|dd d S )Nr   r   u   うr   Tr   r   r   r   r   r$   *test_unicode_with_unicode_column_and_index  s    z>TestConvertMetadata.test_unicode_with_unicode_column_and_indexc              
   C   s   t ddgddgd}ddgddgdd	gfD ]r}t j|td
|_tt tj	
| W 5 Q R X | }|jjt|_tt t||dd W 5 Q R X q.d S )Nr   r&   rs   r   r   r      a2      ?r   Tr   )r3   r4   r   r   r   r   r   r   rE   rG   rH   rk   rl   r   r    rY   )rw   rR   colsrS   r   r   r$   test_mixed_column_names  s    z+TestConvertMetadata.test_mixed_column_namesc                 C   s   t dt tj  kr"t dk r0n n
td dg}d }||i}t|}tjj	|dd}|
 }|jd |jd kst|jjd |jjd kst|jd |kstd S )	N2.0.02.1.0Regression in pandas 2.0.0r   r   Tr   r   )r   r3   __version__r   skipencoder4   rE   rG   rH   rI   rl   rK   r   r   )rw   Zcolumn_datakeyr   rR   r   r   r   r   r$   test_binary_column_name  s    &

z+TestConvertMetadata.test_binary_column_namec                 C   sP   d}t t|}tjjdddg|gddgd}tjd|i|d}t|d	d
 d S )Nrs   r   barZfoobarZsome_numbersr   numbersr   Tr   )r   r9   r3   r   r   r4   rY   )rw   Znum_rowsr	  r   rR   r   r   r$    test_multiindex_duplicate_values  s    z4TestConvertMetadata.test_multiindex_duplicate_valuesc                 C   sb   t dddgi}tj|}|jj}d|ks2t|d d }|d dksNt|d	 d
ks^td S )Nr   s
   some_bytesZsome_unicodemixedr   r   r   bytesr   r   r3   r4   rE   rG   rH   r=   r   rK   )rw   rR   rW   r   data_columnr   r   r$   test_metadata_with_mixed_types  s    z2TestConvertMetadata.test_metadata_with_mixed_typesc                 C   sb   t jdddgdddgddd	d
gd}tj|}|jdd}||j  }t	
|| d S )Nr   r&   rs   r   r  bazr   r   r   threer   T)Zignore_metadata)r3   r4   rE   rG   rH   rI   castr=   remove_metadatarP   rQ   )rw   rR   rW   rX   rS   r   r   r$   test_ignore_metadata  s    z(TestConvertMetadata.test_ignore_metadatac                 C   s   t ddgdddgdgd gi}ttjdtt dg}tjj||d	}|jj	}d
|ksft
|d d }|d dkst
|d dkst
d S )Nr   r   r&   rs   r   r   r   r[   r=   r  r   r   r   zlist[int64]r   r   )r3   r4   rE   r=   ru   list_r/   rG   rH   r   rK   )rw   rR   r=   rW   r   r  r   r   r$   test_list_metadata  s    " z&TestConvertMetadata.test_list_metadatac                 C   sN   t dddddddgi}tj|}|jj}|d d d	 d
ksJtd S )Ndictsr   r&   r   rs   r   r   r   r   r   r  )rw   rR   rW   r   r   r   r$   test_struct_metadata  s    z(TestConvertMetadata.test_struct_metadatac                 C   s   t dtdtd gi}tj|}|jj}d|ks@t	|d d }|d dks\t	|d	 d
kslt	|d dddkst	d S )Ndecimals394092382910493.12341234678z314292388910493.12343437128r  r   r   r   decimalr   r   r         )	precisionscale)
r3   r4   r  DecimalrE   rG   rH   r=   r   rK   )rw   rS   rW   r   r  r   r   r$   test_decimal_metadata  s    
z)TestConvertMetadata.test_decimal_metadatac                 C   s   t jdddgddt jdddd	fD ]}t jd
ddgdddgd|d}tj|}|d
}| }|dg }t	|j
t jrd |j
_t|| |d
}| }t||dg jdd q$d S )Nr   r   r   r   r\   r   rs   Europe/Brusselsr   r   r   r&   皙?皙?333333?r   r   TZdrop)r3   r   r   r4   rE   rG   rH   Zremove_columnrI   r   r   r   r   rP   rQ   reset_index)rw   r   rR   rW   Ztable_subsetrX   rS   Ztable_subset2r   r   r$   !test_table_column_subset_metadata  s$    


z5TestConvertMetadata.test_table_column_subset_metadatac                 C   sv   t ttdttdtdd}tj|	ddg}|
ddg}| }|ddg 	d}t|| d S )Nr   )firstsecondvaluer,  r-  r.  )r3   r4   r   r9   r5   r6   rE   rG   rH   r   selectrI   rP   rQ   )rw   rR   rW   ZsubsetrX   rS   r   r   r$   'test_to_pandas_column_subset_multiindex2  s    

z;TestConvertMetadata.test_to_pandas_column_subset_multiindexc              	   C   s   dgddgd g}g g g g}t dtj|tt dfdtj|tt dfg}tjt| t|	 }tj
|g}| }tj
|}|jj}| }	tt d|fd|fg}
t|	|
 |d ddd dd	d
ddd ddd
gkstd S )Ntestr   r   c1r  c2r   r   zlist[unicode])r]   r   r   r   r   zlist[empty])r   rE   r^   r  stringrF   r   r   rl   keysrG   r   rI   rH   r=   r   r3   r4   rP   rQ   rK   )rw   r2  r3  arraysrbtblrR   Ztbl2Zmd2r   rS   r   r   r$   test_empty_list_metadata@  s>    


z,TestConvertMetadata.test_empty_list_metadatac                 C   s@   t dddgdddgd}tj|}|jjd d k	s<td S )Nr   r&   rs   r   pandas_versionr  rv   r   r   r$   test_metadata_pandas_versionl  s    z0TestConvertMetadata.test_metadata_pandas_versionc           
      C   s   t dt jdddi}tj|}|d tjddd}|dt	d|j
|}tdtjdddfg}tjj||d	}| }|d jd|d< ||fD ](}|jjd k	st| }	t|	| qd S )
Nr   z
2020-01-01rs   r   nsUTCr   r   r  )r3   r4   r   rE   rG   rH   r  	timestampZ
set_columnru   r[   r=   rk   dttz_localizer   rK   rI   rP   rQ   )
rw   rR   rW   Znew_colZ
new_table1r=   Z
new_table2rS   Z	new_tablerX   r   r   r$   test_mismatch_metadata_schemaq  s       z1TestConvertMetadata.test_mismatch_metadata_schemaN)(__name__
__module____qualname____doc__rx   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r
  r  r  r  r  r#  r+  r0  r9  r;  rC  r   r   r   r$   rr      sJ   
		"
	
,rr   c                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.S )/TestConvertPrimitiveTypesz>
    Conversion tests for primitive (e.g. numeric) types.
    c           
      C   s   i }g }dt  fdt  fdt  fg}d}|D ]4\}}tj|}||||< |t 	|| q2t
|}t |}	t||	d d S )Nf2f4f8r   rT   )rE   float16r0   r1   r5   r7   r8   r   appendru   r3   r4   r=   rY   )
rw   r   fieldsdtypes
num_valuesZnumpy_dtypearrow_dtyperl   rR   r=   r   r   r$   test_float_no_nulls  s    




z-TestConvertPrimitiveTypes.test_float_no_nullsc                 C   s  d}t jjdd|ddk }dt fdt fdt fg}dddg}g }g }g }|D ]Z\}}	t j||}
tj	|
d	|d
}|
| |
t||	 t j|
|< |
|
 qVtjtt|||d}tj||}|jt|st| }t|| d S )Nr   r   
   r:   rs   rI  rJ  rK  T)rH   rf   r   )r5   r7   randintrE   rM  r0   r1   r8   r   r^   rN  ru   nanr3   r4   dictziprG   r   r=   rJ   rK   rI   rP   rQ   )rw   rQ  	null_maskrP  r   expected_colsr6  rO  r]   rR  rl   rd   ex_framerW   rX   r   r   r$   test_float_nulls  s0    





z*TestConvertPrimitiveTypes.test_float_nullsc                 C   sr   t dddtjgi}ttjdt ddg}tjj	||dd}|d 
 d	d
d gks^tt||  d S )Nr   ry   rz   TnullableFr=   r   r   r   r&   )r3   r4   r5   NaNrE   r=   ru   r-   rG   rH   	to_pylistrK   rP   rQ   rI   rw   rR   r=   rW   r   r   r$   test_float_nulls_to_ints  s
    z2TestConvertPrimitiveTypes.test_float_nulls_to_intsc                 C   s>   t dddd dg}t dddd dg}t||t d d S )N        ry   rz   g      FTrS   r[   )r3   r_   rm   rE   bool_rw   ra   rS   r   r   r$   test_float_nulls_to_boolean  s    z5TestConvertPrimitiveTypes.test_float_nulls_to_booleanc                 C   sF   t dtjg}tj|dd}|jdks,tt|d 	 sBtd S )Nrd  FrH   r   r   )
r3   r_   r5   rV  rE   r^   ri   rK   isnanas_py)rw   ra   rd   r   r   r$   'test_series_from_pandas_false_respected  s    zATestConvertPrimitiveTypes.test_series_from_pandas_false_respectedc                 C   s  t  }g }dt fdt fdt fdt fdt fdt fdt fdt	 fd	t fd
t	 fg
}d}|D ]h\}}t
|}t
jjt|jt
t
jjt|jt
t
jj|d}||||< |t|| qzt|}	t|}
t|	|
d d S )Ni1i2i4i8u1u2u4u8longlong	ulonglongr   r:   rL  )r   rE   r,   r-   r.   r/   r(   r)   r*   r+   r5   Ziinfor7   rU  maxminint_r   rN  ru   r3   r4   r=   rY   )rw   r   rO  numpy_dtypesrQ  r   rR  inforl   rR   r=   r   r   r$   test_integer_no_nulls  s2    
 

 

 

 

 



z/TestConvertPrimitiveTypes.test_integer_no_nullsc                 C   s   t  }ddddddddd	d
ddddddddg}|D ]}tjd|d||< q2t|}t| | D ]"}t|}|	 |
 ksdtqdd S )Nrm  rn  ro  rp  rq  rr  rs  rt  byteZubyteshortZushortZintcZuintcry  Zuintru  rv     r   )r   r5   r6   r3   r4   rY   rl   rE   r^   ra  tolistrK   )rw   r   rz  r   rR   np_arrrd   r   r   r$   test_all_integer_types  s(            

z0TestConvertPrimitiveTypes.test_all_integer_typesc                 C   s   ddddddddg}|D ]}d	D ]|}t jd
ddg|| d}||d d d fD ]L}|jjrzt|}| | kstqNt	t
 t|}W 5 Q R X qNq qd S )Nrm  rn  ro  rp  rq  rr  rs  rt  z=<>r   r&   *   r   )r5   r^   r   ZisnativerE   ra  r  rK   r   r   NotImplementedError)rw   
int_dtypesrA  orderr   r  rd   r   r   r$   test_integer_byteorder  s    
z0TestConvertPrimitiveTypes.test_integer_byteorderc                 C   s   ddddddddg}d	}t jjd
d|ddk }g }g }|D ]L}t jjd
d	|d}tj||d}|| |d}	t j|	|< ||	 q:tj	t
t|||d}
tj||}| }t||
 d S )Nrm  rn  ro  rp  rq  rr  rs  rt  r   r   rT  r:   rs   rf   rK  r   )r5   r7   rU  rE   r^   rN  r   rV  r3   r4   rW  rX  rG   r   rI   rP   rQ   )rw   r  rQ  rY  rZ  r6  r]   rl   rd   rS   r[  rW   rX   r   r   r$   test_integer_with_nulls  s$    


z1TestConvertPrimitiveTypes.test_integer_with_nullsc                 C   sF   t jddd}t }tj||d}t|d}||sBtd S )NrT  r/   r   r  r,   )r5   r6   rE   r,   r^   r   rJ   rK   )rw   rd   Ztarget_typerX   rS   r   r   r$    test_array_from_pandas_type_cast,  s
    z:TestConvertPrimitiveTypes.test_array_from_pandas_type_castc                 C   sV   d}t jd tdt j|dki}tdt }t	|g}t
||d d S )Nr   r   boolsrL  )r5   r7   seedr3   r4   r8   rE   ru   rf  r=   rY   )rw   rQ  rR   ru   r=   r   r   r$   test_boolean_no_nulls5  s    z/TestConvertPrimitiveTypes.test_boolean_no_nullsc                 C   s   d}t jd t jjdd|ddk }t jjdd|ddk }tj||d}|t}d ||< tdt	 }t
|g}td|i}tj|gdg}	|	j
|st|	 }
t|
| d S )	Nr   r   rT  r:   rs   r   r  r  )r5   r7   r  rU  rE   r^   r   r   ru   rf  r=   r3   r4   rG   r   rJ   rK   rI   rP   rQ   )rw   rQ  rf   rl   rd   rS   ru   r=   r[  rW   rX   r   r   r$   test_boolean_nulls?  s    
z,TestConvertPrimitiveTypes.test_boolean_nullsc                 C   sF   t dddddgd }t dddddgd }t||t d d S )NTFr&   r   r   re  )r3   r_   rm   rE   r/   rg  r   r   r$   test_boolean_to_intV  s    z-TestConvertPrimitiveTypes.test_boolean_to_intc              	   C   sj   t jdddddgd td}t dddddgd }d}tjtj|d t||t d	 W 5 Q R X d S )
NTFr&   r   r   r   zExpected integer, got boolmatchre  )	r3   r_   r   r   r   rE   ArrowTypeErrorrm   r/   )rw   ra   rS   Zexpected_msgr   r   r$   test_boolean_objects_to_int\  s
    z5TestConvertPrimitiveTypes.test_boolean_objects_to_intc                 C   sF   t dddd dgd }t dddd dgd }t||t d d S )NTFr&   ry   rd  re  )r3   r_   rm   rE   r1   rg  r   r   r$   test_boolean_nulls_to_floatd  s    z5TestConvertPrimitiveTypes.test_boolean_nulls_to_floatc                 C   s(   t jtjdddddgd}t| d S )N)rs   r&   r2   r   r   r   r   )r3   r4   r5   onesrY   r   r   r   r$   test_boolean_multiple_columnsj  s    z7TestConvertPrimitiveTypes.test_boolean_multiple_columnsc                 C   sn   t jd dt dgd td}td|i}tdt|i}tdt }t	|g}t
|||d d S )Nr   g      @r   r   floatsrS   rT   )r5   r^   r1   r   r3   r4   
to_numericrE   ru   r=   rY   rw   rd   rR   rS   ru   r=   r   r   r$   test_float_object_nullso  s    z1TestConvertPrimitiveTypes.test_float_object_nullsc              	   C   s   t tjddtjg}t t t t t	 t
 t t g}|D ]}tj||d}tjd ddd g|d}||stt d|i}ttd|g}tjj||dd}|d	 d	|sNtqNd S )
Nry   rz   r  r   r&   Z	has_nullsFr=   r>   r   )r3   r_   r5   rV  rE   r,   r-   r.   r/   r(   r)   r*   r+   r^   rJ   rK   r4   r=   ru   rG   rH   chunk)rw   ra   typestyrX   rS   rR   r=   r   r   r$   test_float_with_null_as_integerx  s"       
z9TestConvertPrimitiveTypes.test_float_with_null_as_integerc                 C   sn   t jd dt dgd td}td|i}tdt|i}tdt }t	|g}t
|||d d S )Nr   rs   r   r   intsr  )r5   r^   r/   r   r3   r4   r  rE   ru   r=   rY   r  r   r   r$   test_int_object_nulls  s    z/TestConvertPrimitiveTypes.test_int_object_nullsc                 C   sR   t jdd dgd td}td|i}tdt }t|g}t	||d d S )NFTr   r   r  rL  )
r5   r^   r   r3   r4   rE   ru   rf  r=   rY   )rw   rd   rR   ru   r=   r   r   r$   test_boolean_object_nulls  s
    z3TestConvertPrimitiveTypes.test_boolean_object_nullsc                    s8   t jd gtd  fdd}|t  |t  d S )Nr   c                    s4   t j | d}|j| kst|d  d ks0td S )Nr  r   )rE   r^   r[   rK   rk  )r   a2rd   r   r$   _check_type  s    zJTestConvertPrimitiveTypes.test_all_nulls_cast_numeric.<locals>._check_type)r5   r^   r   rE   r.   r1   )rw   r  r   r  r$   test_all_nulls_cast_numeric  s    z5TestConvertPrimitiveTypes.test_all_nulls_cast_numericc                 C   s   t jdt jgt jd}tj|t d}| \}}t|t jsDt|dksPtt|t js`tt |snttj|t dd}| \}}t|t jst|dkst|d kstd S )Nr   r   r  Tr[   rH   )	r5   r^   rV  rM  rE   ra  r   rK   rj  )rw   rd   r   r#   yr   r   r$   test_half_floats_from_numpy  s    z5TestConvertPrimitiveTypes.test_half_floats_from_numpyN)rD  rE  rF  rG  rS  r\  rc  rh  rl  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r$   rH    s.   	
		rH  r   rm  rn  ro  rp  rq  rr  rs  rt  c                 C   sz   d}t jjdd|ddk }t jjdd|| d}tj||d}| rX|d}d ||< n|}|jd	d
}t j	|| d S Nr   r   rT  r:   rs   r;   r   r  OTinteger_object_nulls)
r5   r7   rU  rE   r^   anyr   rI   testingZassert_equal)r   rQ  rY  rl   r^   rS   rX   r   r   r$   &test_array_integer_object_nulls_option  s    

r  c                 C   s   d}t jjdd|ddk }t jjdd|| d}tj||d}| rX|d}d ||< n|}t| |i}tj	
|g| g}|jd	d
}t|| d S r  )r5   r7   rU  rE   r^   r  r   r3   r4   rG   r   rI   rP   rQ   )r   rQ  rY  rl   r^   rS   rW   rX   r   r   r$   &test_table_integer_object_nulls_option  s    

r  c                   @   s  e Zd ZdZdd Zdd Zejdddd	d
gdd Z	dd Z
dd Zee ejB ejdddd Zdd Zdd Zdd Zdd Zejddeddddddggd d! Zejd"d#d$gd%d& Zejd'd#d$gd(d) Zejd*e e e de de d	e d
e dd+e dd+e d	d+e d
d+g
d,d- Z!ejd*e e e de de d	e d
e dd+e dd+e d	d+e d
d+g
d.d/ Z"d0d1 Z#d2d3 Z$d4d5 Z%d6d7 Z&d8d9 Z'd:d; Z(d<d= Z)d>d? Z*d@dA Z+dBdC Z,dDdE Z-ejdFe e gdGdH Z.dIdJ Z/ejj0e1dKe1ej2  kope1dLk n  dMdNdOdP Z3ejdddd	d
gdQdR Z4ejdddd	d
gdSdT Z5dUdV Z6dS )WTestConvertDateTimeLikeTypeszQ
    Conversion tests for datetime- and timestamp-like types (date64, etc.).
    c                 C   sL   t dtjdddgddi}tdtd}t|g}t||d d S )	N
datetime642007-07-13T01:23:34.1234567892006-01-13T12:34:56.4325397842010-08-13T05:46:57.437699912r   r   r=  rL  	r3   r4   r5   r^   rE   ru   r@  r=   rY   rw   rR   ru   r=   r   r   r$   #test_timestamps_notimezone_no_nulls  s      z@TestConvertDateTimeLikeTypes.test_timestamps_notimezone_no_nullsc                 C   sL   t dtjdd dgddi}tdtd}t|g}t||d d S )Nr  r  r  r   r   r=  rL  r  r  r   r   r$    test_timestamps_notimezone_nulls  s      z=TestConvertDateTimeLikeTypes.test_timestamps_notimezone_nullsunitra   msusr=  c                 C   s   t tjt dk r$|dkr$td tdtjdddgd| d	d
i}|d j	d|d< t
| t|d  tdtjdd ddgd| d	d
i}|d j	d|d< t
| d S )Nr   r=  z0pandas < 2.0 only supports nanosecond datetime64r  2007-07-13T01:23:34.1232006-01-13T12:34:56.4322010-08-13T05:46:57.437zdatetime64[]r   
US/Easternr  r  r  )r   r3   r  r   r  r4   r5   r^   rA  rB  rY   re   )rw   r  rR   r   r   r$   test_timestamps_with_timezone  s8    
 
 
z:TestConvertDateTimeLikeTypes.test_timestamps_with_timezonec                 C   s   dd t dD }tdtj|tdi}tj|}t|d 	dtj
sPt| }tdtj|ddi}t|| d S )Nc                 S   s   g | ]}t  t|d  qS ))days)r   todayr   r!   r   r   r$   r%   #  s     zETestConvertDateTimeLikeTypes.test_python_datetime.<locals>.<listcomp>rT  r   r   r   r   )r9   r3   r4   r_   r   rE   rG   rH   r   r  TimestampArrayrK   rI   rP   rQ   )rw   
date_arrayrR   rW   rX   expected_dfr   r   r$   test_python_datetime!  s      z1TestConvertDateTimeLikeTypes.test_python_datetimec              
   C   sZ   t d}|j|d|dfD ]2}tdddddd|dg}td	|i}t| q"d S )
Npytzr  r   r   r     -   tzinfor   )	r   importorskiputcr   FixedOffsetr   r3   r4   rY   )rw   r  r   rl   rR   r   r   r$   %test_python_datetime_with_pytz_tzinfo4  s
    
zBTestConvertDateTimeLikeTypes.test_python_datetime_with_pytz_tzinfoN)deadlinec              	   C   sL   t |dkrtd tdddddd|dg}td	|i}t|d
d d S )N)zbuild/etc/localtimeFactoryz Localtime timezone not supportedr   r   r  r  r  r  r   FrD   )r    r   r  r   r3   r4   rY   )rw   r   rl   rR   r   r   r$   'test_python_datetime_with_pytz_timezone<  s
    
zDTestConvertDateTimeLikeTypes.test_python_datetime_with_pytz_timezonec           
   	   C   s   t d}ddlm} tdddddd|jd	g}tjd
|i|d}t|dd d}|t|d}|	|d }tdddddd|d	g}tdddddd|d	g}tjd
|i|d}tjd
|i|d}	t||	dd d S )Nr  r   )r   r   r   r  r  r  r  r   r   Tr   )hours<   r   )
r   r  r   r   r  r3   r4   rY   r   r  )
rw   r  r   rl   rR   r  Ztz_timezoneZtz_pytzZ
values_expZdf_expr   r   r$   )test_python_datetime_with_timezone_tzinfoE  s    
zFTestConvertDateTimeLikeTypes.test_python_datetime_with_timezone_tzinfoc                 C   s   G dd dt }|ddddddg}tdtj|tdi}tj|}t|d 	dtj
sbt| }tdtj|ddi}t|d |d< t|| d S )	Nc                   @   s   e Zd ZdZdS )zNTestConvertDateTimeLikeTypes.test_python_datetime_subclass.<locals>.MyDatetimerd  N)rD  rE  rF  Z
nanosecondr   r   r   r$   
MyDatetimeZ  s   r    r   r   r   r   r   )r   r3   r4   r_   r   rE   rG   rH   r   r  r  rK   rI   to_datetimerP   rQ   )rw   r  r  rR   rW   rX   r  r   r   r$   test_python_datetime_subclassX  s    z:TestConvertDateTimeLikeTypes.test_python_datetime_subclassc                 C   s   G dd dt }|dddg}tdtj|tdi}tj|}t|d 	dtj
s\t| }tdtjt dddgtdi}t|| d S )Nc                   @   s   e Zd ZdS )zFTestConvertDateTimeLikeTypes.test_python_date_subclass.<locals>.MyDateNrD  rE  rF  r   r   r   r$   MyDates  s   r  r  r   r   r   r   )r   r3   r4   r_   r   rE   rG   rH   r   r  ZDate32ArrayrK   rI   r5   r^   rP   rQ   )rw   r  r  rR   rW   rX   r  r   r   r$   test_python_date_subclassq  s    z6TestConvertDateTimeLikeTypes.test_python_date_subclassc                 C   sV   t tdddd g}t |g}| }t jj|t  d}||	dsRt
d S )N  rT  r  r  date32)rE   r^   r   chunked_arrayrI   ArrayrH   r  rJ   r  rK   )rw   rd   r   ra   arr2r   r   r$   test_datetime64_to_date32  s
    z6TestConvertDateTimeLikeTypes.test_datetime64_to_date32rf   TFc                 C   s   t ddddddg}tjj|t |d}ttdd	d
tdd	dtdd	dtdd	d
tdd	dtdd	dg}tj||t d}|	|st
d S )Nz2018-05-10T00:00:00z2018-05-11T00:00:00z2018-05-12T00:00:00z2018-05-10T10:24:01z2018-05-11T10:24:01z2018-05-12T10:24:01r[   rf   r   r   rT  r  r  rf   r[   )r3   r  rE   r  rH   date64r5   r^   r   rJ   rK   )rw   rf   ra   rd   r   rS   r   r   r$   test_pandas_datetime_to_date64  s&    





z;TestConvertDateTimeLikeTypes.test_pandas_datetime_to_date64zcoerce_to_ns,expected_dtype)Fr   )Tr   c           
      C   s   t dddd t dddt dddg}tjdd dd	gd
d}ttjtdk rPd}tjdd dd	g|d}t|t|gg}|D ]`}|j|d}|	t
}	|j|	jkstt||	 |jd|d}|j|jkstt|| qd S )Nr  r       r&   r  
2000-01-01z
1970-01-01z
2040-02-26datetime64[D]r   r   r   coerce_temporal_nanosecondsFdate_as_objectr  )r   r5   r^   r   r3   r  rE   r  rI   r   r   r   rK   ro   rp   )
rw   coerce_to_nsZexpected_dtyper   Zexpected_daysrS   objectsobjrX   Zexpected_objr   r   r$   test_array_types_date_as_object  s<    






z<TestConvertDateTimeLikeTypes.test_array_types_date_as_objectzcoerce_to_ns,expected_typec              
   C   s~   t dtdddd tdddtdddgi}tjj|dd	}|jd|d
}| }tj|	||dd tj||dd d S )Nr   r  r   r  r  r&   r  Fr   r  Tr  )
r3   r4   r   rE   rG   rH   rI   rP   rQ   r   )rw   r  expected_typerR   rW   Zdf_datetimeZ	df_objectr   r   r$   !test_table_convert_date_as_object  s"     


z>TestConvertDateTimeLikeTypes.test_table_convert_date_as_object
arrow_typer>  c                 C   s   t dddtdddg}t|}t||}|jddd}d }t|dr\|j	d k	r\d}t
d	| }t||| d S )
Nr  r     TFr  r  r   r>  r=  )r   r   r3   r_   rE   r^   r  rI   hasattrr   r@  to_pandas_dtyperP   r`   r   )rw   r  r   rS   rd   rX   expected_tzr  r   r   r$   &test_array_coerce_temporal_nanoseconds  s    
 zCTestConvertDateTimeLikeTypes.test_array_coerce_temporal_nanosecondsc           	      C   s   t dddtdddg}ttd|g}td|i}tjt|g|d}|j	ddd}d }t
|d	rz|jd k	rzd
}td| }t||| d S )Nr  r   r  r   r  TFr  r   r>  r=  )r   r   rE   r=   ru   r3   r4   rW   r^   rI   r  r   r@  r  rP   rQ   r   )	rw   r  r   r=   r  rW   Z	result_dfr   r  r   r   r$   &test_table_coerce_temporal_nanoseconds  s     zCTestConvertDateTimeLikeTypes.test_table_coerce_temporal_nanosecondsc              
   C   s   t dtdddd tdddtdddgi}tjj|dd	}tdt }tj|g|jj	d
}|j
|srt| }t|| d S )Nr   r  r   r  r  r&   r  Fr   r   )r3   r4   r   rE   rG   rH   ru   r  r=   r   rJ   rK   rI   rP   rQ   )rw   rR   rW   ru   rT   rX   r   r   r$   test_date_infer   s     


z,TestConvertDateTimeLikeTypes.test_date_inferc                 C   sx   t jtdddtdddgdd}ddg}tj|t |d}t jd tdddgdd}tj|dd	}||sttd S )
Nr  r   rs   r  r   TFr  ri  )r5   r^   r   rE   rJ   rK   )rw   rd   rf   rX   rS   r   r   r$   test_date_mask  s    z+TestConvertDateTimeLikeTypes.test_date_maskc                 C   st  t jtdddd tdddtdddgtd}t jdddd	gd
d}|dd }t ddddg}t }t }tj||d}tj||d}tj|||d}	tj|||d}
||	st	||
st	ddg}tj
||g|}t jddddgdd}tjj|d< |d}tj||d|d}|jdd}t|| | }|d}tj||d|d}t|| d S )Nr  r   rs   r   r   kC  r   lC  mC  r.   r/    \&FTr  r  r  r  z
2017-04-03z
2017-04-04z
2017-04-05r  r   r   )r  r  r   )r  r   )r5   r^   r   r   r   rE   r  r  rJ   rK   rG   r   r3   NaTr.  r4   rI   rP   rQ   )rw   rd   Zarr_i4Zarr_i8rf   Zt32Zt64Za32Za64Za32_expectedZa64_expectedZcolnamesrW   	ex_valuesZex_datetime64msZexpected_pandasZtable_pandasZtable_pandas_objectsZ
ex_objectsZexpected_pandas_objectsr   r   r$   test_date_objects_typed  sZ    





z4TestConvertDateTimeLikeTypes.test_date_objects_typedc                 C   s   t tdd }tjtdddtj|gtd}tjtdddd d gtd}tj|dd}tj|dd}||spt	|j
dks~t	tjtjgddjt kst	tj|gddjt kst	d S )NNAr  r   r   Tri  r&   )getattrr3   r5   r^   r   r
  r   rE   rJ   rK   ri   r[   null)rw   Zpd_NArl   Zvalues_with_nonerX   rS   r   r   r$   test_pandas_null_valuesJ  s     z4TestConvertDateTimeLikeTypes.test_pandas_null_valuesc                 C   s   t  }t  }tjdddgdd}|dd }t j||d}t j||d}td	d
d}|d  |ksnt|d  |kstd S )Nr  r  r  r.   r   r/   r	  r  r  r   rs   r   )	rE   r  r  r5   r^   r   r   rk  rK   )rw   t1t2rd   r  a1r  rS   r   r   r$   test_dates_from_integersY  s    z5TestConvertDateTimeLikeTypes.test_dates_from_integersc                 C   s  t ddddt ddddg}td}tj|d g td	}t|}|j|ksRt|d
  |d
 ksjt|d  |d kst|d  d kstt	
d|i}tj|}|d
 |sttjdd |D dd	}tj|tdd}|d
  |d
 ksttj|d tdd}	|	d
  |d
 ks:ttj|d dtdd}
|
d
  |d
 jddkszttj|d dtdd}|d
  |d
 jd
dkstd S )Nr   r&   rs   L  r   r   r   r  r   r   timesc                 S   s   g | ]}t |qS r   _pytime_to_microsr"   vr   r   r$   r%   {  s     zHTestConvertDateTimeLikeTypes.test_pytime_from_pandas.<locals>.<listcomp>r/   r    r=  ro  r  microsecond@B ra   )r   rE   time64r5   r^   r   r[   rK   rk  r3   r4   rF   rH   rJ   r   time32replace)rw   pytimesr  ZaobjsparrrR   r   rd   r  r  a3a4r   r   r$   test_pytime_from_pandasg  s8    

"z4TestConvertDateTimeLikeTypes.test_pytime_from_pandasc                 C   s  t ddddt ddddt dddg}t|d d d g }td	d
 |d d D d g }tdd
 |d d D d g }tjdd
 |D dd}tjdd
 |D dd}tjdddgtd}tj||tdd}tj|d |tdd}tj|d d|tdd}	tj|d d|tdd}
ddddg}tj	|||	|
g|}||f||f|	|f|
|ffD ]&\}}|
 }|j|k sttqt|
 }tj||||d|d}t|| d S ) Nr   r&   rs   r  r   r   r   r   c                 S   s   g | ]}|j d dqS )r  r  r!  r!   r   r   r$   r%     s   zJTestConvertDateTimeLikeTypes.test_arrow_time_to_pandas.<locals>.<listcomp>c                 S   s   g | ]}|j d dqS )r   r  r'  r!   r   r   r$   r%     s   c                 S   s   g | ]}t |qS r   r  r  r   r   r$   r%     s     r/   r   c                 S   s   g | ]}t |qS r   r  r  r   r   r$   r%     s     FTr  r  r  r=  ro  r  r  ra   
time64[us]
time64[ns]
time32[ms]	time32[s])r(  r)  r*  r+  r   )r   r5   r^   r2   rE   r  r   r   rF   r   rI   rl   allrK   r3   r4   rP   rQ   )rw   r"  rS   Zexpected_msZ
expected_srd   rY  r  r  r$  r%  r   r   Zexpected_valuesZresult_pandasrR   r  r   r   r$   test_arrow_time_to_pandas  sd    




z6TestConvertDateTimeLikeTypes.test_arrow_time_to_pandasc                 C   s|   t jdd ddgdd}t| t jdd ddgd	d}t| t jd
d ddgdd}t| t jdd ddgdd}t| d S )Nr  r  r  r   r   z2007-07-13T01:23:34.123456z2006-01-13T12:34:56.432539z2010-08-13T05:46:57.437699r   r  r  r  r   z2007-07-13T01:23:34z2006-01-13T12:34:56z2010-08-13T05:46:57r   r5   r^   rq   )rw   Zdatetime64_nsZdatetime64_usZdatetime64_msZdatetime64_sr   r   r$   test_numpy_datetime64_columns  sH    z:TestConvertDateTimeLikeTypes.test_numpy_datetime64_columnsc                 C   s   t tjt dkrtd tdddgtd}ttj	dddgdd}|
 }t|| t|g}|
 }t|| d S )Nr   z0pandas >= 2.0 supports non-nanosecond datetime64r   r&   rs   r  )r  )r   r3   r  r   r  rE   r^   r@  r_   r  rI   rP   r`   r  )rw   rd   rS   ra   r   r   r$   &test_timestamp_to_pandas_coerces_to_ns  s    
zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_coerces_to_nsc                 C   s   dD ]}dD ]}t tdddgt j||d}t d|i}d}tjt|d |jdd	 W 5 Q R X tjt|d |jdd	 W 5 Q R X tjt|d |	djdd	 W 5 Q R X |jd
dd |jd
dd |	djd
dd qqd S )Nra   r  r  )Nr   r   r?  r   z'would result in out of bounds timestampr  Tr  F)r   r  )
rE   r^   r   r@  rW   r   r   r   rI   column)rw   r  r   rd   rW   msgr   r   r$   &test_timestamp_to_pandas_out_of_bounds  s"     
 zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_out_of_boundsc                 C   sN   t dt jg t ddi}| }tdtjg ddi}t	|| d S )Nr   r  r  r   r   )
rE   rW   r  r@  rI   r3   r4   r_   rP   rQ   )rw   rW   rX   rS   r   r   r$   &test_timestamp_to_pandas_empty_chunked  s    zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_empty_chunkedr   c                 C   s&   t jdd ddgdd}t||d d S )N
2007-07-13z
2006-01-15z
2010-08-19r  r   r  r.  )rw   r   Zdatetime64_dr   r   r$   test_numpy_datetime64_day_unit  s    z;TestConvertDateTimeLikeTypes.test_numpy_datetime64_day_unitc                 C   sz   t dddg}ttdddtdddtdddg}tjj||d}td tdddd g}tj||svt	d S )NTFi  r   i  i  r  )
r5   r^   r3   r_   r   rE   r  rH   rJ   rK   )rw   mr   rX   rS   r   r   r$   %test_array_from_pandas_date_with_mask  s    


zBTestConvertDateTimeLikeTypes.test_array_from_pandas_date_with_maskz1.16.0z1.16.1z#Until numpy/numpy#12745 is resolved)reasonc                 C   s*   t dt dt jgi}t|dd d S )Nr   z2012-11-11 00:00:00+01:00Fr  )r3   r4   	Timestampr
  rY   r   r   r   r$   test_fixed_offset_timezone#  s    
z7TestConvertDateTimeLikeTypes.test_fixed_offset_timezonec                 C   sj   t tjt dk rd}tdtjdddgd| dd	i}tdt|}t	|g}t
||d
 d S )Nr   r=  timedelta64r   l     qa    @B1timedelta64[r  r   rL  r   r3   r  r4   r5   r^   rE   ru   durationr=   rY   rw   r  rR   ru   r=   r   r   r$   test_timedeltas_no_nulls3  s     
z5TestConvertDateTimeLikeTypes.test_timedeltas_no_nullsc                 C   sj   t tjt dk rd}tdtjdd dgd| ddi}tdt|}t	|g}t
||d	 d S )
Nr   r=  r=  r   r>  r?  r  r   rL  r@  rB  r   r   r$   test_timedeltas_nullsB  s     
z2TestConvertDateTimeLikeTypes.test_timedeltas_nullsc              
   C   sN   ddl m} tdd |dddddgi}tdt fg}t||d d S )	Nr   )
DateOffsetZdate_offseti  rs   iX  )r  monthsmicrosecondsnanosecondsrL  )Zpandas.tseries.offsetsrE  r3   r4   rE   r=   Zmonth_day_nano_intervalrY   )rw   rE  rR   r=   r   r   r$   test_month_day_nano_intervalQ  s     z9TestConvertDateTimeLikeTypes.test_month_day_nano_interval)7rD  rE  rF  rG  r  r  r   markparametrizer  r  r  hgivenstnonepastZ	timezonessettingsr  r  r  r  r  r5   r^   r  r  r  rE   r  r  r@  r  r  r  r  r  r  r  r&  r-  r/  r0  r4  r5  r7  r9  Zskipifr   r  r<  rC  rD  rI  r   r   r   r$   r    s   





  

 


  

 


	1%0!
	&


r  c                   @   s   e Zd Zdd Zdd Zejjdd Zejjej	ddd	gd
d Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*S )+TestConvertStringLikeTypesc                 C   st   d}dd ddt jg}td|| i}tdt }t|g}dd ddd g}td|| i}t|||d d S )Nr  r   r  u   mañanastringsr  )	r5   rV  r3   r4   rE   ru   r4  r=   rY   )rw   repeatsrl   rR   ru   r=   r  rS   r   r   r$   test_pandas_unicoded  s    z.TestConvertStringLikeTypes.test_pandas_unicodec                 C   sr   ddd t ddtjg}td|i}tj|}|d jt	 ksFt
ddd ddd g}td|i}t|| d S )Nqux   foos   barzrS  r      qux)	bytearrayr5   rV  r3   r4   rE   rG   rH   r[   binaryrK   rY   )rw   rl   rR   rW   Zvalues2rS   r   r   r$   test_bytes_to_binaryo  s    z/TestConvertStringLikeTypes.test_bytes_to_binaryc                 C   s   dd }dd }t d|gd |g dgd  i}t|d }t|tjsRt|jdks`td }tj	|}|d jdkstd S )	N   x krS     r#   r&   r   )
r3   r4   rE   r^   r   ChunkedArrayrK   
num_chunksrG   rH   )rw   v1v2rR   rd   rW   r   r   r$   test_bytes_exceed_2gbz  s     z0TestConvertStringLikeTypes.test_bytes_exceed_2gbcharr#   r\  c                 C   s   |d }|d }t d|ggd |gg dgg i}tj|d dd}t|tjsXt|jdksftt|	d	d
ks|tt|	ddkstd S )Nr]  r^  rS  r_  r\  Tri  r&   r      r   )
r3   r4   rE   r^   r   r`  rK   ra  r   r  )rw   re  rb  rc  rR   rd   r   r   r$   +test_auto_chunking_pandas_series_of_strings  s     zFTestConvertStringLikeTypes.test_auto_chunking_pandas_series_of_stringsc                 C   s   dd t dd d dg}td|i}ttdtdg}tjj||d}|jd j	|d j	ksft
|jd j|d jkst
| }t|| d S )NrW     bar   heyrS  rs   r  r   )rY  r3   r4   rE   r=   ru   rZ  rG   rH   r[   rK   r]   rI   rP   rQ   )rw   rl   rR   r=   rW   rX   r   r   r$   test_fixed_size_bytes  s    z0TestConvertStringLikeTypes.test_fixed_size_bytesc              	   C   sd   dd dd d dg}t d|i}ttdtdg}ttj tj	j
||d W 5 Q R X d S )NrW  s   bari  rS  rs   r  )r3   r4   rE   r=   ru   rZ  r   r   ArrowInvalidrG   rH   )rw   rl   rR   r=   r   r   r$   5test_fixed_size_bytes_does_not_accept_varying_lengths  s
    zPTestConvertStringLikeTypes.test_fixed_size_bytes_does_not_accept_varying_lengthsc                 C   s&   t dddd g}t|t d d S )N   123    r   rb   )r3   r_   re   rE   rZ  rw   ra   r   r   r$   test_variable_size_bytes  s    z3TestConvertStringLikeTypes.test_variable_size_bytesc                 C   sB   t tdtdtdd g}t|t d t|t d d S )Nrm  rn  r   ro  rc   )r3   r_   rY  re   rE   rZ  rp  r   r   r$   test_binary_from_bytearray  s
    z5TestConvertStringLikeTypes.test_binary_from_bytearrayc                 C   sP   t dddd g}t|t d t d|i}t|tdt fgd d S )Nrm  rn  r   ro  r   r  )r3   r_   re   rE   large_binaryr4   rY   r=   rw   ra   rR   r   r   r$   test_large_binary  s     z,TestConvertStringLikeTypes.test_large_binaryc                 C   sP   t dddd g}t|t d t d|i}t|tdt fgd d S )NZ123r'   r   ro  r  )r3   r_   re   rE   large_stringr4   rY   r=   ru  r   r   r$   test_large_string  s     z,TestConvertStringLikeTypes.test_large_stringc           
      C   s   dddddg}t d|i}tdt }t|g}tjj||d}|jdd}t d|i}t	j
||dd |jdd}t dt |i}	t	j
||	dd d S )Nr'   rS  r  Fstrings_to_categoricalTr  )r3   r4   rE   ru   r4  r=   rG   rH   rI   rP   rQ   Categorical)
rw   rl   rR   ru   r=   rW   result1	expected1r   	expected2r   r   r$   test_table_empty_str  s    z/TestConvertStringLikeTypes.test_table_empty_strc                 C   s   dddddg}t d|i}tdt }t|g}tjj||d}t d|i}t dt |i}|j	dgd}t
j||dd |j	g d}	t
j|	|dd |j	dd}
t
j|
|dd |j	t d}t
j||dd d S )Nr'   rS  r  
categoriesTr  )rS  )r3   r4   rE   ru   r4  r=   rG   rH   r{  rI   rP   rQ   tuple)rw   rl   rR   ru   r=   rW   Zexpected_strZexpected_catr|  r   result3Zresult4r   r   r$   test_selective_categoricals  s    z6TestConvertStringLikeTypes.test_selective_categoricalsc                 C   s8   t jg t  d}t jj|gdgd}|jdgd d S )Nr  colr6  r   r  )rE   r^   r.   rG   r   rI   )rw   r^   rW   r   r   r$   &test_to_pandas_categorical_zero_length  s    zATestConvertStringLikeTypes.test_to_pandas_categorical_zero_lengthc                 C   sN   t ddddg }t jj|gdgd}|jdgd}| |sJtd S )Nr   r  r  r  r  )rE   r^   dictionary_encoderG   r   rI   rJ   rK   )rw   r^   rW   rX   r   r   r$   ,test_to_pandas_categories_already_dictionary  s    zGTestConvertStringLikeTypes.test_to_pandas_categories_already_dictionaryc              	   C   s   dddddg}t d|i}tdt }t|g}tjj||d}|jdd}t dt 	|i}t
j||dd ttj |jddd	 W 5 Q R X d S )
Nr   r   r   rS  r  Try  r  rz  zero_copy_only)r3   r4   rE   ru   r4  r=   rG   rH   rI   r{  rP   rQ   r   r   rk  rw   rl   rR   ru   r=   rW   rX   rS   r   r   r$   (test_table_str_to_categorical_without_na  s    zCTestConvertStringLikeTypes.test_table_str_to_categorical_without_nac              	   C   s   d ddt jg}td|i}tdt }t|g}tjj	||d}|j
dd}tdt|i}tj||dd ttj |j
ddd W 5 Q R X d S )	Nr   r   rS  r  Try  r  r  )r5   rV  r3   r4   rE   ru   r4  r=   rG   rH   rI   r{  rP   rQ   r   r   rk  r  r   r   r$   %test_table_str_to_categorical_with_na  s    z@TestConvertStringLikeTypes.test_table_str_to_categorical_with_nac                 C   s2   t tjdgtdt  }|jt  ks.td S )Nr\  r   )rE   r^   r5   r   r4  r[   rK   )rw   	convertedr   r   r$   test_array_of_bytes_to_strings  s    z9TestConvertStringLikeTypes.test_array_of_bytes_to_stringsc              	   C   s>   t jtjjdd" ttjdgtdt  W 5 Q R X d S )Nzwas not a utf8 stringr  s   r   )	r   r   rE   librk  r^   r5   r   r4  rw   r   r   r$   'test_array_of_bytes_to_strings_bad_data  s
    zBTestConvertStringLikeTypes.test_array_of_bytes_to_strings_bad_datac              	   C   s,  t jdddgdd}tj|tdd}tjt|tdd}||sNtt dd	dg}tj|td|d
}tjdd dgtdd}||sttjtj	j
dd. t jdddgdd}tj|tdd W 5 Q R X tjtj	j
dd. t jdddgdd}tj|tdd W 5 Q R X d S )NrW  rh  s   bazz|S3r   rs   r  FTr  z)Got bytestring of length 3 \(expected 4\)r  r   z*Got bytestring of length 12 \(expected 3\)z|U3)r5   r^   rE   rZ  r   rJ   rK   r   r   r  rk  )rw   rd   r  rS   rf   r   r   r$   ,test_numpy_string_array_to_fixed_size_binary!  s&    
zGTestConvertStringLikeTypes.test_numpy_string_array_to_fixed_size_binaryN)rD  rE  rF  rU  r[  r   rJ  large_memoryrd  rK  rg  rj  rl  rq  rs  rv  rx  r  r  r  r  r  r  r  r  r  r   r   r   r$   rR  b  s,   

rR  c                   @   s  e Zd ZdZededgZededgZededgZe	j
de	jeed	d
dde	jeedddde	jeeddddgdd Ze	j
de	jedde	jedde	jeddgdd Zdd Zdd Zdd Zdd Zd d! Zd"S )#TestConvertDecimalTypesz,
    Conversion test for decimal types.
    z	-1234.123z1234.439z-129934.123331z129534.123731r  z-314292388910493.12343437128)rl   r  r   rs   	decimal32idr  r   	decimal64r  r  
decimal128c                 C   sR   t d|i}tjj|dd}td|}tj|g|jjd}|j|sNt	d S )Nr  Fr   r  )
r3   r4   rE   rG   rH   ru   r=   r   rJ   rK   )rw   rl   r  rS   rW   ru   rT   r   r   r$   test_decimal_from_pandasJ  s
    z0TestConvertDecimalTypes.test_decimal_from_pandasrl   c                 C   s2   t d|i}tj|}| }t|| d S )Nr  )r3   r4   rE   rG   rH   rI   rP   rQ   )rw   rl   rS   r  rR   r   r   r$   test_decimal_to_pandasX  s    z.TestConvertDecimalTypes.test_decimal_to_pandasc              	   C   s   t dg}tdd}ttj tj||d W 5 Q R X t dg}tdd}ttj tj||d W 5 Q R X d S )Nz1.234rT  r&   r  z1.2345rs   )r  r"  rE   r  r   r   rk  r^   )rw   data1Ztype1data2Ztype2r   r   r$   "test_decimal_fails_with_truncationc  s    z:TestConvertDecimalTypes.test_decimal_fails_with_truncationc                 C   s   t dt dg}t|}t|}| |ks8t|jt	ddksNttj|t	ddd}t dt dg}| |kstd S )	Nz0.01z0.001rs   r  r   r  z0.01000z0.00100)
r  r"  r3   r_   rE   r^   ra  rK   r[   r  )rw   r   seriesr^   rS   r   r   r$   &test_decimal_with_different_precisionsn  s    

z>TestConvertDecimalTypes.test_decimal_with_different_precisionsc                 C   sP   t tdd g}t|tddd t d gd }t|tddd d S )N3.14r  r   ro  r&   r3   r_   r  r"  re   rE   r  rw   r  r   r   r$   $test_decimal_with_None_explicit_type|  s    z<TestConvertDecimalTypes.test_decimal_with_None_explicit_typec                 C   s,   t tdd g}t|tddd d S )Nr  rs   r&   rr  r  r  r   r   r$   !test_decimal_with_None_infer_type  s    z9TestConvertDecimalTypes.test_decimal_with_None_infer_typec                 C   s0   ddidt did}tj|}t| d S )Nr   r   z0.0r   )r  r"  r3   r4   	from_dictrY   )rw   Ztmpdirr   rR   r   r   r$   test_strided_objects  s
    z,TestConvertDecimalTypes.test_strided_objectsN)rD  rE  rF  rG  r  r"  r  r  r  r   rJ  rK  paramrE   r  r  r  r  r  r  r  r   r   r   r$   r  9  s6   
	
r  c                
   @   s  e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zejd,ejd-d.gd/gd0gd0d/gd0gfejd1d2gd3gd0gd0d3gd0gfejd0d0gd0gd0gd0d0gd0gfgd4d5 Z d6d7 Z!d8d9 Z"d:d; Z#d<d= Z$d>d? Z%ejj&d@dA Z'dBdC Z(dDdE Z)dFdG Z*dHdI Z+dJdK Z,dLdM Z-d0S )NTestConvertListTypesz,
    Conversion tests for list<> types.
    c                 C   st   t  \}}t|||d tjj||dd}||jj}|j|sHt	|j
D ] }||}t|| |jd qNd S Nr=   rT   Fr  r  )r   rY   rE   rG   rH   with_metadatar=   r   rJ   rK   r   ru   rm   r[   rw   rR   r=   rW   rT   r2  ru   r   r   r$   test_column_of_arrays  s    


z*TestConvertListTypes.test_column_of_arraysc                 C   s   d}t jt jd|dt jd|dd t jd|dgtd}tt }tj||d}|d  tt	dkspt
|d  tt	dkst
|d  d kst
|d	  dgkst
d S )
Nrm  rT  r   r   r   r  r   r&   rs   )r5   r^   r6   r   rE   r  r,   rk  r   r9   rK   )rw   r   rd   rb   r#  r   r   r$   test_column_of_arrays_to_py  s    z0TestConvertListTypes.test_column_of_arrays_to_pyc                 C   s   t jddgdggt t  d}t jj|gdgd}| }tdddgdggi}t	
|| |d  }t	jt||d dd d S )NTFr  r   r   r   rg   )rE   r^   r  rf  rG   r   rI   r3   r4   rP   rQ   r`   r_   )rw   r^   rW   rR   r  ra   r   r   r$   test_column_of_boolean_list  s    "z0TestConvertListTypes.test_column_of_boolean_listc                 C   s   t jtdtdgtdggt t ddd}t jj|gdgd}| }t	
dtdtdgtdggi}t|| d S )	Nr   r   z3.3r&   r   r  r   r   )rE   r^   r  r"  r  r  rG   r   rI   r3   r4   rP   rQ   )rw   r^   rW   rR   r  r   r   r$   test_column_of_decimal_list  s    

z0TestConvertListTypes.test_column_of_decimal_listc              
   C   s   t tjtjtjgtd}tt t	t t
tddgfD ]X}tj||d}tjd d g|d}||s|ttt tj|j|d W 5 Q R X qJd S )Nr   f0r.   r  )r3   r_   r5   r^   rV  r   rE   r  r/   
large_liststructru   rJ   rK   r   r   	TypeErrorrl   )rw   ra   r  rX   rS   r   r   r$   +test_nested_types_from_ndarray_null_entries  s    z@TestConvertListTypes.test_nested_types_from_ndarray_null_entriesc                 C   st   t  \}}t|||d tjj||dd}||jj}|j|sHt	|j
D ] }||}t|| |jd qNd S r  )r   rY   rE   rG   rH   r  r=   r   rJ   rK   r   ru   rm   r[   r  r   r   r$   test_column_of_lists  s    


z)TestConvertListTypes.test_column_of_listsc                 C   sZ   g dddgddddgg dgg}t dd |D }t|}t | }t|| d S )	Nr&   rs   r   r   r   r   c                 S   s   g | ]}t j|td qS )r   )r5   r^   float)r"   ra   r   r   r$   r%     s     zITestConvertListTypes.test_column_of_lists_first_empty.<locals>.<listcomp>)r3   r_   rE   r^   rI   rP   r`   )rw   Z	num_listsr  rd   rX   r   r   r$    test_column_of_lists_first_empty  s
    
z5TestConvertListTypes.test_column_of_lists_first_emptyc                 C   s   t dtjddgd ddgddgddgd	d
ggtdi}ttdtt	 g}tj
j|d d |d}tj
j|dd  |d}t||g}| }t|| d S )Nlistsr   r&   rs   r   r   r   r   r   	   r   r  )r3   r4   r5   r^   r   rE   r=   ru   r  r/   rG   rH   concat_tablesrI   rP   rQ   )rw   rR   r=   r  r  rW   rX   r   r   r$   test_column_of_lists_chunked  s*     z1TestConvertListTypes.test_column_of_lists_chunkedc              	   C   s^   t dtjg tdi}ttdtt	 g}tj
j||d}| }t|| d S )Nr  r   r  )r3   r4   r5   r^   r   rE   r=   ru   r  r/   rG   rH   rI   rP   rQ   )rw   rR   r=   rW   rX   r   r   r$   "test_empty_column_of_lists_chunked  s     z7TestConvertListTypes.test_empty_column_of_lists_chunkedc           
   	   C   s   ddgddgddgddgd	d
gddgddgddgg}ddgddgg}t |}t |}t jj|gdgd}t jj|gdgd}t ||g}| }td|| i}	t	||	 d S )Nr   r   r&   rs   r   r   r   r   rT  r  r                 r   r        r   r   )
rE   r^   rG   r   r  rI   r3   r4   rP   rQ   )
rw   r  r  r  r  r  r  ZconcatenatedrX   rS   r   r   r$   test_column_of_lists_chunked2  s      

z2TestConvertListTypes.test_column_of_lists_chunked2c                 C   sP   t  \}}tj|gd dd}|d jd d d }|jd dksDtt| d S )Nr   T)Zignore_indexr/   rs   r   r   )r   r3   concatrl   stridesrK   rm   )rw   rR   r=   rd   r   r   r$   test_column_of_lists_strided+  s
    
z1TestConvertListTypes.test_column_of_lists_stridedc                 C   s   t jd d gd gtd}t|}tt|}||s<t|jtt	 ksTtt jd d d d gt jd d gtdgtd}t|}td d d d gd d gg}||std S )Nr   )
r5   r^   r   rE   r   rJ   rK   r[   r  r  )rw   r   rd   rS   r  r   r   r$   test_nested_lists_all_none4  s    

z/TestConvertListTypes.test_nested_lists_all_nonec                 C   sR   t g g g g}t|}tt|}||s6t|jtt	 ksNtd S N)
r3   r_   rE   r^   r   rJ   rK   r[   r  r  rw   r   rd   rS   r   r   r$   test_nested_lists_all_emptyC  s
    
z0TestConvertListTypes.test_nested_lists_all_emptyc                 C   sR   t g dgg}t|}tt|}||s6t|jtt	 ksNtd S Nr   )
r3   r_   rE   r^   r   rJ   rK   r[   r  r4  r  r   r   r$   test_nested_list_first_emptyK  s
    
z1TestConvertListTypes.test_nested_list_first_emptyc                 C   s   t tjdddgddd g}t|}t|j}tjdddgd gtt d}||sdt	||srt	t tjdddgddd g}t|}tjdddgd gtt
 d}||st	d S )Nr   r&   rs   rm  r   r  rJ  )r3   r_   r5   r^   rE   rl   r  r,   rJ   rK   r0   )rw   r   rX   r   rS   Zdata3r  Z	expected3r   r   r$   test_nested_smaller_intsS  s    
"
"z-TestConvertListTypes.test_nested_smaller_intsc                 C   s   t dtjdgddggfdddgddggfdd dgd	d
ggfdd dgd	d
ggd gfg}t|}ttdtt	 tdtt	 tdtt
 tdttt
 g}t||d d S )NZnan_intsr   r&   rs   r  r   strsr   r   dZnested_strsrL  )r   r5   rV  r3   r4   rE   r=   ru   r  r/   r4  rY   rw   r   rR   rT   r   r   r$   test_infer_listsb  s    
z%TestConvertListTypes.test_infer_listsc           
      C   s   t jt  dd}t t  }ddddgd dddd	gd
dddgg}t j||d}t j||d}| }| }t||D ](\}}	|d kr|	d kstt||	 qzd S )Nr   )Z	list_sizer   r   r&   rs   r   r   r   r   r  rT  r  r  )	rE   r  r/   r^   rI   rX  rK   ro   rp   )
rw   Zfixed_tyZvariable_tyr   Z	fixed_arrZvariable_arrrX   rS   leftrightr   r   r$   test_fixed_size_listt  s    $z)TestConvertListTypes.test_fixed_size_listc              	   C   sj   t dtjddgtjdtjddgtjdgfg}t|}ttdt	t g}t
||d d S )Nr  r   r   r   r&   rs   rL  )r   r5   r^   r/   r3   r4   rE   r=   ru   r  rY   r  r   r   r$   test_infer_numpy_array  s    
z+TestConvertListTypes.test_infer_numpy_arrayc              	   C   s   t dddgt  }t ddgddgdd	ggt t  }t j||gd
dg}t jdddg|}t	dddgdgdddgdddd	gdgg}t	|
 }t   tddt t|| W 5 Q R X d S )Nr   r&   rs   r   r   r   r  eff1rI  r   )r  rI  rA   rB   )rE   r^   r.   r  r4  StructArrayr   	ListArrayr3   r_   rI   rL   rM   rN   rO   rP   r`   )rw   r  rS  structsr   rS   r  r   r   r$   test_to_list_of_structs_pandas  s$    
z3TestConvertListTypes.test_to_list_of_structs_pandasc                 C   s   t tjt dkr.t tjt dk r.td dddgfddd	gfggd
g fddd dgfgdddgfggg}t|tt	t
 tt
 }| }t|}t   tddt t|| W 5 Q R X d S )N1.25.0.dev0r   $Regression in pandas with numpy 1.25r   r   r   r  r   r  r  rV  Nquuxr  quzr  grA   rB   )r   r5   r  r3   r   r  rE   r^   r  map_utf8rI   r_   rL   rM   rN   rO   rP   r`   )rw   r   rd   r  rS   r   r   r$   test_to_list_of_maps_pandas  s    
"(

z0TestConvertListTypes.test_to_list_of_maps_pandasc           	   
   C   sN  t tjt dkr.t tjt dk r.td tdddddd	dgd
d}tdgdgddgddgg dddggt	t
 dd}tjdddg||}tjdd
dg|}| }tdddgfdddgfggdg fdggg}|d
d }tdg fdggg}t , tddt t|| t|| W 5 Q R X dS )zL
        A slightly more rigorous test for chunk/slice combinations
        r  r   r  rA   r   r  r  rV  r  r   r   r   r   r   r  Nr  r&   r   r   r  rB   )r   r5   r  r3   r   r  rE   r^   slicer  r4  MapArrayr   r  rI   r_   rL   rM   rN   rO   rP   r`   )	rw   r5  itemsmaprd   r  rS   Zseries_slicedZexpected_slicedr   r   r$   "test_to_list_of_maps_pandas_sliced  sH    
     
z7TestConvertListTypes.test_to_list_of_maps_pandas_slicedzt,data,expectedr   r&   rs   NZaaabbr   c                 C   sZ   t dddg}t|}tjj||t| d}tjj|t| d|sVt	d S )NTFr  r  )
r5   r^   r3   r_   rE   r  rH   r  rJ   rK   )rw   r   r   rS   r8  ra   rX   r   r   r$   ,test_array_from_pandas_typed_array_with_mask  s    

zATestConvertListTypes.test_array_from_pandas_typed_array_with_maskc                 C   sX   t jdtd}|g  tt dddg|d}tj	|}|
 }t|| d S )N)rs   r   r   r   3r   )r5   emptyr   fillr3   r4   r^   rE   rG   rH   rI   rP   rQ   )rw   Zempty_list_arrayrR   r8  rX   r   r   r$   test_empty_list_roundtrip  s    
z.TestConvertListTypes.test_empty_list_roundtripc                 C   s`   t  \}}|D ]L}||j j}tjt||jd}t|}|j|jksLt||stqd S Nr  )	r   r]   rl   rE   r^   r   r[   rK   rJ   )rw   rR   r=   ru   rd   rS   rX   r   r   r$   test_array_from_nested_arrays	  s    

z2TestConvertListTypes.test_array_from_nested_arraysc              	   C   s   t jdddgdggd gt t t  d }t N tddtj	 tddt
 tj|tjdddgdggd gtd	d
d W 5 Q R X d S )Nr   r&   rs   r   r  rA   z&Creating an ndarray from ragged nestedrB   r   Frg   )rE   r^   r  r/   rI   rL   rM   rN   r5   ZVisibleDeprecationWarningrO   rP   r`   r3   r_   r   rp  r   r   r$   test_nested_large_list	  s     

 z+TestConvertListTypes.test_nested_large_listc              	   C   s   t jt jfD ]}t jddgd dgg g|t  d }tj|t	ddgd dgg gdd	 t jddgd dgg g|t 
 d }tj|t	ddgd dgg gdd	 qd S )
Nr   r  ccr  s   aas   bbs   ccFrg   )rE   r  r  r^   rt  rI   rP   r`   r3   r_   rw  )rw   Zlist_type_factoryra   r   r   r$   test_large_binary_list#	  s"    

 

 z+TestConvertListTypes.test_large_binary_listc                 C   sz   t ddd dg }t jdddddg|}t| }t	|
 | |ddd dg}d |d< t	|
 | d S )Nr   r  r   r   rs   r   r&   )rE   r^   r  r  r   r3   r_   ra  rP   r`   rI   take)rw   childrd   rS   r   r   r$   test_list_of_dictionary2	  s    z,TestConvertListTypes.test_list_of_dictionaryc                 C   s~   d}t jttj|dfddt|d}tj	|}|d }|j
dksNtt|ddksdtt|d	d	ksztd S )
Ni    i   r(   r   r   r   r&   i r   )r3   r4   r  r   r5   zerosr9   rE   rG   rH   ra  rK   r   r  )rw   nrR   rW   Zcolumn_ar   r   r$   #test_auto_chunking_on_list_overflow?	  s    z8TestConvertListTypes.test_auto_chunking_on_list_overflowc                 C   sX   ddgdgdddgdgg}t d|i}tdtt t fg}t||d	 d S )
Nr   r      br&      crs      dr      er      fr      gr   r  r  )r3   r4   rE   r=   r  rZ  r.   rY   )rw   r   rR   r=   r   r   r$   test_map_array_roundtripN	  s     z-TestConvertListTypes.test_map_array_roundtripc                 C   s   ddgdgdddgdgg}dd	 |D }t j|t t  t  d
}t j|t t  t  d
}t ||g}t|| }| }t	j
||dd d S )Nr  r  r  r  r  r
  r  c                 S   s   g | ]}d d |D qS )c                 S   s   g | ]\}}||d  fqS )r&   r   r"   kr  r   r   r$   r%   ^	  s     zJTestConvertListTypes.test_map_array_chunked.<locals>.<listcomp>.<listcomp>r   r"   rowr   r   r$   r%   ^	  s     z?TestConvertListTypes.test_map_array_chunked.<locals>.<listcomp>r  Frg   )rE   r^   r  rZ  r.   r  r3   r_   rI   rP   r`   )rw   r  r  arr1r  rd   rS   actualr   r   r$   test_map_array_chunkedY	  s    z+TestConvertListTypes.test_map_array_chunkedc                 C   sj   ddgd dddgdgg}dd |D }t |}tj|tt t d	}| }tj	||d
d d S )Nr  r  r  r  )r  Nr  c                 S   s&   g | ]}|d k	rdd |D nd qS )Nc                 S   s(   g | ] \}}||d k	rt |nd fqS r  )r  r  r   r   r$   r%   o	  s     zMTestConvertListTypes.test_map_array_with_nulls.<locals>.<listcomp>.<listcomp>r   r  r   r   r$   r%   o	  s    zBTestConvertListTypes.test_map_array_with_nulls.<locals>.<listcomp>r  Frg   )
r3   r_   rE   r^   r  rZ  r.   rI   rP   r`   )rw   r   rS   rd   r  r   r   r$   test_map_array_with_nullsh	  s    
z.TestConvertListTypes.test_map_array_with_nullsc                 C   s   t dddg}t dddddg }t ttt|}t j|||}t	dd	d
gddgg}|
 }tj||dd d S )Nr   rs   r   r   r   r   r  )r   r   )r   r   )r&   r   )rs   r   )r   r  Frg   )rE   r^   r  r   r9   r   r  r   r3   r_   rI   rP   r`   )rw   offsetsr  r5  rd   rS   r  r   r   r$   !test_map_array_dictionary_encodedw	  s    z6TestConvertListTypes.test_map_array_dictionary_encodedc           	   	   C   sp  t ddgdddgd dd gddgg}t |d	d|ddg}| }tjd
dgdddgd dtjggdd}t||D ]*\}}|d kr||kstq|t	
|| q|td
dddddtjgg}t	
|d	 j| |dd }tjdddgd dtjggdd}t||D ]0\}}|d kr2||ks>tnt	
|| qtddddtjgg}t	
|d	 j| d S )Nr   r&   rs   r   r   r   r   r   r   ry   rz   r{   r|   r}   r~   r   r   )rE   r^   r  r  to_numpyr5   r`  rX  rK   ro   rp   base)	rw   rd   Zchunked_arrr  rS   r  r  Zexpected_baseZnp_arr_slicedr   r   r$   test_list_no_duplicate_base	  s,    & 
z0TestConvertListTypes.test_list_no_duplicate_basec              
   C   s   t jjt ddddgt dddddd gt dd	dgd
}|jdd}tjddgd dtjggdd}t||D ]*\}}|d kr||kstqrt	
|| qrd S )Nr   r&   r   r   r   c   rs   FT)r  rl   rf   r  ry   rz   r{   r   r   )rE   r  r   r^   r  r5   r`  rX  rK   ro   rp   )rw   rd   r  rS   r  r  r   r   r$   test_list_values_behind_null	  s    z1TestConvertListTypes.test_list_values_behind_null).rD  rE  rF  rG  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   rJ  rK  rE   r/   r4  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r   r   r   r$   r    sd   	'



		
r  c                   @   sh   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Ze	j
je	j
jdd Zdd Zdd Zdd ZdS )TestConvertStructTypesz,
    Conversion tests for struct types.
    c              	   C   sf   t dddddddgi}tdtdt fdt fgfg}t||d	 t|||d
 d S )Nr  r   r&   r   rs   r   r   r   rL  r  )r3   r4   rE   r=   r  r/   rY   )rw   rR   rT   r   r   r$   test_pandas_roundtrip	  s       z,TestConvertStructTypes.test_pandas_roundtripc                 C   s   t jd ddgt  d}t jdd dgt  d}t jddd gt  d}t j|||gdd	d
g}td ddddd ddddd dg}t|	 }t
|| d S )Nr&   rs   r  r   r   TFr  r  r  )r  r  r  )rE   r^   r/   r4  rf  r  r   r3   r_   rI   rP   r`   )rw   r  r  r  rd   rS   r  r   r   r$   test_to_pandas	  s    


z%TestConvertStructTypes.test_to_pandasc           
      C   s   t   t }tjdgt d}tjdgt d}tj|gdg}tj|gdg}t||g}t	
ddiddig}t	
| }t|| ~~~~~~t }	|	|kstd S )Nr   r  r&   r  )gccollectrE   total_allocated_bytesr^   r/   r  r   r  r3   r_   rI   rP   r`   rK   )
rw   Zbytes_startZints1ints2r  r  rd   rS   r  Z	bytes_endr   r   r$   test_to_pandas_multiple_chunks	  s*    z5TestConvertStructTypes.test_to_pandas_multiple_chunksc                 C   s\  t dt jfdt jfg}ttdt tdt g}t jg |d}tj||d}| g kslt	t jddg|d}tj||d}| dd	d
ddd
gkst	tj|t dd	g|d}| dd	d
d gkst	t g }tg }t jg |d}tj||d}| g ks"t	t jddg|d}tj||d}| i i gksXt	d S )Nr#   )Zy_titler  r  r   r  )r  T)+   Fr  T)r#   r  r'  Fr  r   )
r5   r   r.   rf  rE   r  ru   r^   ra  rK   rw   rA  r  r   rd   r   r   r$   test_from_numpy	  s0    


z&TestConvertStructTypes.test_from_numpyc                 C   s  t dt dt jfdt jfgfdt jfdt jfg}|jdksDtt	t
dt	t
dt t
dt gt
dt t
dt g}t jg |d}tj||d}| g kstt jd	d
g|d}tj||d}| ddddddddddddgkstd S )Nr#   xxyyr  zr  r   r  ))r   Tr&   r   ))rs   Fr   r  r   T)r*  r+  r&   r   )r#   r  r,  rs   Fr   r  )r5   r   r,   rf  r-   object_itemsizerK   rE   r  ru   r4  r^   ra  r(  r   r   r$   test_from_numpy_nested
  s6    z-TestConvertStructTypes.test_from_numpy_nestedc                    s  d}t dt jfdg}d|j }d| }|||j  }t j||d}t j||d< ||d< t j|d |d dk < t	t
dt t
dt g}tj||d	d
}|jdkstdd  d fdd	}	|	|| ~t j|dk }
tj|||
d	d}|jdkst|	|||
 ~d S )Nl        r#   )r  r   i      .r   r  r'  Tr  r&   c                 s   s   |   D ]}|E d H  qd S r  )Z
iterchunks)rd   r  r   r   r$   iter_chunked_array<
  s    zHTestConvertStructTypes.test_from_numpy_large.<locals>.iter_chunked_arrayc           	   	      s   t | t |kst|d }|d }t | D ]\}}zr| }|d k	r`|| r`|d kstnH|| }t|r|d d kstn|d |kst|d || kstW q0 tk
r   td|  Y q0X q0d S )Nr#   r  zFailed at index)r   rK   	enumeraterk  r5   rj  	Exceptionprint)	rd   r   rf   ZxsZysir  r  r#   r1  r   r$   check@
  s     

z;TestConvertStructTypes.test_from_numpy_large.<locals>.check)r[   rf   rH   )N)r5   r   r1   r.  r  r7   Zrandom_samplerV  rE   r  ru   rZ  r^   ra  rK   )rw   Ztarget_sizerA  bsblockr  r   r  rd   r7  rf   r   r6  r$   test_from_numpy_large(
  s.    

z,TestConvertStructTypes.test_from_numpy_largec              	   C   s   t t dt  t dt  g}tdtjfdtjfg}tjg |d}tj	t
dd t j||d W 5 Q R X tg }tj	tdd t j||d W 5 Q R X d S )	Nr#   r  r,  r   zMissing field 'y'r  r  zExpected struct array)rE   r  ru   r.   rf  r5   r   r^   r   r   r   r  )rw   r  rA  r   r   r   r$   test_from_numpy_bad_input_
  s     
z0TestConvertStructTypes.test_from_numpy_bad_inputc                 C   s   t dddgi}t dddddddgi}td	t fd
t fg}t|d }t||d |d td|fg}t	||||d d S )NZtuples)r   r&   rs   r   r   r&   r   rs   r   r   r   re  )rS   r=   rT   )
r3   r4   rE   r  r/   r5   asarrayrm   r=   rY   )rw   rR   r  Zstruct_typerd   rT   r   r   r$   test_from_tuplesn
  s$        z'TestConvertStructTypes.test_from_tuplesc                    s   ddg t dddg t ddd g g}t jj| d}tdd |D  } fd	d
|D }t|}t	|
 | |dd dg}d |d< t	|
 | d S )Nr  r  i  i  r   r   c                 s   s   | ]}|  V  qd S r  )ra  )r"   r  r   r   r$   	<genexpr>
  s     zCTestConvertStructTypes.test_struct_of_dictionary.<locals>.<genexpr>c                    s   g | ]}t t |qS r   )rW  rX  r  r   r   r$   r%   
  s     zDTestConvertStructTypes.test_struct_of_dictionary.<locals>.<listcomp>r   r&   r   )rE   r^   r  r  r   rX  r3   r_   rP   r`   rI   r  )rw   childrenrd   Zrows_as_tuplesZrows_as_dictsrS   r   r   r$   test_struct_of_dictionary
  s    
z0TestConvertStructTypes.test_struct_of_dictionaryN)rD  rE  rF  rG  r   r!  r&  r)  r/  r   rJ  Zslowr  r:  r;  r>  rA  r   r   r   r$   r  	  s   5r  c                   @   sp   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )TestZeroCopyConversionz@
    Tests that zero-copy conversion works with some types.
    c                 C   s.   t dddgjdd}t|dddg d S )Nr   r   r&   Tr  rn   )rw   rX   r   r   r$   test_zero_copy_success
  s    z-TestZeroCopyConversion.test_zero_copy_successc                 C   s^   t jtddgtjdgdd}|jdd}tddg}tj	t
|t
|dd d S )	Nr   r   r/   r   Tr  Frg   )rE   r   r   r5   r^   rI   r3   r{  rP   r`   r_   )rw   rd   rX   rl   r   r   r$   test_zero_copy_dictionaries
  s    z2TestZeroCopyConversion.test_zero_copy_dictionariesc                 C   s2   t jdgdd}t|jdd}t|| d S )Nr6  r   r   Tr  r5   r^   rE   rI   ro   rp   rw   rd   rX   r   r   r$   test_zero_copy_timestamp
  s    z/TestZeroCopyConversion.test_zero_copy_timestampc                 C   s2   t jdgdd}t|jdd}t|| d S )Nr   r   r   Tr  rE  rF  r   r   r$   test_zero_copy_duration
  s    z.TestZeroCopyConversion.test_zero_copy_durationc              	   C   s(   t tj |jdd W 5 Q R X d S )NTr  )r   r   rE   rk  rI   rw   rd   r   r   r$   check_zero_copy_failure
  s    z.TestZeroCopyConversion.check_zero_copy_failurec                 C   s   |  tdddg d S )NABCrJ  rE   r^   r  r   r   r$   &test_zero_copy_failure_on_object_types
  s    z=TestZeroCopyConversion.test_zero_copy_failure_on_object_typesc                 C   s   |  tddd g d S )Nr   r   rN  r  r   r   r$   *test_zero_copy_failure_with_int_when_nulls
  s    zATestZeroCopyConversion.test_zero_copy_failure_with_int_when_nullsc                 C   s   |  tddd g d S )Nrd  ry   rN  r  r   r   r$   ,test_zero_copy_failure_with_float_when_nulls
  s    zCTestZeroCopyConversion.test_zero_copy_failure_with_float_when_nullsc                 C   s   |  tddg d S )NTFrN  r  r   r   r$   $test_zero_copy_failure_on_bool_types
  s    z;TestZeroCopyConversion.test_zero_copy_failure_on_bool_typesc                 C   s2   t jddgddggt t  d}| | d S )Nr   r&   r   r  r  )rE   r^   r  r/   rJ  rI  r   r   r$   $test_zero_copy_failure_on_list_types
  s    $z;TestZeroCopyConversion.test_zero_copy_failure_on_list_typesc                 C   s&   t jdd gdd}| t| d S )Nr   r   r   r5   r^   rJ  rE   rI  r   r   r$   .test_zero_copy_failure_on_timestamp_with_nulls
  s    zETestZeroCopyConversion.test_zero_copy_failure_on_timestamp_with_nullsc                 C   s&   t jdd gdd}| t| d S )Nr   r   r   rT  rI  r   r   r$   -test_zero_copy_failure_on_duration_with_nulls
  s    zDTestZeroCopyConversion.test_zero_copy_failure_on_duration_with_nullsN)rD  rE  rF  rG  rC  rD  rG  rH  rJ  rO  rP  rQ  rR  rS  rU  rV  r   r   r   r$   rB  
  s   rB  c                  C   s$   t  } t| dd t| ddd d S )NFr?   Tr@   rU   r<   rY   rR   r   r   r$   _non_threaded_conversion
  s    rZ  c                  C   s$   t  } t| dd t| ddd d S )NTr?   rW  rX  rY  r   r   r$   _threaded_conversion
  s    r[  c                   @   s  e Zd ZdZeje feje feje fej	e	 fej
e
 feje feje feje feje feje feje feje feje fejedfejee	 fgZdd Zdd Zdd Zd	d
 Zdd Zdd Zdd Zdd Zdd Zej !dddddgiej"fddddgiej"fdddgiej"fdddgiej#fdddgiej#fdddgiej#fgdd Z$dd Z%dd  Z&d!d" Z'd#d$ Z(d%d& Z)d'd( Z*d)d* Z+d+d, Z,d-d. Z-d/S )0TestConvertMiscz)
    Miscellaneous conversion tests.
    rT  c                 C   s    t dd d d gi}t| d S r  r   r   r   r   r$   test_all_none_objects
  s    z%TestConvertMisc.test_all_none_objectsc                 C   s2   t dd d d gi}|d d|d< t| d S )Nr   r   )r3   r4   r   rY   r   r   r   r$   test_all_none_category
  s    z&TestConvertMisc.test_all_none_categoryc                 C   s.   | j D ]"\}}tjg |d}t||d qd S )Nr   r  )
type_pairsr5   r^   rm   )rw   r   pa_typerd   r   r   r$   test_empty_arrays
  s    z!TestConvertMisc.test_empty_arraysc                 C   s
   t   d S r  )rZ  r  r   r   r$   test_non_threaded_conversion  s    z,TestConvertMisc.test_non_threaded_conversionc                 C   s0   t d}z|t W 5 |  |  X d S )Nr&   )mpZPoolclosejoinapplyr[  )rw   poolr   r   r$   %test_threaded_conversion_multiprocess  s
    
z5TestConvertMisc.test_threaded_conversion_multiprocessc                 C   s   d}dd ddt jg}dddddg}d	d d
dt jg}t|| tj|| ddgdt|| t|| tj|| dddgdd|| || || || || d
}t|}t| |D ]}t||  qd S )Nr   r   r  rV  r   r   r   r   rW  rh  rX  r  Tr  r   )
Zcat_stringsZcat_strings_with_naZcat_intsZ
cat_binaryZcat_strings_orderedr  r%  rS  Zstrings2Zstrings3)r5   rV  r3   r{  r4   rY   rm   )rw   rT  rb  rc  Zv3r6  rR   r  r   r   r$   test_category  s2    
 
zTestConvertMisc.test_categoryc                 C   sP   dd }t jdddgddgdt jdddgddgddg}|D ]}|| q>d S )	Nc                 S   s.   t | }| }tt|t|  d S r  )rE   r^   rI   rP   r`   r3   r_   )r  rd   rX   r   r   r$   _check)  s    
zBTestConvertMisc.test_category_implicit_from_pandas.<locals>._checkr   r   r   r  Tri  )r3   r{  )rw   rk  r6  rd   r   r   r$   "test_category_implicit_from_pandas'  s    z2TestConvertMisc.test_category_implicit_from_pandasc                 C   s    t dt g i}t| d S )Ncat)r3   r4   r{  rY   r   r   r   r$   test_empty_category6  s    z#TestConvertMisc.test_empty_categoryc                 C   s   t  dft  dffD ]\}}t g t t  |}| }tjg t	j
g |dd}tt|t| t d|i}| }td|i}t|| qd S )Nr   r/   r   r  r   )rE   r4  r/   r  
dictionaryr,   rI   r3   r{  r5   r^   rP   r`   r_   rW   r4   rQ   )rw   r`  r   r   rX   rS   rW   r   r   r$   test_category_zero_chunks;  s     z)TestConvertMisc.test_category_zero_chunkszdata,error_typer   r   rz   Try   c              	   C   s8   t |}d}tj||d tj| W 5 Q R X d S )N/Conversion failed for column a with type objectr  )r3   r4   r   r   rE   rG   rH   )rw   r   Z
error_typerR   r3  r   r   r$   test_mixed_types_failsH  s    
z&TestConvertMisc.test_mixed_types_failsc              
   C   sH  g }dddg}d\}}t j|| d }dddd	d
dddddg
}|D ]}||| qH|t jdd t|| D td	||  t jdddg| td	|| }d |d< || |t j
d|| dd	||  |dktd d df }	|D ]8}
tj|
|d}|d }t| t| t||	d q
d S )Nr   r   r   )r   rs   r   rm  rn  ro  rp  rq  rr  rs  rt  rJ  rK  c                 S   s   g | ]}t d qS rT  r   r"   r5  r   r   r$   r%   g  s     z<TestConvertMisc.test_strided_data_import.<locals>.<listcomp>r   TFr   r   r   r   r   r  )r5   r7   r8   rk   rN  r   r^   r9   r   Zreshaper6   r2   r3   r4   rY   rm   )rw   casesr   NKZrandom_numbersZnumeric_dtypes	type_nameZboolean_objectsZstrided_maskcaserR   r  r   r   r$   test_strided_data_importY  sF    
  
 
 
z(TestConvertMisc.test_strided_data_importc                 C   sX   dd }|t jd gd td |t jtjgd td |t jd tjd gtd d S )Nc                 S   sT   t | }t|t jstt|dks*t|jdks8t|D ]}|t jks<tq<d S )Nrs   )rE   r^   r   Z	NullArrayrK   r   ri   r  )ra   r  itemr   r   r$   _check_series  s    
z5TestConvertMisc.test_all_nones.<locals>._check_seriesrs   r   )r3   r_   r   r5   rV  )rw   r}  r   r   r$   test_all_nones  s    zTestConvertMisc.test_all_nonesc              	   C   s   t ddddddgfdtjdd	dd
dgtjdfddd	dd
dgfg}t|}ttdt	 tdt	 g}t
|||ddg |d d S )Nr   r   r   r&   rs   r   r   ir   rT  r   r   )r=   rS   rT   )r   r5   r^   r.   r3   r4   rE   r=   ru   r/   rY   )rw   r   rR   Zpartial_schemar   r   r$   test_partial_schema  s    

z#TestConvertMisc.test_partial_schemac                 C   s   t i }t|d d t|d dd t jt g d}t||dd t||ddd t ji dddgd	}t|dd t|ddd
 d S )Nr   T)r>   rU   r   Fr   r   r&   r   )rU   r>   )r3   r4   rY   r   )rw   rR   rS   r   r   r   r$    test_table_batch_empty_dataframe  s    
z0TestConvertMisc.test_table_batch_empty_dataframec              	   C   s   t jg t  d}ttjg td}t|	 ttjg tjd t jg t 
 d}t|	 | t jg t t  d}t|	 | t jg t t dt  gd}t|	 | d S )Nr  r   r   )rE   r^   r/   r3   r_   r5   r   rP   r`   rI   r4  r  r  ru   )rw   rd   Zempty_objectsr   r   r$   test_convert_empty_table  s    
"z(TestConvertMisc.test_convert_empty_tablec                 C   s   t dt jfdt jfg}t jddg|d}|jdks:ttj|d t d}| dd	gksdttj|d t d}| d
dgkstdS )zt
        ARROW-2172: converting from a Numpy array with a stride that's
        not a multiple of itemsize.
        r#   r  )r  r   )r&   r   )r   r  r  r  r   r&   N)	r5   r   r.   r-   r^   r  rK   rE   ra  )rw   r   r   rd   r   r   r$   test_non_natural_stride  s    z'TestConvertMisc.test_non_natural_stridec                 C   sV   t jddt jdddd }tj|t d}tjdd	d
dgt d}|| d S )Nr   rT  r   r   r   r&   r  ry   r{   r}   g      @)r5   r6   r0   rE   r^   r1   rJ   )rw   r  Zpa_arrrS   r   r   r$   #test_array_from_strided_numpy_array  s    z3TestConvertMisc.test_array_from_strided_numpy_arrayc              	   C   s   t tdtdddd}ttdt tdt	 g}t
t tjj||d W 5 Q R X tjj||d	d
}|djt	 kstd S )Nabcr   r   rs   )rK  rL  rK  rL  r  Fr_  )r3   r4   r   r5   ZlinspacerE   r=   ru   r4  r.   r   r   r   rG   rH   r2  r[   rK   rb  r   r   r$   test_safe_unsafe_casts  s    z&TestConvertMisc.test_safe_unsafe_castsc                 C   s~   z"t dt jdtjdgi}W n0 tk
rR   t dt dtjdgi}Y nX tjt	dd t
j| W 5 Q R X d S )Nr   r   rs   zSparse pandas datar  )r3   r4   r6  ZSparseArrayr5   rV  AttributeErrorr   r   r  rE   rG   rH   r   r   r   r$   test_error_sparse  s    ""z!TestConvertMisc.test_error_sparseN).rD  rE  rF  rG  r5   r,   rE   r-   r.   r/   r(   r)   r*   r+   rM  r0   r1   r-  r4  rZ  r  r_  r]  r^  ra  rb  rh  rj  rl  rn  rp  r   rJ  rK  r  rk  rr  r{  r~  r  r  r  r  r  r  r  r   r   r   r$   r\  
  s\   	
(r\  c                  C   sT   t ddd dg} tjj| t dd}tjddd dgt d}||sPtd S )Nr   r&   r   Tr[   r   r  )	r3   r_   rE   r  rH   r.   r^   rJ   rK   )rl   rd   rS   r   r   r$   *test_safe_cast_from_float_with_nans_to_int  s    r  c                  C   s   t jt jddddttjddgtddg} t jddd}||	d	|	d
|d d d 	d	d
dddgd t ddgd 
djddgd tjdtjjddddt jddddt jddddd}t j|| dS )Nr  r   r<  r&   r   r  r   rT  r  r  r   TFr   r   r:   Z2013Mr   r   r   )r   r   r   )r   r   r&   rs   r   r   r   r   r   r  rT  r   )r3   r   r   r   repeatr5   Ztiler^   r   rB  r   r_   rl   r7   r8   rU  period_rangeinterval_ranger4   )r   r2  r   r   r   r$   _fully_loaded_dataframe_example  s$    


r  r   rW  r   c                 C   s   t dt tj  kr"t dk r0n n
td tj| d}tj|}tj|	 }|
|sft|j
|jsxt|jj|jjkstd S )Nr   r  r  r   )r   r3   r  r   r  r4   rE   rG   rH   rI   rJ   rK   r=   r   )r   rR   table1table2r   r   r$   !test_roundtrip_with_bytes_unicode	  s    &
r  c                 C   s>   | d }| d } | d }| d } | d }| d }t ||||S )Nr  r  )r   )valrG  secondsminutesr  r   r   r$   _pytime_from_micros  s    r  c                 C   s$   | j d | jd  | jd  | j S )Nl    $'- i r  )hourminuter-  r  )Zpytimer   r   r$   r  #  s    r  c               	   C   sR   G dd d} t d|  |  gi}d}tjt|d tj| W 5 Q R X d S )Nc                   @   s   e Zd ZdS )z6test_convert_unsupported_type_error_message.<locals>.ANr  r   r   r   r$   rK  .  s   rK  r   rq  r  )r3   r4   r   r   r   rE   rG   rH   )rK  rR   r3  r   r   r$   +test_convert_unsupported_type_error_message*  s
    r  c                 C   s,   |   }tj|| jdd}|| s(td S )NTr  )rI   rE   r^   r[   rJ   rK   )rd   ra   r   r   r   r$   test_array_to_pandas_roundtrip<  s    r  c                 C   s   dd t | D }|| S )Nc                 S   s   g | ]}t d qS rs  )r   ru  r   r   r$   r%   H  s     z+_generate_dedup_example.<locals>.<listcomp>)r9   )nuniquerT  unique_valuesr   r   r$   _generate_dedup_exampleG  s    r  c                 C   s   t dd | D |kstd S )Nc                 S   s   h | ]}t |qS r   r  r!   r   r   r$   	<setcomp>M  s     z"_assert_nunique.<locals>.<setcomp>)r   rK   )r  rS   r   r   r$   _assert_nuniqueL  s    r  c                  C   sp   d} d}t | |}tj|t dtj|t dt||gfD ](}t| |  t|jddt| qBd S )Nr   rT  r  Fdeduplicate_objects)	r  rE   r^   rZ  r  r  r  rI   r   )r  rT  rl   rd   r   r   r$   .test_to_pandas_deduplicate_strings_array_typesP  s    
r  c                  C   sz   d} d}t | |}t|}tj|gdg}tj|g}||fD ]0}t| d |  t|jddd t	| qDd S )Nr   rT  r   Fr  )
r  rE   r^   rF   r   rG   r   r  rI   r   )r  rT  rl   rd   r7  r8  r  r   r   r$   .test_to_pandas_deduplicate_strings_table_types\  s    

r  c                  C   sn   d} d}t tjjdd| d}d || d < t|| }t|jdd|  t|jdd	d
| d | d  d S )Nr   rT  i i ʚ;r:   r&   Tr  F)r  r  r   )r   r5   r7   rU  rE   r^   r  rI   )r  rT  r  rd   r   r   r$   .test_to_pandas_deduplicate_integers_as_objectsk  s    r  c            	      C   s   d} d}t t| }ddddifddddifdd	i fdd
i fg}|D ]V\}}}tj|| |d}||}t|jf ||  t|jf ddi|t| qDd S )Nr   rT  r.   r  r  Tr/   r  r*  r(  r  r  F)r   r9   rE   r^   r  r  rI   r   )	r  rT  r  rv  Zraw_typeZ
array_typeZpandas_optionsZraw_arrZ
casted_arrr   r   r$   $test_to_pandas_deduplicate_date_time|  s&    
r  c               	   C   sv   t dddgtjddgd} ttjdt dd	tjd
t dd	g}t	
t tjj| |d W 5 Q R X d S )N333333?g @r   r4  r   r   r   Fr]  r   r  )r3   r4   r5   rV  rE   r=   ru   r1   r  r   r   r   rG   rH   )rR   r=   r   r   r$   /test_table_from_pandas_checks_field_nullability  s    
r  c               
   C   s   t tdddddgfddddgddgd d gfdd d d	d
gfg} | dddg }tdt fdtt fdt fg}tdt fdt fdtt fg}tjj	| dd}tjj	|dd}|j
|st|j
|std S )N	partitionr   r   r6  r&   rs   r   r  r   ffffff
@Fr   )r3   r4   r   rE   r=   r/   r  r1   rG   rH   rJ   rK   )df1r   schema1schema2r  r  r   r   r$   6test_table_from_pandas_keeps_column_order_of_dataframe  s(    



r  c               
   C   s   t tdddddgfddddgddgd d gfdd d d	d
gfg} tdt fdtt fdt fg}| | jdk }| | jdk dddg }tj	j
||dd}tj	j
||dd}|j|st|j|jstd S )Nr  r   r   r6  r&   rs   r   r  r   r  Fr  )r3   r4   r   rE   r=   r1   r  r.   r  rG   rH   rJ   rK   )rR   r=   r  r   r  r  r   r   r$   3test_table_from_pandas_keeps_column_order_of_schema  s     

r  c               
   C   s   t tdddddgfddddgddgd d gfdd d d	d
gfg} dddg}tdtt fdt fdt fg}ddg}tdt fdt fg}tjj	| |dd}tjj	| |dd}|j
|st|j
|std S )Nr  r   r   r6  r&   rs   r   r  r   r  F)r   r>   )r3   r4   r   rE   r=   r  r/   r1   rG   rH   rJ   rK   )rR   Zcolumns1r  Zcolumns2r  r  r  r   r   r$   ;test_table_from_pandas_columns_argument_only_does_filtering  s(    




r  c               
   C   s   t tdddddgfddddgddgd d gfdd d d	d
gfg} tdt fdtt fdt fg}ddg}t	t
 tjj| ||d W 5 Q R X d S )Nr  r   r   r6  r&   rs   r   r  r   r  )r=   r   )r3   r4   r   rE   r=   r.   r  r1   r   r   r   rG   rH   )rR   r=   r   r   r   r$   @test_table_from_pandas_columns_and_schema_are_mutually_exclusive  s    

r  c                  C   s~   t dddddgi} ttjdt ddg}tj| }|jdjdksTt	tjj| |d	}|jdjdkszt	d S )
Nr   r   r&   rs   r   Fr]  Tr  )
r3   r4   rE   r=   ru   r/   rG   rH   r^  rK   rR   r=   rW   r   r   r$   /test_table_from_pandas_keeps_schema_nullability   s    r  c               	   C   s  t dddgdddgd} tdt fd	t fd
t fg}tjtdd tj	j
| |d W 5 Q R X d
| j_t| |d|d tjtdd tj	j
| |dd W 5 Q R X tjtdd tj	j
| |d d W 5 Q R X t jdddgd
d| _t| |d |d t| |d|d td
t fdt fd	t fg}t| |d |d t| |d|d tdt fd	t fg}|  }|jdd}t| |d ||d t| |d||d t jjdddgddgd| _tdt fdt fdt fd	t fg}t| |d|d t| |d |d tdt fdt fd	t fg}|  }|jddd}t| |d||d t| |d ||d d S )Nr   r&   rs   r&  r'  r(  r   r   r   r   zname 'index' present in ther  r  T)r=   r>   rT   z'preserve_index=False' wasFr  z=name 'index' is present in the schema, but it is a RangeIndexr   r\   r)  )r=   r>   rT   rS   )r   r   )r   r&   )r   r   Zlevel1Zlevel2r   )r3   r4   rE   r=   r/   r1   r   r   KeyErrorrG   rH   r   r]   rY   r   r   rk   r*  r   from_tuplesr4  )rR   r=   rS   r   r   r$   +test_table_from_pandas_schema_index_columns  s    







  






  r  c                  C   s   t dddgdddgd} tdt fd	t fd
t fg}tjj| dd}tjj| d|d}|j	 
|sztt jdddgdddgddddgd} tj| }tjj| |d}|j	 
|std S )Nr   r&   rs   r&  r'  r(  r   r   r   r   Tr   )r>   r=   r   r   r  )r3   r4   rE   r=   r/   r1   SchemarH   rG   r  rJ   rK   )rR   rT   r=   rW   r   r   r$   :test_table_from_pandas_schema_index_columns__unnamed_indexh  s    


&r  c                  C   sH   t  } tj| ddi}tjj| |d}|jj	ddksDt
d S )NmetaTruer  s   metas   True)r3   r4   rE   r  rH   r  rG   r=   r   getrK   r  r   r   r$   2test_table_from_pandas_schema_with_custom_metadata}  s    r  c               	   C   s*  t t jddddtjdd} ttjdt	 ddtjd	tj
d
ddddg}tjj| |d}|j|sxt|jjd d }|d dkst|d d kst|jjd d }|d d	kst|d ddikst| }ddi}tt jtdkrd|d	< | dd	g |}t|| d S )Nz2020-01-01T00:00:00ZHr&   )r   r   )r   r  r  Tr]  r   ra   r>  r?  Fr  r   r   r]   r   r   r   r0   r   zdatetime64[s, UTC])r3   r4   r   r5   r7   r8   rE   r=   ru   r0   r@  rG   rH   rJ   rK   r   rI   r   r  r   rP   rQ   )rR   r=   rW   Zmetadata_floatZmetadata_datetimerX   Zcoerce_cols_to_typesrS   r   r   r$   2test_table_from_pandas_schema_field_order_metadata  s,    
r  c                  C   s~   t tjdddddgddtjdddddgddtjdd	d
d ddgdddddgd} tj| }|	 }t
| | d S )Nr   r&   rs   r   r   r/   r   r*   r   r  r  rV  FT)r2  r3  c3c4Zc5)r3   r4   r5   r^   r7   r8   rE   rF   rH   rI   rP   rQ   )r   r   rX   r   r   r$   test_recordbatch_from_to_pandas  s    
r  c                  C   s   t tjdddgddtjdddgddd	d d
gddd gd} t tjddgddtjddgddd	d	gddgd}tj| }tj|}tj||g}|	 }t 
| |gjd	d}t|| d S )Nr   r&   r*   r   ry   rz   r{   r1   TFr   r  )r2  r3  r  r  rs   r   r|   r}   r  rV  r)  )r3   r4   r5   r^   rE   rF   rH   rG   r   rI   r  r*  rP   rQ   )r  r  Zbatch1Zbatch2rW   rX   r   r   r   r$   test_recordbatchlist_to_pandas  s$    r  c                  C   sp   t jt ddddggdgd} t jt ddddggdgd}| d  jdksVt|d  jdksltd S )Nr   r&   rs   r   Za0r   r   )rE   Zrecord_batchr^   rW   rI   r]   rK   )r7  r   r   r   r$   *test_recordbatch_table_pass_name_to_pandas  s      r  r[   rS   r  r2   r,   r-   r.   r/   r(   r)   r*   r+   rM  r0   r1   r   r  r   )lengthr   zlist[list[int16]]r  rs   r  r  r   r  r>  r   ra   r   c                 C   s   t | |kstd S r  )r   rK   r  r   r   r$   test_logical_type  s    r  c                  C   s   d} t jtj| tjdtjjdd| dtjd}t	
  t  }| }t  || d  ksdtd }t	
  t  |kstt tj| tjd}t  }| }t  |kstd S )N'  r   r   r&   r:   r  r   )rE   r^   r5   r6   r/   r7   rU  r   rf  r"  r#  r$  rI   rK   )rw  rd   prior_allocationr#   r   r   r$   test_array_uses_memory_pool  s    r  c                  C   sd   t t tjdtjdgdg} t| dd t  }|  }|d j	j
jsPtt  |ks`td S )Nr  r   r  Tsplit_blocks)rE   rW   r^   r5   r6   r/   !_check_to_pandas_memory_unchangedr$  rI   rl   flagsZ	writeablerK   )r   r  rX   r   r   r$   test_singleton_blocks_zero_copy  s    "r  c                 K   s(   t  }| jf |}t  |ks$td S r  )rE   r$  rI   rK   )r  kwargsr  r#   r   r   r$   r  &  s    r  c                  C   s   t t jdddddgddt jdddddgddt jdddddgd	dt jdddddgd
dt jdddddgddt jdddddgddt jdddddgddt jdddddgddgdd tdD } t| d t| dd d S )Nr   r&   rs   r   r   rm  r  ro  rp  rJ  rK  c                 S   s   g | ]}d  |qS zf{}formatru  r   r   r$   r%   9  s     z/test_to_pandas_split_blocks.<locals>.<listcomp>r   Tr  )rE   rW   r^   r9   _check_blocks_createdr  )r   r   r   r$   test_to_pandas_split_blocks.  s    	
r  c                 C   s"   t tjt dk r| jS | jS d S )Nz1.1.0)r   r3   r  _dataZ_mgrrY  r   r   r$   _get_mgr?  s    r  c                 C   s&   | j dd}tt|j|ks"td S )NTr  )rI   r   r  blocksrK   )r   numberr#   r   r   r$   r  F  s    r  c                     s:   d  fdd} |  }t |ddd |  }t |dd d S )N2   c                      s(   t dd t D dd t D S )Nc                 S   s(   g | ] }t tjd ddd qS )r  Nr&   )rE   r^   r5   r7   r8   ru  r   r   r$   r%   O  s   zEtest_to_pandas_self_destruct.<locals>._make_table.<locals>.<listcomp>c                 S   s   g | ]}d  |qS r  r  ru  r   r   r$   r%   S  s     )rE   rW   r9   r   rx  r   r$   _make_tableN  s
    
z1test_to_pandas_self_destruct.<locals>._make_tableT)r  self_destruct)r  )r  )r  r   r   r  r$   test_to_pandas_self_destructK  s    r  c                  C   s~   d} t tj| tjd}t |||gdddg}t  }| }t  |d|  d  ks^td }t	
  t  |ksztd S )Nr  r   r  r  rI  rs   r   )rE   r^   r5   r6   r/   rW   r$  rI   rK   r"  r#  )rw  rd   r   r  r#   r   r   r$   test_table_uses_memory_pool]  s    r  c                  C   sl   t ddig} |  }|jtdks,t|d }t|}t||ksPt~t||d kshtd S )Nr   r   r   r   )rE   r^   rI   r   r5   rK   sysgetrefcount)rd   r  r  refcountr   r   r$   test_object_leak_in_numpy_arraym  s    
r  c                  C   s   t ddig} t | gdg}| d }|jtdks@t|d }t|}t||ksdt~t||d ks|td S )Nr   r   r  r   r   )	rE   r^   rW   rI   r   r5   rK   r  r  )rd   rW   r  r  r  r   r   r$   test_object_leak_in_dataframey  s    
r  c                  C   s   ddgddgg} t  }t j| d |d}t j| t |d}tj| d tjd}ttj| d tjdtj| d tjdg}|j|kst|	t |st|	t |std S )	Nr  g333333@g      "@g      E@r   r  r   r   )
rE   r0   r^   r  r5   r3   r_   r[   rK   rJ   )r   r   r  r  r}  r~  r   r   r$   test_array_from_py_float32  s    r  c            
   	   C   s  t  } t| g}|jdjd}tjddd}tj	j
||d}|j|ksTttj	j
|tdd}|d  |d  kst|d  |d  ksttjdgddtd	}tjd
gddtd}|jtd	ksttd}tt || W 5 Q R X |j|dd}||s6tttdtdtdg}	tjdddgtdd}tt tj|	tdd W 5 Q R X tt tj	j
|	tdd W 5 Q R X tj	j
|	tddd}||sttj|	tddd}||std S )Nz	tzlocal()r   r  r?  r  r   i r/   r  {   ra   Fr   r   rT  r  r  )r   nowr3   r_   rA  rB  Z
tz_convertrE   r@  r  rH   r[   rK   rk  to_pydatetimer^   r  r   r   r   rJ   r;  )
r  ra   Zs_nycZ
us_with_tzrd   r  rS   targetrX   r  r   r   r$   test_cast_timestamp_unit  s6    
" r  c                  C   sf   t j } |  }tj|gtjdddd}tj||gddg}|	 }t|}|
|sbtd S )Nr  r   r?  r  r   stop)r3   r;  r  r  rE   r^   r@  r  r   rI   rJ   rK   )tsts_dtrd   r  rX   r   r   r   r$   (test_nested_with_timestamp_tz_round_trip  s    

r  c            
      C   s  t j } |  }dD ]v}|dkr.dd }ndd }tj| gt|d}tj| gtj|ddd}tj||gd	d
g}tj||gd	d
g}|	 }t
|d d	 tst|d d	 jd kstt
|d d
 tst|d d
 jd kst|	 }t
|d d	 tst|d d	 jd k	s*t|d d	 tj}	||	jd d||ks\tt
|d d
 tstt|d d
 jd k	sttd|i	 }t
|d d d	 tst|d d d	 jd kstt
|d d d
 tst|d d d
 jd ksttd|i	 }t
|d d d	 ts<t|d d d	 jd k	sXtt
|d d d
 tstt|d d d
 jd k	stqd S )Nr1  )ra   r  c                 S   s   | j ddS )Nr   r  r'  r#   r   r   r$   truncate  rn  z/test_nested_with_timestamp_tz.<locals>.truncatec                 S   s   | S r  r   r  r   r   r$   r    rn  r  r   r?  r   r  r   r  r   )r3   r;  r  r  rE   r^   r@  r  r   rI   r   r   rK   r  
astimezoner   r  r!  rW   )
r  r  r  r  rd   r  Zarr3Zarr4rX   Zutc_dtr   r   r$   test_nested_with_timestamp_tz  s@    


r  c               
   C   sB  t dddgd} t jdddgtd}t ddd	dddg}d
D ]}| |}tj||}tjj|||d}|d dkrt	t
 |  W 5 Q R X q@| }tjj||d}tt|t| | }	|	  dkst|}
|d dkr
||dd  }
tjjt |d|
|d}tt|	t| q@d S )Nr   r   r&   r   r  r  r   FT)r(   r,   r)   r-   r*   r.   r+   r/   r  ur  r   )r5   r  r^   r   r   rE   r   r   r   r   r  rI   r3   r{  
from_codesrP   r`   r_   rh   rj   rK   where)Zsrc_indicesro  rf   
index_typer   d1d2Zpandas1Z
ex_pandas1Zpandas2Zsigned_indicesZ
ex_pandas2r   r   r$   test_dictionary_with_pandas  s2    
r  c                    sd   |d k	r$|t jjdt|| d }n t j fddt| D td}|dkr`d |t j| |k < |S )Nr   r:   c                    s   g | ]}t  qS r   rt  ru  	item_sizer   r$   r%   7  s     z"random_strings.<locals>.<listcomp>r   )r5   r7   rU  r   r^   r9   r   rand)r  r  Zpct_nullro  rX   r   r  r$   random_strings3  s    r  c                  C   s  t jd tjtdddd} tjtdddd}tjtdddd}tjt jjd	t	| d
dd| }tjt jjd	t	|d
dd|}tjt jjd	t	|d
dd|}tjt jjd	t	|d
ddt j
d
dk d}tj||}t| ||g}t||||g}	t|d|d|d|dg}
|	 }|
 }|jj| k s`t|d}d || < t|| d S )Ni90  r       r4  r  r  r  rT  r   r  ro  r  r&  r  r    )r5   r7   r  rE   r^   r  r   r   rU  r   r  Zconcat_arraysr  r  rI   rm  r  r,  rK   r   rh   rP   r`   )r  r  Zd3r  r  r$  ro  r%  expected_dictr   Za_denserX   Zresult_denseZexpected_denser   r   r$   "test_variable_dictionary_to_pandas@  sD     
r   c                  C   sp   t dddddg } t jdddg| }| }ttjdddgt	dtjddgt	dg}t
|| d S )Nr   r   r   rs   r   r   )rE   r^   r  r  r   rI   r3   r_   r5   r   rP   r`   )r  rd   rX   rS   r   r   r$   (test_dictionary_encoded_nested_to_pandasl  s    r  c                  C   s   t dddg} tt t }t| }| dddgksDt|j	
|sTtt ddd dg} t| }| ddd dgkst|j	
|sttj| tddddgd}| ddd d gkst|j	
|std S )Nr   r   FTr  )r3   r{  rE   ro  r,   r4  r^   ra  rK   r[   rJ   r5   )rm  r  rX   r   r   r$   test_dictionary_from_pandasy  s    

r  c               	   C   sZ  t jtjddgddtjddgtd} tjt t	 d}tj| |d}|j
|s^t| ddgksrttjt t d}ttj tj| |d}W 5 Q R X tjt t	 d	d
}d}tjt|d tj| |d}W 5 Q R X | ddgksttjt t	 d}tj| |tdd	gd}|j
|sJt| dd gks`tt g } tjt t	 d}tj| |d}|j
|st| g ksttjt t d}tj| |d}|j
|st| g kstt ddg} tj| t	 d}tjddgt	 d}||s@t| ddgksVtd S )Nr   r   r,   r   r   r   )r  
value_typer  T)r  r  r   z4The 'ordered' flag of the passed categorical values r  Fr  )r3   r{  r  r5   r^   r   rE   ro  r-   r4  r[   rJ   rK   ra  r,   r/   r   r   rk  r   )rm  typrX   r3  rS   r   r   r$   *test_dictionary_from_pandas_specified_type  sL       
r  c                  C   s   t tjt dk rtd tjddddgidd} | d} t| d j	j
}t| d j	jj}t|tjstttj| d }tj||}||kstd S )	Nz1.3.0z:PyArrow backed string data type introduced in pandas 1.3.0r#   r   r  zstring[pyarrow]r   r   )r   r3   r  r   r  r4   r   rE   r^   rm  codesr  rl   r   r  rK   rH   r   r   )rR   r   ro  rS   rX   r   r   r$   :test_convert_categories_to_array_with_string_pyarrow_dtype  s    

r  c                  C   s2  t dt jddd gddi} t| }tddd gt }|d d|sVt	t
dt fg}tj| |d}tddd gt }|d d|st	t| d }||st	tj| d t d}||st	t| d j}||st	tj| d jt d}||s.t	d S )	Nr   r   r&   Int64r   r   r  r  )r3   r4   r_   rE   rW   r^   r/   r  rJ   rK   r=   r1   rl   )rR   rX   rS   r=   r~  r   r   r$   test_array_protocol  s     
r	  c                   @   s   e Zd Zdd Zdd ZdS )DummyExtensionTypec                 C   s   t j| t   d S r  rE   PyExtensionType__init__r/   r  r   r   r$   r    s    zDummyExtensionType.__init__c                 C   s   t dfS Nr   )r
  r  r   r   r$   
__reduce__  s    zDummyExtensionType.__reduce__N)rD  rE  rF  r  r  r   r   r   r$   r
    s   r
  c                 C   s(   t jdddgt  d}t jt |S )Nr   r&   rs   r  )rE   r^   r/   ExtensionArrayfrom_storager
  )rw   r[   storager   r   r$   PandasArray__arrow_array__  s    r  c                 C   s   t jdddgt  d}t jt |}| jtjj	dt
dd | jtjjdt
dd tjddd	d
jtddjfD ]}t |}||stt t|}||stt t|}||stt td|idd}||sttqtd S )Nr   r&   rs   r  Z__arrow_array__F)Zraising
2012-01-01Dr  r   r   r   )rE   r^   r/   r  r  r
  setattrr3   r6  ZPeriodArrayr  ZIntervalArrayr  r  rJ   rK   r_   r   rW   r4   r2  r  )monkeypatchr  rS   rd   rX   r   r   r$   *test_array_protocol_pandas_extension_types  s*      
 r  c                 C   s   t |tjr|}n
|d}| }tj|d dd|j|jt|  }|d }|d k	rtj	
t t|d |g}t|}ntjt|td}tjj| | dd}|S )Nr   r   r/   r   F)rk   )r   rE   r  r  buffersr5   Z
frombufferoffsetr   ZBooleanArrayZfrom_buffersrf  r=  r  r2   r3   r6  ZIntegerArrayrk   )rw   r^   rd   Zbuflistr   Zbitmaskrf   Zint_arrr   r   r$   _Int64Dtype__from_arrow__  s&    
   r  c                 C   s  dd l m  m} tdddgtjdddgddddd	gd
}t|}| }t	t
|jd |jrltt
|jd jjtdkstt	t
|jd |jstt|| tdtjddd gddi}t|}| }t	t
|jd |jstt|| ttjtdk r2| tjjjjd n| tjjjjd | }tt
|jdksftt	t
|jd |jrtd S )Nr   r   r&   rs   r   r  r   r   r   r   r   r   r/   r   	1.3.0.dev__from_arrow__)pandas.core.internalscore	internalsr3   r4   r^   rE   rW   rI   r   r  r  ExtensionBlockrK   rl   r   r5   rP   rQ   r   r  delattrr6  integer_IntegerDtypeNumericDtyper   )r  _intrR   rW   rX   r   r  r   r   r$   test_convert_to_extension_array4  s:    
 

 
 r(  c                   @   s$   e Zd Zdd Zdd Zdd ZdS )MyCustomIntegerTypec                 C   s   t j| t   d S r  r  r  r   r   r$   r  \  s    zMyCustomIntegerType.__init__c                 C   s   t dfS r  )r)  r  r   r   r$   r  _  s    zMyCustomIntegerType.__reduce__c                 C   s   t  S r  )r3   
Int64Dtyper  r   r   r$   r  b  s    z#MyCustomIntegerType.to_pandas_dtypeN)rD  rE  rF  r  r  r  r   r   r   r$   r)  Z  s   r)  c              	   C   s|  dd l m  m} tddddgt }tjt |}t	d|i}|
 }tt|jd |jsjttjddddgdd}t|| |
 }tt|jd |jsttdtjddddgddi}t|| ttjtd	k r| tjjjjd
 n| tjjjjd
 |
 }tt|jd |jr<ttddddg}t|| tt |
  W 5 Q R X d S )Nr   r   r&   rs   r   r   r  r   r  r  ) r  r   r!  rE   r^   r/   r  r  r)  rW   rI   r   r  r  r"  rK   r3   r_   rP   r`   r4   rQ   r   r  r#  r6  r$  r%  r&  r   r   r   )r  r'  r  rd   rW   rX   rS   r   r   r$   /test_conversion_extensiontype_to_extensionarrayf  s6     
 
 r+  c               	   C   s>  t dt dddgt  i} |  }|d jtdks@t| jt  t	 ij
d}t|d jtj	sptt dt dddgt  i} | jt  t	 ij
d}|d jtdkstt tdtjdddgdd	i} |  }t|d jtj	st| jt  td
ij
d}t|d jtjs:td S )Nr   r   r&   rs   r/   types_mapperr.   r  r   r  )rE   rW   r^   r/   rI   r   r5   rK   r3   r*  r  r   r.   r4   ZPeriodDtype)rW   rX   r   r   r$   'test_to_pandas_extension_dtypes_mapping  s      $r.  c                  C   s   t tjt dk rtd tjddddjtddjfD ]X} t| 	 }t
| }t|| td	| id	 	 }tj
| d	d
}t|| q<d S )Nz1.1z'ExtensionDtype to_pandas method missingr  rs   r  r  r   r   r  r\   )r   r3   r  r   r  r  r^   r  rE   rI   r_   rP   r`   rW   )rd   rX   rS   r   r   r$   test_array_to_pandas  s    

r/  c                  C   sT   t jt jdddd} t| }| jt jdddddddddgdd	k d S )
Nr   rs   )r   endr   r   )r  r  r&   r   r   )r3   r4   r  rE   rW   rI   r   r   )rR   rW   r   r   r$   5test_roundtrip_empty_table_with_extension_dtype_index  s    
r1  r   r   r   c                 C   s~   t tjt dk rtd tjddgddgddgd	tt d
	| }|
 }t|}|jtjd}t|| d S )Nz1.5.0zArrowDtype missingr   r&   rs   r   r   r   r  r   r,  )r   r3   r  r   r  r4   Z
ArrowDtyperE   r/   r   rk   rW   rI   rP   rQ   )r   rR   rS   rW   rX   r   r   r$   !test_to_pandas_types_mapper_index  s    

r2  c                  C   s   t tjt dk rtd tdddgt } t t ij	}| j
|d}|jt ksbtt d ij	}| j
|d}|jtdkstt t ij	}| j
|d}|jtdkstd S Nz1.2.0z$Float64Dtype extension dtype missingr   r&   rs   r,  r/   )r   r3   r  r   r  rE   r^   r/   r*  r  rI   r   rK   r5   r1   Float64Dtyper   r-  rX   r   r   r$   !test_array_to_pandas_types_mapper  s    
r6  c                  C   s   t tjt dk rtd ttdddgt g} t	| tj
sJtt t ij}| j|d}|jt ksztt d ij}| j|d}|jtdkstt t ij}| j|d}|jtdkstd S r3  )r   r3   r  r   r  rE   r  r^   r/   r   r`  rK   r*  r  rI   r   r5   r1   r4  r5  r   r   r$   )test_chunked_array_to_pandas_types_mapper  s    
r7  c                  C   s  ddd dg} ddddg}t j| dd}t j|dd}t jdd	d
dgdd}d}d}tjd| itjddd	ddd}t jj||gddgd}|dt	dgd d ddddidgddddd dddddd dgddi}|
 }	t|	| tjd| itjddd	ddd}
t jj||gd|gd}|dt	|gd d ddddidgddddd dd|ddd dgddi}|
 }t||
 tjd| itjddd	d dd}t jj||gd|gd}|dt	|gd d ddddidgddddd dd |ddd dgddi}|
 }t|| tjd| itjddd	dd|gd}t jj|||gdd|gd}|dt	d|gd d ddddidgddddd dddddd dd |ddd dgddi}|
 }t|| tjd| itjddd	d d|gd}t jj|||gd||gd}|dt	||gd d ddddidgddddd dd |ddd dd |ddd dgddi}|
 }t|| d S )Nr   r  r  r   r   r  r  r   r&   r   r   r/   r   Z__index_level_1__r   rV  r   r   r      pandasr   r   r   r   )r]   r   r   r   r   z0.23.4)r   r   r   r:  )rE   r^   r3   r4   r   rG   r   replace_schema_metadatajsondumpsrI   rP   rQ   )a_valuesb_valuesa_arrowb_arrowZrng_index_arrowZ
gen_name_0Z
gen_name_1e1r  r1e2r  r2Ze3t3Zr3Ze4Zt4Zr4Ze5Zt5Zr5r   r   r$   )test_metadata_compat_range_index_pre_0_12  s     
  
  
    rE  c                  C   s   ddddg} ddddg}t j| d	d
}t j|dd
}tj| |dtjdddddd}t ||d}|dtd d d dddgd dd	d	dd ddddgddddddgddi}|	 }t
|| d S )Nr   r&   rs   r   r   r   r   r  r/   r  r  r   r   r   rV  r   r   r8  r   zmixed-integer)r   r   r]   r   r   )r   r]   r   r   r   r9   )kindr]   r   r   r  z0.25.0)r   r   r   r:  )rE   r^   r3   r4   r   rW   r9  r:  r;  rI   rP   rQ   )r<  r=  r>  r?  rS   rW   rX   r   r   r$   'test_metadata_compat_missing_field_name  sX     rG  c                  C   sL   t d} ttjtjdd| dd}|jj}|d d d dksHt	d S )	Nr   r   r   r\   r   r   r]   6)
r5   r/   rE   rW   r3   r4   r   r=   r   rK   r]   rW   r   r   r   r$   .test_metadata_index_name_not_json_serializable  s    
rJ  c                  C   sF   d} t tjtjdd| dd}|jj}|d d d dksBtd S )Nr   r   r   r\   r   r   r]   )rE   rW   r3   r4   r   r=   r   rK   rI  r   r   r$   -test_metadata_index_name_is_json_serializable  s    rK  c               
   C   s   t tddtddtddtddgtddtddtddtddgd	} | jj| jjftd
tdfkst| S )Nz0001-01-01 00:00r  z2012-05-02 12:35z2012-05-03 15:42z3000-05-03 15:42z1991-01-01 00:00r=  z2050-05-03 15:42)
dateTimeMs
dateTimeNsr  zM8[ns])r3   r4   r5   r  rL  r   rM  rK   rY  r   r   r$   make_df_with_timestamps  s"    







 
rN  z)ignore:Parquet format '2.0':FutureWarningc                 C   sP   t  }tj|}| d }tj||dd t|}|jdd}t	|| d S )Nztimestamps_from_pandas.parquetz2.0)versionTZtimestamp_as_object)
rN  rE   rG   rH   pqZwrite_tableZ
read_tablerI   rP   rQ   )tempdirrR   rW   filenamerX   r   r   r   r$    test_timestamp_as_object_parquet  s    
rT  c                  C   s.   t  } tj| }|jdd}t| | d S )NTrP  )rN  rE   rG   rH   rI   rP   rQ   )rR   rW   r   r   r   r$   %test_timestamp_as_object_out_of_range  s    rU  
resolutionr   r   rA  i  i  c                 C   s   t j|gt j| |dd}t d|i}|jdd|jddd fD ]r}|jtksVtt|d t	sht|r|d j
d k	s~t|d j
|}n|d j
d kst|}|d |ksDtqDd S )Nr?  r  r   TrP  r   )rE   r^   r@  rW   rI   r   r   rK   r   r   r  fromutc)rV  r   rA  rd   rW   rX   rS   r   r   r$   'test_timestamp_as_object_non_nanosecond  s    
rX  c               	   C   sl   t d} dd l}| d}||ddddd}td	t|gi}|jd
d}t||ksht	d S )Nr  r   x   i  r   r  r  9   Ztimestamp_colTrP  )
r   r  r   r  ZlocalizerE   rW   r^   rI   rK   )r  r   r   rA  rW   rX   r   r   r$   %test_timestamp_as_object_fixed_offset7  s    

r[  c                   C   s   t d d S )Nzpandas_threaded_import.py)r
   r   r   r   r$   test_threaded_pandas_importC  s    r\  c                  C   s^   t d} t t j| ksttdt  gi}tj|}|	  t t j| ksZtd S )NZm8r   )
r5   r   r=  rK   r3   r4   rE   rG   rH   rI   )rS   rR   r   r   r   r$   $test_does_not_mutate_timedelta_dtypeG  s    
r]  c                  C   sf   ddl m}  d| dddig}| ddd	d
g}t||d}| }|d d  |d ksbtd S )Nr   )r   timedelta_1r  r   )r  rG  rs   (   r  )r  r  r  )r^  timedelta_2r`  )r   r   rE   rW   rI   Zto_pytimedeltarK   )r   r^  r`  rW   rR   r   r   r$   %test_does_not_mutate_timedelta_nestedS  s    ra  c                  C   sT  t t dt t t  t t dt  gg} t jdggd dddifgddd ifdddifgggg| d	}t	
ddggd dddifgddd ifdddifgggi}t	
dd
d igd dddiidd iddidggi}| }|jdd}t|| t|| t jj|| d	}||s0tt jj|| d	}||sPtd S )Nr   r   )r   Nr&   s   abcrs   r   s   defr  r   r<  strictZmaps_as_pydicts)rE   r=   ru   r  r  r,   r  rZ  rW   r3   r4   rI   rP   rQ   rG   rH   rJ   rK   )r=   rW   Zexpected_default_dfZexpected_as_pydicts_dfZ
default_dfZas_pydicts_dfZtable_default_roundtripZtable_as_pydicts_roundtripr   r   r$   ,test_roundtrip_nested_map_table_with_pydicts`  sJ    "rd  c               	      s  t ddg} t ddddg}t g t  }t ddgd	d
ggt t  }t g dddgddggt t  }t g t t  }t jddg| |}t jdddg||}t jddg||}t t jddg|dt jddg|ddt jddg|dg}	|		 }
t
dddgfdd	d
gfggdddgfggg g}|	j	dd}t
ddgd	d
gdgdddgigg g}|	dd}|	 }t
dddgfggg g}|j	dd}t
dddgigg g}t D tddt t|
| t|| t|| t|| W 5 Q R X t t t  t t   t
jdd fdd}||
|	 |||	 ||| ||| dS )z=
    Slightly more robust test with chunking and slicing
    r   r  r  rV  r  r  r   r   r   r  Nr  r  r  r   r&   rs   r   r   rb  rc  r   r  rA   rB   )r  returnc                    s(   t t jj|  d}||s$td S r  )rE   r  r  rH   rJ   rK   )r  r   Zarray_roundtripr  r   r$   assert_roundtrip  s    zMtest_roundtrip_nested_map_array_with_pydicts_sliced.<locals>.assert_roundtrip)rE   r^   r4  r  r  r   r  r  r  rI   r3   r_   rL   rM   rN   rO   rP   r`   r  )Zkeys_1Zkeys_2Zkeys_3Zitems_1Zitems_2Zitems_3Zmap_chunk_1Zmap_chunk_2Zmap_chunk_3r  series_defaultexpected_series_defaultseries_pydictsexpected_series_pydictsZslicedZseries_default_slicedZexpected_series_default_slicedZseries_pydicts_slicedZexpected_series_pydicts_slicedrh  r   rg  r$   3test_roundtrip_nested_map_array_with_pydicts_sliced  sx    
 


rm  c               	   C   s:  t dddg} t ddgddgddggt t  }d	d
g}t j|| |}t t  t t  }tt j	j
 |jdd W 5 Q R X |jdd}tddgddgdg}|t jj||drt| }tdddgfdddgfdddgfgg}|t jj||dstt|t|ks4tt||D ]n\}	}
t|	t|
ks\tt|	 |
 D ]:\}}|d	 |d	 kstt|d |d sntqnq>t|t|kstt||D ]f\}	}
t|	t|
kstt|	|
D ]:\}}|d	 |d	 kstt|d |d stqqd S )Nr   r  r   r   r   r  r   r   r   rs   rb  rc  lossyre  r  r   )rE   r^   r  r4  r  r   r  r   r   r  ZArrowExceptionrI   r3   r_   rJ   r  rH   rK   r   rX  r  r5   array_equal)r5  r  r  mapsr  rk  rl  ri  rj  row1row2tup1tup2r   r   r$   4test_roundtrip_map_array_with_pydicts_duplicate_keys  s@     "ru  c            
   	   C   sZ  t ddgddgg dgd dgddggt t  } t d	d
ddddg}dddg}t j|| |}tt |j	dd W 5 Q R X |	 }t
ddgd	fddgd
fgg dfdgdfd dgdfddgdfgg}t|t|kstt||D ]d\}}t|t|kstt||D ]:\}}	t|d |	d s:t|d |	d kstqqd S )Nr   r   r   r  r  r  r  rL  r   r  r  rV  r  r  r   r&   r   rn  rc  r   )rE   r^   r  r4  r  r   r   r   r  rI   r3   r_   r   rK   rX  r5   ro  )
r5  r  r  rp  r  rj  rq  rr  rs  rt  r   r   r$   %test_unhashable_map_keys_with_pydicts  s(     
$rv  c                  C   s^   t jt jdddddd} tdt| i}|d}| }|jdksNt	t
||  d S )N2012r&   r$  r%  Zdatetime_columnr\   )r3   r_   r   rE   rW   r^   r2  rI   r]   rK   rP   r`   )r  rW   Z	table_colrX   r   r   r$   )test_table_column_conversion_for_datetime/  s    
rx  c                  C   s8   t t jdddd} t| }| }t||  d S )Nrw  r&   r$  r%  )r3   r_   r   rE   r^   rI   rP   r`   )r  rd   rX   r   r   r$   "test_array_conversion_for_datetime>  s    
ry  )r   )NFNTNFF)NN)NNN)N)r   N)N)r"  r  r:  multiprocessingrc  r  rL   collectionsr   r   r   r   r   r   Z
hypothesisrL  Zhypothesis.strategies
strategiesrN  numpyr5   Znumpy.testingr  ro   r   Zpyarrow.pandas_compatr   r	   Zpyarrow.tests.utilr
   r   r   Zpyarrow.tests.strategiestestsrP  Zpyarrow.vendored.versionr   ZpyarrowrE   r   rQ  ImportErrorZpandasr3   Zpandas.testingrP   Zpandas_examplesr   r   rJ  Z
pytestmarkr<   rY   re   rm   rq   rr   rH  rK  r  r  r  rR  r  r  r  rB  rZ  r[  r\  r  r  r  r  r  r  rM  r6  Zpandas_compatible_typesr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rf  r,   r-   r.   r/   r(   r)   r*   r+   rM  r0   r1   r  r  rZ  r4  r  r  r@  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r	  r  r
  r  r  r  r(  r)  r+  r.  r/  r1  r2  r6  r7  rE  rG  rJ  rK  rN  rN   rT  rU  rX  r[  r\  r]  ra  rd  rm  ru  rv  rx  ry  r   r   r   r$   <module>   s        
 
   w  '

      X[    " a<  	



Z%

















14$,5	&&	

 *3
	 -M4
!