U
    -e1                     @   s   d dl mZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Zd dlZd dlZd dlmZmZmZ dd Zdd
dZdd Zdd Zdd ZG dd dZG dd dee	jZG dd dee	jZdS )    )OrderedDict)DecimalN)	read_jsonReadOptionsParseOptionsc                  c   s2   t j} | E d H  | D ]}| D ]}|| V  qqd S N)stringascii_lowercase)lettersfirstsecond r   X/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/test_json.pygenerate_col_names"   s
    
r      
   
c                 C   s   t jdjdd| |fd}ttt | }g }|jD ],}t	dd t
||D }|t| q8|| }dd |D }	tj|	|}
||
fS )N*   r   i  )sizec                 S   s   g | ]\}}|t |fqS r   )int).0kvr   r   r   
<listcomp>0   s     z$make_random_json.<locals>.<listcomp>c                 S   s   g | ]}t j|t  d qS ))type)paarrayint64)r   colr   r   r   r   3   s     )nprandomZRandomStaterandintlist	itertoolsislicer   Tr   zipappendjsondumpsjoinencoder   TableZfrom_arrays)num_colsnum_rowslinesepZarrZ	col_nameslinesrowZjson_objdatacolumnsexpectedr   r   r   make_random_json+   s    
r5   c                 K   sH   | f |}t t j|t jd}| D ]\}}t|||ks(tq(d S )N)protocol)pickleloadsr)   HIGHEST_PROTOCOLitemsgetattrAssertionError)clsZattr_valuesoptsZnew_optsnamevaluer   r   r   check_options_class_pickling8   s    

rA   c                  C   s   t } |  }|jdkstd|_|jdks,t|jdks:td|_|jdksNt| ddd}|jdksht|jdksvtt| ddd d S )Nr   i90  TFi  )
block_sizeuse_threads)r   rB   r<   rC   rA   )r=   r>   r   r   r   test_read_options@   s    rD   c               	   C   s   t } |  }|jdkst|jd ks&td|_|jdks:tttdt g}||_|j|ksft|jdksttdD ]}||_|j|ksxtqxt	
t d|_W 5 Q R X t| |ddd d S )	NFTfooinfer)ignoreerrorrF   zinvalid-valuerG   )explicit_schemanewlines_in_valuesunexpected_field_behavior)r   rJ   r<   rI   r   schemafieldZint32rK   pytestraises
ValueErrorrA   )r=   r>   rL   r@   r   r   r   test_parse_optionsT   s&    rQ   c                   @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd ZdS ) BaseTestJSONReadc                 K   s   | j t|f|S r   )r   r   Z	py_buffer)selfbkwargsr   r   r   
read_bytesp   s    zBaseTestJSONRead.read_bytesc                 C   s.   |j t|kstdd |jD |ks*td S )Nc                 S   s   g | ]
}|j qS r   )r?   )r   cr   r   r   r   u   s     z0BaseTestJSONRead.check_names.<locals>.<listcomp>)num_columnslenr<   r3   )rS   tablenamesr   r   r   check_namess   s    zBaseTestJSONRead.check_namesc              	   C   sh   d}dgdgd}t |}| |}| |ks6tt | }tt	 | | W 5 Q R X d S )Ns   {"a": 1, "b": 2}
   r   )arT   )
ioBytesIOr   	to_pydictr<   StringIOdecoderN   rO   	TypeError)rS   r2   Zexpected_dataZbiorZ   sior   r   r   test_file_objectw   s    

z!BaseTestJSONRead.test_file_objectc                 C   s   d}t  }t }||d fD ]}dD ]z}||_d|_tjtdd | j|||d W 5 Q R X tdd	D ]4}||_| j|||d}|	 d
dddgiksht
qhq$qd S )Ns   {"a": 1}
{"a": 2}
{"a": 3}   
FT   ztry to increase block sizematchread_optionsparse_options	      r^   r]   r      )r   r   rJ   rB   rN   rO   rP   rV   rangera   r<   )rS   rowsrm   rn   r2   rJ   rB   rZ   r   r   r   test_block_sizes   s&    z!BaseTestJSONRead.test_block_sizesc                 C   s6   d}|  |}| ddgddgddgdks2td S )	Ns/   {"a": 1,"b": 2, "c": 3}
{"a": 4,"b": 5, "c": 6}r]   ri   r      rq      r^   rT   rW   )rV   ra   r<   )rS   rs   rZ   r   r   r   test_no_newline_at_end   s    
z'BaseTestJSONRead.test_no_newline_at_endc                 C   sl   d}|  |}tdt fdt fdt fg}|j|ksDt| ddgddgd	d
gdkshtd S )Ns0   {"a": 1,"b": 2, "c": 3}
{"a": 4,"b": 5, "c": 6}
r^   rT   rW   r]   ri   r   ru   rq   rv   rw   )rV   r   rL   r   r<   ra   rS   rs   rZ   rL   r   r   r   test_simple_ints   s    


z!BaseTestJSONRead.test_simple_intsc                 C   s|   d}|  |}tdt fdt fdt fdt fg}|j|ksNt| ddgdd	gd
dgddgdksxtd S )NsQ   {"a": 1,"b": 2, "c": "3", "d": false}
{"a": 4.0, "b": -5, "c": "foo", "d": true}
r^   rT   rW   d      ?g      @r   3rE   FT)r^   rT   rW   r{   )	rV   r   rL   float64r   r   bool_r<   ra   ry   r   r   r   test_simple_varied   s    



z#BaseTestJSONRead.test_simple_variedc              	   C   s   d}|  |}tdt fdt fdt fdt fdt fg}|j|ksXt|	 dd dgd	d
d gd ddgd d d gd ddgdkstd S )Ns   {"a": 1, "b": 2, "c": null, "d": null, "e": null}
{"a": null, "b": -5, "c": "foo", "d": null, "e": true}
{"a": 4.5, "b": null, "c": "nan", "d": null,"e": false}
r^   rT   rW   r{   er|   g      @r   r}   rE   nanTF)r^   rT   rW   r{   r   )
rV   r   rL   r   r   r   nullr   r<   ra   ry   r   r   r   test_simple_nulls   s     




z"BaseTestJSONRead.test_simple_nullsc                 C   sP   d}|  |}tdtt fg}|j|ks6t| dg giksLtd S )N	   {"a": []}r^   )rV   r   rL   list_r   r<   ra   ry   r   r   r   test_empty_lists   s
    
z!BaseTestJSONRead.test_empty_listsc                 C   sF   d}|  |}tg }|j|ks&t|jdks4t|jdksBtd S )Ns   {}
{}
r   r   )rV   r   rL   r<   rX   r.   ry   r   r   r   test_empty_rows   s    

z BaseTestJSONRead.test_empty_rowsc              
   C   s   d}t t|d}dd dgfdd g gfdd g dgggfdd i gfd	d d
d id
ddiigffD ]F\}}| j|| |d}d|i}| |kst|djdks\tq\d S )Ns"   {                               }
)rB   s   {"a": 0}r   r   s   {"a": []}
{"a": [[1]]}r]   s	   {"a": {}}s    {"a": {}}
{"a": {"b": {"c": 1}}}rT   rW   )rm   r^   )r   rY   rV   ra   r<   columnZ
num_chunks)rS   	first_rowrm   Z	next_rowsZexpected_pylistrZ   r4   r   r   r   test_reconcile_accross_blocks   s     



z.BaseTestJSONRead.test_reconcile_accross_blocksc                 C   s   d}dt dt dt dd gi}tjtjfD ]P}td|ddfg}t|d}| j||d	}|j|kslt| |ks,tq,d S )
Ns'   {"a": 1}
{"a": 1.45}
{"a": -23.456}
{}
r^   1z1.45z-23.456ro   ri   rI   rn   )	r   r   Z
decimal128Z
decimal256rL   r   rV   r<   ra   )rS   rs   r4   Ztype_factoryrL   r>   rZ   r   r   r   test_explicit_schema_decimal   s     
z-BaseTestJSONRead.test_explicit_schema_decimalc              	   C   s   d}t dt  fg}t|d}| j||d}|jt dt  fdt  fgksXt| ddgdd	gd
ksvtt|dd}| j||d}|jt dt  fgkst| dddgikstt|dd}tj	t j
dd | j||d W 5 Q R X d S )Ns2   {"foo": "bar", "num": 0}
{"foo": "baz", "num": 1}
rE   r   r   nums   bars   bazr   r]   )rE   r   rG   )rI   rK   rH   z"JSON parse error: unexpected fieldrj   )r   rL   binaryr   rV   r   r<   ra   rN   rO   ZArrowInvalid)rS   rs   rL   r>   rZ   r   r   r   .test_explicit_schema_with_unexpected_behaviour  s@    




 
z?BaseTestJSONRead.test_explicit_schema_with_unexpected_behaviourc                 C   sP   t ddd\}}| |}|j|jks*t||s8t| | ksLtd S )Nr   r   r-   r.   )r5   rV   rL   r<   equalsra   )rS   r2   r4   rZ   r   r   r   test_small_random_json+  s
    
z'BaseTestJSONRead.test_small_random_jsonc           	      C   s   t ddd\}}t }t }||dfD ]b}dD ]X}||_dD ]H}||_| j|||d}|j|jksjt|	|s@|
 |
 ks@tq@q2q*d S )Nr   d   r   s   
rh   )      %   rl   )r5   r   r   rstriprJ   rB   rV   rL   r<   r   ra   )	rS   Z	data_baser4   rm   rn   r2   rJ   rB   rZ   r   r   r   test_stress_block_sizes2  s    
z(BaseTestJSONRead.test_stress_block_sizesN)__name__
__module____qualname__rV   r\   rf   rt   rx   rz   r   r   r   r   r   r   r   r   r   r   r   r   r   rR   n   s   	%rR   c                   @   s   e Zd Zdd ZdS )TestSerialJSONReadc                 O   s.   | dt }d|_t||}|jdd |S )Nrm   FTfull
setdefaultr   rC   r   validaterS   argsrU   rm   rZ   r   r   r   r   G  s
    
zTestSerialJSONRead.read_jsonNr   r   r   r   r   r   r   r   r   E  s   r   c                   @   s   e Zd Zdd ZdS )TestParallelJSONReadc                 O   s.   | dt }d|_t||}|jdd |S )Nrm   Tr   r   r   r   r   r   r   Q  s
    
zTestParallelJSONRead.read_jsonNr   r   r   r   r   r   O  s   r   )r   r   r   )collectionsr   decimalr   r_   r#   r(   r7   r   Zunittestnumpyr   rN   Zpyarrowr   Zpyarrow.jsonr   r   r   r   r5   rA   rD   rQ   rR   ZTestCaser   r   r   r   r   r   <module>   s(   	
 X
