U
    -e#                    @   sR  d dl Z d dlZd dlmZmZ d dlmZ d dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& dd Z'dBddZ(dd Z)dd Z*dd Z+G dd dZ,dd Z-dd Z.dd Z/dd Z0G dd de j1Z2G d d! d!e2Z3G d"d# d#e3Z4G d$d% d%e3Z5G d&d' d'e2Z6G d(d) d)e6ej7Z8G d*d+ d+e6ej7Z9G d,d- d-Z:G d.d/ d/e:ej7Z;G d0d1 d1e:ej7Z<d2d3 Z=d4d5 Z>d6d7 Z?d8d9 Z@ejABd:d;d< d=d< d>d< d?d< fd@dA ZCdS )C    N)datedatetime)Decimal)
open_csvread_csvReadOptionsParseOptionsConvertOptionsISO8601	write_csvWriteOptions	CSVWriter
InvalidRow)utilc                  c   s2   t j} | E d H  | D ]}| D ]}|| V  qqd S N)stringascii_lowercase)lettersfirstsecond r   W/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/test_csv.pygenerate_col_names1   s
    
r      
   
Tc           
      C   s   t jdjdd| |fd}t }ttt	 | }|rT|
d| |
| |jD ]$}|
dtt| |
| qZ|  }dd |D }tj||}	||	fS )N*   r     )size,c                 S   s   g | ]}t j|t  d qS ))type)paarrayint64).0ar   r   r   
<listcomp>E   s     z#make_random_csv.<locals>.<listcomp>)nprandomZRandomStaterandintioStringIOlist	itertoolsislicer   writejoinTmapstrgetvalueencoder!   Tablefrom_arrays)
num_colsnum_rowslinesepwrite_namesZarrcsv	col_namesrowcolumnsexpectedr   r   r   make_random_csv:   s    

rA   c                 C   s.   t  }|d|  |d |  S )Nr   
)r*   r+   r/   r0   r4   r5   )column_namesr<   r   r   r   make_empty_csvJ   s    
rD   c              	   K   s   |  }|  D ]R\}}t|||d ks4td| |D ]&}t||| t|||ks8tdq8qtt d|_W 5 Q R X dd |  D }| f |}|  D ]\}}t|||kstqdS )zD
    Check setting and getting attributes of an *Options class.
    r   zincorrect default value for zfailed setting valueTc                 S   s   i | ]\}}||d  qS )   r   )r$   namevaluesr   r   r   
<dictcomp>b   s      z'check_options_class.<locals>.<dictcomp>N)itemsgetattrAssertionErrorsetattrpytestraisesAttributeErrorZzzz_non_existent)clsattr_valuesoptsrF   rG   vZnon_defaultsvaluer   r   r   check_options_classQ   s    
rU   c                 K   sH   | f |}t t j|t jd}| D ]\}}t|||ks(tq(d S )N)protocol)pickleloadsdumpsHIGHEST_PROTOCOLrI   rJ   rK   )rP   rQ   rR   Znew_optsrF   rT   r   r   r   check_options_class_picklingi   s    

r[   c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
InvalidRowHandlerc                 C   s   || _ g | _d S r   )resultrows)selfr]   r   r   r   __init__r   s    zInvalidRowHandler.__init__c                 C   s   | j | | jS r   )r^   appendr]   )r_   r>   r   r   r   __call__v   s    zInvalidRowHandler.__call__c                 C   s   t |to|j| jkS r   
isinstancer\   r]   r_   otherr   r   r   __eq__z   s    

zInvalidRowHandler.__eq__c                 C   s   t |t p|j| jkS r   rc   re   r   r   r   __ne__~   s    
zInvalidRowHandler.__ne__N)__name__
__module____qualname__r`   rb   rg   rh   r   r   r   r   r\   q   s   r\   c               	   C   sr  t } |  }t| ddgddgg ddggddgddgdd	gd
 t| ddddgddd	d
 |jdksdtd|_|jdksxt| dd}|jdkst|  d}tjtj	|d |  }d|_|  W 5 Q R X d}tjtj	|d |  }d|_
|  W 5 Q R X d}tjtj	|d |  }d|_|  W 5 Q R X d}tjtj	|d  |  }d|_d|_|  W 5 Q R X d S )NTFr      abcdutf8utf16   )use_threads	skip_rowsrC   autogenerate_column_namesencodingskip_rows_after_names90  i  
block_sizez-ReadOptions: block_size must be at least 1: 0matchz-ReadOptions: skip_rows cannot be negative: -1z9ReadOptions: skip_rows_after_names cannot be negative: -1zTReadOptions: autogenerate_column_names cannot be true when column_names are providedr%   b)r   rU   r[   ry   rK   validaterM   rN   r!   ArrowInvalidrs   rv   rt   rC   rP   rR   r{   r   r   r   test_read_options   sV    


r   c               
   C   s  t } td}t| ddgddgdddgddgddgddgd |gd	 t| dddddd|d	 |    |  }d
|_|  d}tjtj	|d |  }d|_|  W 5 Q R X tjtj	|d |  }d|_|  W 5 Q R X d}tjtj	|d |  }d|_
|  W 5 Q R X tjtj	|d |  }d|_
|  W 5 Q R X d}tjtj	|d |  }d|_|  W 5 Q R X tjtj	|d |  }d|_|  W 5 Q R X d S )Nskipr   xFy"zT)	delimiterescape_char
quote_charZdouble_quotenewlines_in_valuesignore_empty_linesinvalid_row_handler	z,ParseOptions: delimiter cannot be \\r or \\nrz   rB   z-ParseOptions: quote_char cannot be \\r or \\nz.ParseOptions: escape_char cannot be \\r or \\n)r   r\   rU   r[   r   r   rM   rN   r!   r   r   r   )rP   Zskip_handlerrR   r{   r   r   r   test_parse_options   sb    

r   c                  C   s@  t } |  }t| ddgddgddgddgg ddggddgddgg tdggd	 t| ddddddgddtdgd	 tt d	|_W 5 Q R X |jd
kst	d|_|jdkst	|j
i kst	t t d|_
|j
t t dkst	ddd|_
|j
t t dkst	tdt fdt fg}||_
|j
t t dksVt	dt fg|_
|j
dt iks~t	tjtdd dd i|_
W 5 Q R X tt d
|_
W 5 Q R X t|jtst	d|jkst	d|jkst	ddg|_|jddgkst	t|jts t	ddg|_|jddgks>t	t|jtsPt	ddg|_|jddgksnt	|jg ks~t	tg|_|jtgkst	| dt iddgddgddgd td!gd"}|j
dt ikst	|jddgkst	|jddgkst	|jddgkst	|jd ks(t	|jtd!gks<t	d S )#NTF.r   defabcz%y-%m)
check_utf8strings_can_be_nullquoted_strings_can_be_nulldecimal_pointinclude_columnsinclude_missing_columnsauto_dict_encodetimestamp_parsersz..r   韆 )r~   cint16null)rS   wr%   r~   r}   r   zDataType expectedrz    N/AZxxxZyyyNnnr1   ttFffi  z%Y-%m-%d)column_typesnull_valuestrue_valuesfalse_valuesauto_dict_max_cardinalityr   )r	   rU   r
   r[   rM   rN   
ValueErrorr   r   rK   r   r!   r   float32r   schemaint32r   binary	TypeErrorrd   r   r,   r   r   r   )rP   rR   r   r   r   r   test_convert_options   s     


 



  r   c               	   C   s   t } |  }t| ddgdddgdddgd	 |jd
ks8td|_|jdksLt| dd}|jdksdt|  d}tjtj|d |  }d
|_|  W 5 Q R X d S )NTFr   r   |needednone	all_valid)include_headerr   quoting_styler   rw   i&  )
batch_sizez.WriteOptions: batch_size must be at least 1: 0rz   )	r   rU   r   rK   r   rM   rN   r!   r   r   r   r   r   test_write_optionsD  s&      
r   c                   @   s\   e Zd ZdZejdd Zeejdd Ze	dd Z
dd	 Zd
d Zdd Zdd ZdS )BaseTestCSVzDCommon tests which are shared by streaming and non streaming readersc                 K   s   t dS )z
        :param b: bytes to be parsed
        :param kwargs: arguments passed on to open the csv file
        :return: b parsed as a single RecordBatch
        NNotImplementedErrorr_   r~   kwargsr   r   r   
read_bytes_  s    zBaseTestCSV.read_bytesc                 C   s   t dS )z#Whether this test is multi-threadedNr   r_   r   r   r   rr   h  s    zBaseTestCSV.use_threadsc                 C   s$   | j t|kst| j|ks td S r   )num_columnslenrK   rC   )tablenamesr   r   r   check_namesn  s    zBaseTestCSV.check_namesc              	   C   s>  d}t  }d|_| j||d}| |ddg | ddgdd	gd
ksLtd|_| j||d}| |dd	g | g g dkstd|_ttj	 | j||d}W 5 Q R X d}d|_| j||d}| |ddg | dgd	gdkstd|_ddg|_
| j||d}| |ddg | g g dks:td S )N   ab,cd
ef,gh
ij,kl
mn,op
rE   read_optionsefghijmnklop)r   r   rl   )r   r         abcd
,,,,,
ij,kl
mn,op
r   )r   r   )r   rs   r   r   	to_pydictrK   rM   rN   r!   r   rC   r_   r^   rR   r   r   r   r   test_header_skip_rowss  sF    
z!BaseTestCSV.test_header_skip_rowsc           	      C   sf  d}t  }d|_| j||d}| |ddg | ddgdd	gd
ksLtd|_| j||d}| |ddg | g g d
kstd|_| j||d}| |ddg | g g d
kstd}d|_ddg|_| j||d}| |ddg | ddgdd	gdkstt  }d}d|_t }d|_| j|||d}| |ddg | dgd	gd
ksjtd|_d|_	| j|||d}| |ddg | dgd	gd
kstt  }t
ddd\}}d|_t|d |_	| j||d}|j|jkst|jdkst| }|  D ]$\}}|dd  || ks tq | j|||d}|j|jksht|jdksxt| }|  D ]$\}}|dd  || kstqt
dddd\}}t  }d|_d |_|jj|_| j|||d}|j|jkst|j|j|j |j ks t| }|  D ],\}}||j|j d  || ks4tq4d S )!Nr   rE   r   rm   rn   r   r   r   r   rm   rn   rl   r   r   r   f0f1r   r   s#   ab,cd
"e
f","g

h"
"ij","k
l"
mn,opTr   parse_options      r   r8   r9   i     d      F)r8   r9   r;   %   )   )r   rv   r   r   r   rK   rC   r   r   ry   rA   r   r   r9   rI   rs   r   )	r_   r^   rR   r   
parse_optsr@   Z
table_dictrF   rG   r   r   r   test_skip_rows_after_names  s    




z&BaseTestCSV.test_skip_rows_after_namesc              	      s    fdd}t dddd\}}t }t|d |_t }dt i|_|d	 }|d
d}tj	tj
|d  j|||d W 5 Q R X |d }|dd|}	tj	tj
|	d  j|||d W 5 Q R X d}
||
 }|dd|
dd d}tj	tj
|d  j|||d W 5 Q R X d|_tj	tj
|d  j|||d W 5 Q R X tj	tj
|	d  j|||d W 5 Q R X tj	tj
|d  j|||d W 5 Q R X d|_t dddd\}}ddddg|_|d	 }|d
d}tj	tj
|d  j|||d W 5 Q R X ||
 }|dd|
dd d}tj	tj
|d  j|||d W 5 Q R X |d }|dd}	|	t|}	tj	tj
|	d  j|||d W 5 Q R X d|_tj	tj
|d  j|||d W 5 Q R X tj	tj
|	d  j|||d W 5 Q R X d S )Nc                    s&    j rd}n
d|}| j|f| S )Nr   z	Row #{}: )rr   format)Z
msg_formatr>   argsZrow_infor   r   r   
format_msg  s    
z@BaseTestCSV.test_row_number_offset_in_errors.<locals>.format_msgr   r   T)r;   rl   r%   s   1,2
z{}Expected 4 columns, got 2f   rz   r   convert_optionss	   a,b,c,d
zDIn CSV column #0: {}CSV conversion error to int32: invalid value 'a's  this is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long row,3
z#{}Expected 4 columns, got 2: {} ...r   `   zutf-8/   Fr~   r   de      )rA   r   r   ry   r	   r!   r   r   rM   rN   r   r   decoderv   rC   r   rs   )r_   r   r<   _r   r   Zcsv_bad_columnsZmessage_columnsZcsv_bad_typeZmessage_valueZlong_rowZcsv_bad_columns_longZmessage_longr   r   r    test_row_number_offset_in_errors	  s    
 
z,BaseTestCSV.test_row_number_offset_in_errorsc              	      sJ  d}t  }tjtdd  j||d W 5 Q R X td|_ j||d}| ddgdd	gd
ksht fdd}t	dd|ddt	dd|ddg}|jj
|ksttd|_tjtdd  j||d W 5 Q R X t	dd|ddg}|jj
|ksttd|_tt|} j||d}| ddgdd	gd
ksFtd S )Ns   a,b
c
d,e
f,g,h
i,j
zExpected 2 columns, got 1: crz   r   r   r   iejr}   c                    s    j r
d S | S r   )rr   )r   r   r   r   row_num  s    z5BaseTestCSV.test_invalid_row_handler.<locals>.row_numr   rE   r   rl   r   zf,g,herror)r   rM   rN   r   r   r\   r   r   rK   r   r^   rW   rX   rY   )r_   r^   r   r   r   Zexpected_rowsr   r   r   test_invalid_row_handlerq  sB    


z$BaseTestCSV.test_invalid_row_handlerN)ri   rj   rk   __doc__r   abstractmethodr   propertyrr   staticmethodr   r   r   r   r   r   r   r   r   r   \  s   

-ihr   c                   @   s"  e Zd ZddddZdd Zdd Zd	d
 Zdd Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Zd1d2 Zd3d4 Zd5d6 Zd7d8 Zd9d: Zd;d< Zd=d> Z d?d@ Z!dAdB Z"dCdD Z#dEdF Z$dGS )HBaseCSVTableReadT)validate_fullc                O   sF   t | jtst|dt }| j|_t|f||}|j|d |S )a=  
        Reads the CSV file into memory using pyarrow's read_csv
        csv The CSV bytes
        args Positional arguments to be forwarded to pyarrow's read_csv
        validate_full Whether or not to fully validate the resulting table
        kwargs Keyword arguments to be forwarded to pyarrow's read_csv
        r   full)rd   rr   boolrK   
setdefaultr   r   r   )r_   r<   r   r   r   r   r   r   r   r   r     s    zBaseCSVTableRead.read_csvc                 K   s   | j t|f|S r   )r   r!   	py_bufferr   r   r   r   r     s    zBaseCSVTableRead.read_bytesc              	   C   sh   d}dgdgd}t |}| |}| |ks6tt | }tt	 | | W 5 Q R X d S )Ns   a,b
1,2
rE   r   r}   )
r*   BytesIOr   r   rK   r+   r   rM   rN   r   )r_   dataexpected_databior   sior   r   r   test_file_object  s    

z!BaseCSVTableRead.test_file_objectc                 C   sB   d}|  |}t|tjst| |dddg |jdks>td S )N   abc,def,gh
r   r   r   r   )r   rd   r!   r6   rK   r   r9   r_   r^   r   r   r   r   test_header  s
    
zBaseCSVTableRead.test_headerc                 C   s0   d}dgdgd}|  |}| |ks,td S )Ns   ﻿a,b
1,2
rE   r   r}   r   r   rK   )r_   r^   r  r   r   r   r   test_bom  s    
zBaseCSVTableRead.test_bomc                 C   sv   ddddg}dD ]`}d|fD ]R}| || }| |}t| dksLt| ddd	gd
ddgdkstqqd S )Ns   a,bs   1,2s   3,4s   56,78)   
      
    rE   rl   8   r   r   N   r}   )r0   r   r   
to_batchesrK   r   )r_   r^   Zline_endingZfile_endingr  r   r   r   r   test_one_chunk  s    
zBaseCSVTableRead.test_one_chunkc              	   C   s  d}t  }ddg|_| j||d}| |ddg | ddddgd	d
ddgdksXtd|_| j||d}| |ddg | dgdgdkstd|_| j||d}| |ddg | g g dkstd|_tt	j
 | j||d}W 5 Q R X d|_dddg|_tjt	j
dd | j||d}W 5 Q R X d}d|_ddg|_| j||d}| |ddg | ddgddgdkstd S )Nr   r   r   r   rm   r   r   r   rn   r   r   r   r   r   rl   r   r   r   r   Expected 3 columns, got 2rz   r   r   )r   rC   r   r   r   rK   rs   rM   rN   r!   r   r   r   r   r   test_header_column_names  sR    



z)BaseCSVTableRead.test_header_column_namesc              	   C   s   d}t  }d|_| j||d}| |ddg | dddd	gd
dddgdksTtd|_| j||d}| |ddg | d	gdgdkstd|_tt	j
 | j||d}W 5 Q R X d S )Nr   Tr   r   r   rm   r   r   r   rn   r   r   r   r   rl   r   )r   rt   r   r   r   rK   rs   rM   rN   r!   r   r   r   r   r   %test_header_autogenerate_column_names
  s&    

z6BaseCSVTableRead.test_header_autogenerate_column_namesc              	   C   s   d}t  }dg|_| j||d}| |dg | ddddgiksHtddg|_| j||d}tdt fdt fg}|j|kst| dd	d
gdddgdkstdddg|_t	j
tdd | j||d W 5 Q R X d S )Nr   rm   r   r   r   r   rn   r   r   r   )rn   rm   xxyyz9Column 'xx' in include_columns does not exist in CSV filerz   )r	   r   r   r   r   rK   r!   r   r   rM   rN   KeyError)r_   r^   r   r   r   r   r   r   test_include_columns#  s0     


z%BaseCSVTableRead.test_include_columnsc                 C   s  d}t  }t }dddg|_d|_| j|||d}tdt fdt fdt fg}|j|ksht	|
 d d d gddd	gd d d gd
kst	ddg|_ddg|_| j|||d}tdt fdt fg}|j|kst	|
 ddddgd d d d gdks
t	t t d|_| j|||d}tdt fdt fg}|j|ks\t	|
 ddddgd d d d gdkst	d S )Nr   r  rm   r  Tr   r   r   r   )r  rm   r  rn   r   r   r   )r  rn   s   cds   ghs   kls   op)r   r	   r   r   r   r!   r   r   r   rK   r   rC   r   r   r   r_   r^   r   r   r   r   r   r   r   test_include_missing_columns@  sZ    









z-BaseCSVTableRead.test_include_missing_columnsc                 C   sl   d}|  |}tdt fdt fdt fg}|j|ksDt| ddgddgd	d
gdkshtd S )Ns   a,b,c
1,2,3
4,5,6
r%   r~   r   rE   r   r   r   rl      r%   r~   r   )r   r!   r   r#   rK   r   r_   r^   r   r   r   r   r   test_simple_intsm  s    


z!BaseCSVTableRead.test_simple_intsc                 C   s|   d}|  |}tdt fdt fdt fdt fg}|j|ksNt| ddgdd	gd
dgddgdksxtd S )Ns    a,b,c,d
1,2,3,0
4.0,-5,foo,True
r%   r~   r   r         ?g      @r   3fooFTr%   r~   r   r   )	r   r!   r   float64r#   r   bool_rK   r   r&  r   r   r   test_simple_varied{  s    



z#BaseCSVTableRead.test_simple_variedc              
   C   s   d}|  |}tdt fdt fdt fdt fdt fdt fg}|j|ksbt	|
 dd d	gd
dd gdddgd d d gdddgd ddgdkst	d S )NsC   a,b,c,d,e,f
1,2,,,3,N/A
nan,-5,foo,,nan,TRUE
4.5,#N/A,nan,,,false
r%   r~   r   r   r   fr(  g      @r   r)  r   r+  nan   3s   nan   TF)r%   r~   r   r   r   r0  )r   r!   r   r-  r#   r   r   r   r.  rK   r   r&  r   r   r   test_simple_nulls  s$    





z"BaseCSVTableRead.test_simple_nullsc                 C   s   t dd}d}| j||d}tdt fdt fg}|j|ksHt| dd dgd	d
dgdksjttdd}| j|||d}tdt fdt fg}|j|kst| dddgddd gdkstd S )N;r   s   a;b
1.25;2,5
NA;-3
-4;NAr   r%   r~   g      ?g      z2,5z-3ZNAr}   r   )r   )r   r   z1.25z-4g      @g      )	r   r   r!   r   r-  r   rK   r   r	   )r_   r   r^   r   r   r   r   r   r   test_decimal_point  s.    



z#BaseCSVTableRead.test_decimal_pointc                 C   s   d}|  |}tdt fdtdfdtdfg}|j|ksHt| ddgtdd	d	tdd
dd	gtdd	d	ddddtdd
dd	dddgdkstd S )Nsk   a,b,c
1970,1970-01-01 00:00:00,1970-01-01 00:00:00.123
1989,1989-07-14 01:00:00,1989-07-14 01:00:00.123456
r%   r~   sr   ns  i  rE         r   ix @ r%  )r   r!   r   r#   	timestamprK   r   r   r&  r   r   r   test_simple_timestamps  s    
z'BaseCSVTableRead.test_simple_timestampsc                 C   st  d}t  }| j||d}tdt fdtdfg}|j|ksFt| ddgtdd	d	tdd	d
gdksttdg|_	| j||d}tdtdfdt fg}|j|kst| tdd	d	tdd	d
gddgdkstdt
g|_	| j||d}tdtdfdtdfg}|j|ks0t| tdd	d	tdd	d
gtdd	d	tdd	d
gdksptd S )Ns6   a,b
1970/01/01,1980-01-01 00
1970/01/02,1980-01-02 00
r  r%   r~   r8  z
1970/01/01z
1970/01/02i  rE   r   r}   z%Y/%m/%dr:  z1980-01-01 00z1980-01-02 00)r	   r   r!   r   r   r>  rK   r   r   r   r
   )r_   r^   rR   r   r   r   r   r   test_timestamp_parsers  s>    

z'BaseCSVTableRead.test_timestamp_parsersc                 C   s  d}|  |}tdt fdt fg}|j|ks:t| tdddtdddgtdddtdddgdksxtt }t t d|_	| j ||d	}tdt fdt fg}|j|kst| tdddtdddgtdddtdddgdkstt }t
d
t
dd|_	| j ||d	}tdt
d
fdt
dfg}|j|ksjt| tdddtdddgtdddtdddgdkstd S )Ns0   a,b
1970-01-01,1970-01-02
1971-01-01,1971-01-02
r%   r~   r:  rE   i  r   r}   r  r8  ms)r   r!   r   Zdate32rK   r   r   r	   Zdate64r   r>  r   )r_   r^   r   r   rR   r   r   r   
test_dates  s@    


zBaseCSVTableRead.test_datesc                 C   s  ddl m} d}| |}tdtdfdt fg}|j|ksHt| |ddd	|d
ddgddgdksvtt	 }t
dtdd|_| j||d}tdt
dfdtdfg}|j|kst| |ddd	|d
ddg|ddd	d|d
dddgdkstd S )Nr   )times0   a,b
12:34:56,12:34:56.789
23:59:59,23:59:59.999
r%   r8  r~      "   r  r   ;   z12:34:56.789z23:59:59.999r}   usrA  r  i
 iX> )r   rC  r   r!   r   Ztime32r   rK   r   r	   Ztime64r   )r_   rC  r^   r   r   rR   r   r   r   
test_times  s,    

zBaseCSVTableRead.test_timesc                 C   s  t dd}d }| j||d}tdtt t fdt fg}ddddgd	d
ddgd}|j|ksrt	|
 |kst	d
|_| j||d}|j|kst	|
 |kst	d	|_| j||d}|jtdt fdt fgkst	|
 |kst	d|_d|_d}| j||dd}|j|ks2t	|d dj}t|d
ksTt	|d  dksjt	|d	  dkst	d|_| j||d}tdtt t fdt fg}dddgd	d
dgd}|j|kst	|
 |kst	d S )NT)r   u   a,b
ab,1
cdé,2
cdé,3
ab,4r  r%   r~   rm   u   cdérE   r   rl   r   r}   2   Fs   a,b
ab,1
cd,2
ab,3)r   r   r   s   cds   ab)r	   r5   r   r!   r   
dictionaryr   r   r#   rK   r   r   r   chunkr   Zas_pyZ	as_bufferr   )r_   rR   r^   r   r   r@   dict_valuesr   r   r   test_auto_dict_encode-  sT    





z&BaseCSVTableRead.test_auto_dict_encodec                 C   s|  t ddgd}d}| j||d}tdt fdt fdt fd	t fg}|j|ks`t| d d gdd
gddgdd gdkstt ddgdd}| j||d}| d d gd d
gddgdd gdkstd|_	| j||d}| d d gdd
gddgdd gdkstt g d}d}| j||d}tdt fdt fg}|j|ks\t| d
gdgdksxtd S )NZXxxZZzz)r   s$   a,b,c,d
Zzz,"Xxx",1,2
Xxx,#N/A,,Zzz
r  r%   r~   r   r   z#N/A1r   r   r,  T)r   r   Fs
   a,b
#N/A,
r}   )
r	   r   r!   r   r   r   r#   rK   r   r   r_   rR   r^   r   r   r   r   r   test_custom_nulls^  sX    




z"BaseCSVTableRead.test_custom_nullsc                 C   s   t ddgddgd}d}| j||d}tdt fd	t fd
t fg}|j|ks\t| dddddgddddd gdddddgdkstd S )Nr1   yesr   no)r   r   s>   a,b,c
True,T,t
False,F,f
True,yes,yes
False,no,no
N/A,N/A,N/A
r  r%   r~   r   TrueFalser   TFtr0  r%  )r	   r   r!   r   r   r.  rK   r   rO  r   r   r   test_custom_bools  s    

z"BaseCSVTableRead.test_custom_boolsc                 C   s  t dddtddddd}d	}| j||d
}tdt fdt fdt fdt fdtddfg}ddgddgddgddgt	dt	dgd}|j|kst
| |kst
t tdt fdt fdt fdtddfdt fgd}| j||d
}|j|kst
| |ks0t
d}ttj}| j||d
 W 5 Q R X t|j}d|ksrt
d|kst
d S )Nr   r   booleanr   r   r   )r~   r   r   r   zzr   s(   a,b,c,d,e
1,2,3,true,1.0
4,-5,6,false,0
r  r%   r~   r   r   r   rE   r   g       @g      r*  6TF1.00z0.00)r%   r~   r   r   r   rX  s(   a,b,c,d,e
1,XXX,3,true,5
4,-5,6,false,7
zIn CSV column #1: z2CSV conversion error to float: invalid value 'XXX')r	   r!   
decimal128r   r   r#   r   r   r.  r   rK   r   rM   rN   r   r3   rT   )r_   rR   r^   r   r   r@   excerrr   r   r   test_column_types  sN    








z"BaseCSVTableRead.test_column_typesc              
   C   s,  dt t  t  fdt t  t  fdt t  t ddfdt t  t  fg}tt|d}d}| j	||d	}t 
|}d
dd
gddd gtdtdtdgdddgd}|j
|kst| |kstdt t  t  f|d< tt|d}tt | j	||d	}W 5 Q R X d S )Nr%   r~   r   r   r   r   rY  s<   a,b,c,d
abc,123456,1.0,zz
defg,123456,0.5,xx
abc,N/A,1.0,xx
r  r   Zdefgr=  r[  z0.50rX  r  r,  r   )r!   rJ  r   ro   r#   r\  Z
large_utf8r	   dictr   r   r   rK   r   Zint8rM   rN   r   )r_   r   rR   r^   r   r   r@   r   r   r   test_column_types_dict  s(    
z'BaseCSVTableRead.test_column_types_dictc                 C   s   d}t ddgd}tdt id}| j|||d}tdt fdt fg}|j|ks`t| ddd	gd
ddgdkstd S )Ns   a,b
c,d
e,f
r   r   rC   rY  r      a   c   er~   r   r0  r  )	r   r	   r!   r   r   r   r   rK   r   r"  r   r   r   #test_column_types_with_column_names  s    
z4BaseCSVTableRead.test_column_types_with_column_namesc                 C   s6   d}|  |}| ddgddgddgdks2td S )	N   a,b,c
1,2,3
4,5,6rE   r   r   r   rl   r$  r%  r  r  r   r   r   test_no_ending_newline  s    
z'BaseCSVTableRead.test_no_ending_newlinec                 C   s&   d}|  |}| dg iks"td S )Ns   ,

r   r  r  r   r   r   test_trivial  s    
zBaseCSVTableRead.test_trivialc                 C   s   d}|  |}| ddgddgdks,ttdd}| j ||d	}| d dd dgd dd dgdksjttdd
}| j |||d}| d dgd dgdkstd S )Ns   a,b
1,2

3,4
rE   rl   r   r   r}   F)r   r   )rs   )r   r   )rN  2)r   r   rK   r   r   )r_   r^   r   r   r   r   r   r   test_empty_lines
  s(    




z!BaseCSVTableRead.test_empty_linesc              
   C   s   d}t jtjdd | | W 5 Q R X d}t jtjdd | | W 5 Q R X dD ]*}t jtjdd | | W 5 Q R X qXd S )Ns   a,b,c
1,2
4,5,6
r  rz   s   a,b,c
1,2,3
4zExpected 3 columns, got 1)r  r  r  r  s   

Empty CSV file)rM   rN   r!   r   r   )r_   r^   r   r   r   test_invalid_csv  s    z!BaseCSVTableRead.test_invalid_csvc                 C   s^   d}|  |}| dgdgdks(ttdd}| j ||d}| dgd	gd
ksZtd S )N   a;b,c
de,fg;eh
defg;eha;br   r5  r6  r   de,fgehr%   b,c)r   r   rK   r   )r_   r^   r   rR   r   r   r   test_options_delimiter+  s    

z'BaseCSVTableRead.test_options_delimiterc                 C   sP   t ddd\}}| |}|j|jks*t||s8t| | ksLtd S Nr   r   r   )rA   r   r   rK   equalsr   )r_   r<   r@   r   r   r   r   test_small_random_csv9  s
    
z&BaseCSVTableRead.test_small_random_csvc           	      C   s   t ddd\}}dddddd	g}||d
g}|D ]T}|D ]J}t|d}| j||d}|j|jksft||s:| | ks:tq:q2d S )Nr     r   r   rD        r   o   r  rx   r   )rA   rstripr   r   r   rK   ry  r   )	r_   csv_baser@   block_sizescsvsr<   ry   r   r   r   r   r   test_stress_block_sizes@  s    

z(BaseCSVTableRead.test_stress_block_sizesc           	      C   s   z
t j}W n tk
r$   t j }Y nX d}dd t|D }t|}| }tdd |d d d D d}| j||d}| | }|d	kst|j|kst|j	d
kst|j
|kstd S )Ni'  c                 S   s   g | ]}d  |qS )zK{})r   )r$   r   r   r   r   r&   V  s     zGBaseCSVTableRead.test_stress_convert_options_blowup.<locals>.<listcomp>c                 S   s   i | ]}|t  qS r   )r!   r   )r$   kr   r   r   rH   Z  s      zGBaseCSVTableRead.test_stress_convert_options_blowup.<locals>.<dictcomp>r   rY  r        $@r   )rC  thread_timerO   rangerD   r	   r   rK   r   r9   rC   )	r_   clockr   r=   r<   t1r   r   dtr   r   r   "test_stress_convert_options_blowupN  s"    

z3BaseCSVTableRead.test_stress_convert_options_blowupc                    s  t  jt  jkrtd t  tj	 fdd}d}d}d}|dk rB|dk rBt
d	| d
d|  }d }zt }zbt j|d}|  t }	z| | W n2 tk
r }
 z|
}t |	 }W 5 d }
~
X Y nX W 5 t|gg |gd X W 5 Q R X W n tk
r   Y nX |d k	r8|jd k	r8qB|d }qF|d krVtd |dksdt|j}
t|
tjs|t|
jkstd S )Nz'test only works from main Python threadc                      s   t d   d S )Ng?)rC  sleepr   raise_signalsignumr   r   signal_from_threadn  s    
z>BaseCSVTableRead.test_cancellation.<locals>.signal_from_threadg        i r   g      @r   zworkload size:s   a,b,c
s   1,2,3
r  targetrl   z0Failed to get an interruption during CSV readingr(  )	threadingcurrent_threadidentmain_threadrM   r   r   Zget_raise_signalsignalSIGINTprintZsignal_wakeup_fdselectThreadstartrC  r   KeyboardInterrupt__context__ZfailrK   rd   r!   ZArrowCancelledr  )r_   r  Zlast_durationZworkload_sizeZattemptsZ	large_csvexc_infoZsigfdrU  r  r   r   r  r   test_cancellationf  sL    


"$



z"BaseCSVTableRead.test_cancellationc                    s(   t j fddd}|  |  d S )Nc                      s
     dS )Ns   f64
0.1)r   r   r   r   r   <lambda>  r  z=BaseCSVTableRead.test_cancellation_disabled.<locals>.<lambda>r  )r  r  r  r0   )r_   rU  r   r   r   test_cancellation_disabled  s
    
z+BaseCSVTableRead.test_cancellation_disabledN)%ri   rj   rk   r   r   r  r  r  r  r  r  r!  r#  r'  r/  r4  r7  r?  r@  rB  rH  rM  rP  rV  r_  ra  rf  rh  ri  rk  rm  rw  rz  r  r  r  r  r   r   r   r   r     sD   3-"$1.) 
Er   c                   @   s   e Zd Zedd ZdS )TestSerialCSVTableReadc                 C   s   dS NFr   r   r   r   r   rr     s    z"TestSerialCSVTableRead.use_threadsNri   rj   rk   r   rr   r   r   r   r   r    s   r  c                   @   s   e Zd Zedd ZdS )TestThreadedCSVTableReadc                 C   s   dS NTr   r   r   r   r   rr     s    z$TestThreadedCSVTableRead.use_threadsNr  r   r   r   r   r    s   r  c                       s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Z fd#d$Z fd%d&Z  ZS )'BaseStreamingCSVReadc                 O   s&   | dt }| j|_t|f||S )z
        Reads the CSV file into memory using pyarrow's open_csv
        csv The CSV bytes
        args Positional arguments to be forwarded to pyarrow's open_csv
        kwargs Keyword arguments to be forwarded to pyarrow's open_csv
        r   )r  r   rr   r   )r_   r<   r   r   r   r   r   r   r     s    zBaseStreamingCSVRead.open_csvc                 K   s   | j t|f|S r   )r   r!   r  r   r   r   r   
open_bytes  s    zBaseStreamingCSVRead.open_bytesc                 C   sl   |j |kstt|}t|t|ks*tt||D ]2\}}|jdd |j |ksVt| |ks4tq4d S )NTr  )r   rK   r,   r   zipr   r   )r_   readerexpected_schemar  ZbatchesbatchZexpected_batchr   r   r   check_reader  s    z!BaseStreamingCSVRead.check_readerc                 K   s   | j |f| S r   )r  read_allr   r   r   r   r     s    zBaseStreamingCSVRead.read_bytesc                 C   s\   d}ddgddgd}t |}| |}tdt fdt fg}| |||g d S )	Ns   a,b
1,2
3,4
rE   rl   r   r   r}   r%   r~   )r*   r  r   r!   r   r#   r  )r_   r  r  r	  r  r  r   r   r   r    s    


z%BaseStreamingCSVRead.test_file_objectc                 C   sH   d}|  |}tdt fdt fdt fg}| ||g  d S )Nr  r   r   r   )r  r!   r   r   r  r_   r^   r  r  r   r   r   r    s    


z BaseStreamingCSVRead.test_headerc                 C   s   d}t dt  fdt  fg}t }t||_| j||d}| ||dddgdd	d
gdg t|d |_| j||d}| ||ddgdd	gddgd
gdg d S )N   a,b
123,456
abc,de
gh,ij
r%   r~   r   Z123r   r   s   456s   des   ijr}   rE   )	r!   r   r   r   r   r   ry   r  r  )r_   r^   r  r   r  r   r   r   test_inference  s.    

z#BaseStreamingCSVRead.test_inferencec              	   C   s   d}t  }t|d |_| j||d}tdt fdt fg}|j|ksRt| 	 dgdgdksptt
jtd	d
 |  W 5 Q R X t
t |  W 5 Q R X d S )Nr  r;  r   r%   r~   {   i  r}   zCSV conversion error to int64rz   )r   r   ry   r  r!   r   r#   rK   read_next_batchr   rM   rN   r   StopIterationr_   r^   r   r  r  r   r   r   test_inference_failure  s$    

 z+BaseStreamingCSVRead.test_inference_failurec              	   C   s   d}t  }d|_tjtjdd | j||d}W 5 Q R X d}d|_| j||d}|  dgd	gd
kspt	tjtjdd |  W 5 Q R X tt
 |  W 5 Q R X d S )Ns   a,b
1,2,3
4,5
6,7
r   zExpected 2 columns, got 3rz   r   s   a,b
1,2
3,4,5
6,7
   rE   r   r}   )r   ry   rM   rN   r!   r   r  r  r   rK   r  )r_   r^   r   r  r   r   r   rm    s(     z%BaseStreamingCSVRead.test_invalid_csvc                 C   s   d}|  |}tdt fdt fg}| ||dgdgdg tdd}| j ||d	}td
t fdt fg}| ||dgdgdg d S )Nrn  rr  r   ro  rp  rq  r5  r6  r   r%   rv  rs  rt  ru  )r  r!   r   r   r  r   )r_   r^   r  r  rR   r   r   r   rw  -  s(    



z+BaseStreamingCSVRead.test_options_delimiterc                 C   s^   d}|  |}tdt fdt fdt fg}| ||ddgddgd	d
gdg d S )Nrg  r%   r~   r   rE   r   r   r   rl   r$  r%  )r  r!   r   r#   r  r  r   r   r   rh  >  s    


z+BaseStreamingCSVRead.test_no_ending_newlinec              	   C   s(   t jtdd | d W 5 Q R X d S )Nrl  rz   r  )rM   rN   r   r  r   r   r   r   test_empty_fileJ  s    z$BaseStreamingCSVRead.test_empty_filec              	   C   s2  d}t  }dddg|_| j||d}tdt fdt fdt fg}| ||ddgdd	gd
dgdg t }ddg|_| j|||d}tdt fdt fg}| ||dd	gd
dgdg dt	 i|_
| j|||d}tdt fdt	 fg}| ||ddgd
dgdg dddg|_tjtdd | j|||d}W 5 Q R X d|_| j|||d}tdt fdt fdt	 fg}| ||d d gddgd
dgdg t	 t d|_
| j|||d}tdt fdt fdt	 fg}| ||d d gddgd
dgdg d S )Ns   1,2,3
4,5,6r   r   r0  r   rE   r   r   r   rl   r$  )r   r   r0  r   )r   r0  rj  5gz,Column 'g' in include_columns does not existrz   T)r  r   r0  )r   r  )r   rC   r  r!   r   r#   r  r	   r   r   r   rM   rN   r   r   r   r-  )r_   r^   r   r  r  r   r   r   r   test_column_optionsN  s    








z(BaseStreamingCSVRead.test_column_optionsc                 C   s   d}t  }| j||d}tdt fdt fg}| ||dgdgdg d|_| j||d}tdt fdt fg}| ||dgd	gdg d
}d|_| j||d}tdt fdt fg}| ||dgd	gdg d S )Ns   a,b
un,lphantr   r%   r~   uns   lphantr}   latin1u
   éléphants    a , b 
 u n ,  l  p h a n t rp   )r   r  r!   r   r   r   r  ru   r  r   r   r   test_encoding  s@    


z"BaseStreamingCSVRead.test_encodingc                 C   sX   t ddd\}}| |}| }|j|jks2t||s@t| | ksTtd S rx  )rA   r  r  r   rK   ry  r   )r_   r<   r@   r  r   r   r   r   rz    s    
z*BaseStreamingCSVRead.test_small_random_csvc           
      C   s   t ddd\}}dddddd	g}||d
g}|D ]v}|D ]l}|d | ddksXtt|d}| j||d}| }	|	j|jkst|	|s:|		 |	 ks:tq:q2d S )Nr   r{  r         r   r   r   r~  r  r  rx   r   )
rA   r  countrK   r   r  r  r   ry  r   )
r_   r  r@   r  r  r<   ry   r   r  r   r   r   r   r    s    
 
z,BaseStreamingCSVRead.test_stress_block_sizesc              	   C   s  t   t }dd }d}t }ddg|_d|_| j||d}||dgd	gd
 t }||dgdgd
 t |kst||dgdgd
 t |kst||dgdgd
 t |kstt	
t |  W 5 Q R X t |kstd }t |kstd S )Nc                 S   s   |   }| |kstd S r   )r  r   rK   )r  r@   r  r   r   r   check_one_batch  s    zABaseStreamingCSVRead.test_batch_lifetime.<locals>.check_one_batchs   10,11
12,13
14,15
16,17
r%   r~   r$  r   r   r   r}   rD  r|  r<        r}  )gcZcollectr!   Ztotal_allocated_bytesr   rC   ry   r  rK   rM   rN   r  r  )r_   Zold_allocatedr  r^   r   r  Zallocated_after_first_batchr   r   r   test_batch_lifetime  s*    
z(BaseStreamingCSVRead.test_batch_lifetimec              	      sX   t    d}t }d|_ddg|_| j||d}tt |	 sJt
W 5 Q R X d S )Nr   r   rm   rn   r   )superr   r   rs   rC   r  rM   rN   r  r  rK   r_   r^   rR   r  	__class__r   r   r     s    

z*BaseStreamingCSVRead.test_header_skip_rowsc              	      s   t    d}t }d|_| j||d}tt | s@t	W 5 Q R X d|_| j||d}tt | svt	W 5 Q R X d S )Nr   rl   r   r   )
r  r   r   rv   r  rM   rN   r  r  rK   r  r  r   r   r     s    
z/BaseStreamingCSVRead.test_skip_rows_after_names)ri   rj   rk   r   r  r  r   r  r  r  r  rm  rw  rh  r  r  r  rz  r  r  r   r   __classcell__r   r   r  r   r    s&   		Ar  c                   @   s   e Zd Zedd ZdS )TestSerialStreamingCSVReadc                 C   s   dS r  r   r   r   r   r   rr     s    z&TestSerialStreamingCSVRead.use_threadsNr  r   r   r   r   r    s   r  c                   @   s   e Zd Zedd ZdS )TestThreadedStreamingCSVReadc                 C   s   dS r  r   r   r   r   r   rr   
  s    z(TestThreadedStreamingCSVRead.use_threadsNr  r   r   r   r   r  	  s   r  c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
BaseTestCompressedCSVReadc                 C   s   t jdd| _d S )Nzarrow-csv-test-)prefix)tempfilemkdtemptmpdirr   r   r   r   setUp  s    zBaseTestCompressedCSVRead.setUpc                 C   s   t | j d S r   )shutilrmtreer  r   r   r   r   tearDown  s    z"BaseTestCompressedCSVRead.tearDownc              
   C   sB   z
t |W S  tjk
r< } ztt| W 5 d }~X Y nX d S r   )r   r!   ZArrowNotImplementedErrorrM   r   r3   )r_   csv_pathr   r   r   r   r     s    
z"BaseTestCompressedCSVRead.read_csvc                 C   sz   t ddd\}}tj| j| j}| || | |}|jdd |j	|j	ksTt
||sbt
| | ksvt
d S )Nr   r   r   Tr  )rA   ospathr0   r  csv_filename
write_filer   r   r   rK   ry  r   )r_   r<   r@   r  r   r   r   r   test_random_csv  s    
z)BaseTestCompressedCSVRead.test_random_csvN)ri   rj   rk   r  r  r   r  r   r   r   r   r    s   r  c                   @   s    e Zd ZdZdd Zdd ZdS )TestGZipCSVReadzcompressed.csv.gzc              	   C   s(   t |dd}|| W 5 Q R X d S )Nwbrl   )gzipopenr/   r_   r  contentsr0  r   r   r   r  +  s    zTestGZipCSVRead.write_filec              	   C   s   t j| j| j}t|dd}|d W 5 Q R X t|dd}|d W 5 Q R X | |}|	 dddgd	d
dgdkst
d S )Nr  rl   s   ab,cd
ef,gh
rm   s   ij,kl
mn,op
r   r   r   r   r   r   r   )r  r  r0   r  r  r  r  r/   r   r   rK   )r_   r  r0  r   r   r   r   test_concatenated/  s    
z!TestGZipCSVRead.test_concatenatedN)ri   rj   rk   r  r  r  r   r   r   r   r  (  s   r  c                   @   s   e Zd ZdZdd ZdS )TestBZ2CSVReadzcompressed.csv.bz2c              	   C   s&   t |d}|| W 5 Q R X d S )Nr   )bz2BZ2Filer/   r  r   r   r   r  @  s    zTestBZ2CSVRead.write_fileN)ri   rj   rk   r  r  r   r   r   r   r  =  s   r  c                  C   s    t d} t|  | jrtd S Nrg  )r*   r  r   closedrK   )bufr   r   r   0test_read_csv_does_not_close_passed_file_handlesE  s    
r  c            	   
   C   s  t jdddgdddggddg} | jd	d
d }| |fD ]|}t }t||tdd |d | t	|ksrt
t }t||tdd |d t| jd}| t	||dks:t
q:d d tddft| jdd tddfd tddtdddft| jdtddtdddffD ]\}}}t }t|| j|d}||  W 5 Q R X |d | t	|||dksnt
t }t|| j|d$}| jdd
D ]}|| qW 5 Q R X |d | t	|||dkst
qd S )NrE   r   rl   r%   r~   r   c1c2r   )Zmax_chunksizer   T)r   Frb  r   r   r6  )r   r   r   write_optionsr   )r!   r6   r7   r  r*   r  r   r   seekr   rK   r   rC   r   r   r   write_tableZwrite_batch)	rU  Zrecord_batchr  r  r   r   r  writerr  r   r   r   test_write_read_round_tripL  sN    "








r  c                  C   sx  t jddd gdd dggddg} t }tddd	ft d
ftdddffD ]H\}}t|| j|d}||  W 5 Q R X |	 |kst
|d qNt jddggdg} t }tdddftddt jjffD ]\}}t|| j|db}z||  W nL tk
rJ } z,t||ks$t
W Y W 5 Q R   qtW 5 d }~X Y nX W 5 Q R X |	 |ksht
|d qd S )NrE   r   r%   r   r  r  r   )r   s   "c1","c2"
1,a
2,
,c
s   "c1","c2"
1,"a"
2,
,"c"
r   s   "c1","c2"
"1","a"
"2",
,"c"
r  r   r   r   r   s   "c1"
","
""""
)r!   r6   r7   r*   r  r   r   r   r  r4   rK   r  libr   	Exceptionr    )rU  r  r  resr  r   r   r   r   test_write_quoting_stylex  s2    "2r  c               	   C   s4   dd } t   |  }| d ks&tW 5 Q R X d S )Nc                  S   s   t d} t| }t|S r  )r*   r  r   weakrefref)r  r   r   r   r   inner  s    
z,test_read_csv_reference_cycle.<locals>.inner)r   Zdisabled_gcrK   )r  wrr   r   r   test_read_csv_reference_cycle  s    
r  type_factoryc                   C   s   t ddS N   rE   r!   r\  r   r   r   r   r    r  r  c                   C   s   t ddS )N&   r  r   r   r   r   r   r    r  c                   C   s   t ddS r  r!   Z
decimal256r   r   r   r   r    r  c                   C   s   t ddS )NL   r   r  r   r   r   r   r    r  c                 C   s`   | }t dt ddg|i}t|| d  t| d }|d||dks\td S )NcolrE   r   zout.csv)r!   r   r"   castr   r   columnrK   )r  r  r    r   outr   r   r   test_write_csv_decimal  s
    r  )r   r   r   T)Dr   r  r   r   decimalr   r  r  r*   r-   r  rW   r  r  r  r   r  r  rC  Zunittestr  rM   numpyr'   Zpyarrowr!   Zpyarrow.csvr   r   r   r   r	   r
   r   r   r   r   Zpyarrow.testsr   r   rA   rD   rU   r[   r\   r   r   r   r   ABCr   r   r  r  r  ZTestCaser  r  r  r  r  r  r  r  r  markZparametrizer  r   r   r   r   <module>   s   0	
6;P  C        D,!