U
    sVcT8                     @   s  d dl Z d dlmZ d dlZd dlZd dlmZ d dlZd dlZ	d dl
Z
d dlmZ d dlm  mZ d dlZd dlmZ e
jdd Ze
jdeddfd	dgfgd
dd Ze
jdG dd dZdd Zdd Zdd Zdd Z dd Z!e
j"dddgdd Z#dd  Z$d!d" Z%d#d$ Z&d%d& Z'd'd( Z(d)d* Z)d+d, Z*d-d. Z+d/d0 Z,d1d2 Z-e
j"d3d4d5gd6d7 Z.d8d9 Z/d:d; Z0d<d= Z1dS )>    N)datetime)Path)EmptyDataErrorc                 C   s   | dddS )Niosasdata )datapathr   r   E/tmp/pip-unpacked-wheel-xj8nt62q/pandas/tests/io/sas/test_sas7bdat.pydirpath   s    r            )paramsc                 C   s   | j \}}tj|d| d}t|}tddd}tj|d dd}|| |d< tj|d dd}|| |d< t|j	d D ]D}	|j
d d |	f }
|
jtjkr||	|j
d d |	f tj q||fS )	NZtest_sas7bdat_z.csvi  r   ZColumn4d)unitZColumn12)paramospathjoinpdread_csvr   Zto_timedeltarangeshapeilocdtypenpZint64Zisetitemastypefloat64)requestr   itest_ixfnamedfepocht1t2kcolr   r   r
   data_test_ix   s    

$r)   z*ignore:can't resolve package:ImportWarningc                   @   s   e Zd Zejjdd Zejjdd Zejjdd Zejjdd Z	e
d	ejjd
d Zejjejddejdedddd Zdd ZdS )TestSAS7BDATc                 C   sF   |\}}|D ]4}t j|d| d}tj|dd}t|| qd S Ntest	.sas7bdatutf-8encoding)r   r   r   r   read_sastmassert_frame_equalselfr   r)   df0r!   r'   r"   r#   r   r   r
   test_from_file+   s
    zTestSAS7BDAT.test_from_filec              
   C   s   |\}}|D ]x}t j|d| d}t|d}| }W 5 Q R X t|}	tj|	dddd}
|
 }W 5 Q R X t	j
||dd	 qd S )
Nr,   r-   rbsas7bdatTr.   formatiteratorr0   FZcheck_exact)r   r   r   openreadr   BytesIOr   r1   r2   r3   )r5   r   r)   r6   r!   r'   r"   fZbytsbufrdrr#   r   r   r
   test_from_buffer3   s    
   zTestSAS7BDAT.test_from_bufferc           	   
   C   s   |\}}|D ]}t j|d| d}tj|dddV}|d}t||jddd d f  |d}t||jdd	d d f  W 5 Q R X qd S )
Nr,   r-   Tr.   r<   r0   r   r         )	r   r   r   r   r1   r?   r2   r3   r   )	r5   r   r)   r6   r!   r'   r"   rC   r#   r   r   r
   test_from_iteratorA   s    

zTestSAS7BDAT.test_from_iteratorc                 C   sJ   |\}}|D ]8}t tj|d| d}tj|dd}t|| qd S r+   )r   r   r   r   r   r1   r2   r3   r4   r   r   r
   test_path_pathlibL   s
    zTestSAS7BDAT.test_path_pathlibzpy.pathc           	      C   sV   ddl m} |\}}|D ]8}|tj|d| d}tj|dd}t|| qd S )Nr   )localr,   r-   r.   r/   )	Zpy.pathrJ   r   r   r   r   r1   r2   r3   )	r5   r   r)   Z	LocalPathr6   r!   r'   r"   r#   r   r   r
   test_path_localpathT   s    z TestSAS7BDAT.test_path_localpath	chunksize)rF   rG   
      r'   r      c              	   C   s`   t j|d| d}tj||dd"}d}|D ]}||jd 7 }q0W 5 Q R X ||jks\td S )Nr,   r-   r.   )rL   r0   r   )r   r   r   r   r1   r   	row_countAssertionError)r5   r   r'   rL   r"   rC   yxr   r   r
   test_iterator_loop_   s    zTestSAS7BDAT.test_iterator_loopc              	   C   sx   t j|d}tj|dddd}||jd }W 5 Q R X tj|ddd}||jd }W 5 Q R X t|| d S )Ntest1.sas7bdatr9   Tr.   r:      rE   )	r   r   r   r   r1   r?   rP   r2   r3   )r5   r   r"   rC   d1Zd2r   r   r
   test_iterator_read_too_muchk   s       z(TestSAS7BDAT.test_iterator_read_too_muchN)__name__
__module____qualname__pytestmarkZslowr7   rD   rH   rI   tdZ
skip_if_norK   parametrizer   rT   rX   r   r   r   r
   r*   )   s    




		r*   c           
   	   C   s   | dddd}t |}t j|dd}|jD ]2}z|| jd||< W q, tk
r\   Y q,X q,t|| ddlm	} t
||d	d
}| }W 5 Q R X t|j|jD ]\}}	||	 kstqd S )Nr   r   r   rU   r.   r/   r   )SAS7BDATReaderF)Zconvert_header_text)r   r1   columnsstrdecodeAttributeErrorr2   r3   Zpandas.io.sas.sas7bdatr`   
contextlibclosingr?   ziprQ   )
r	   r"   Zdf1Zdf2r(   r`   rC   Zdf3rS   rR   r   r   r
   test_encoding_optionsx   s    

rh   c                 C   sj   | dddd}t j|dd}| dddd}t j|dgd	}d
dddg}|| tj||< t|| d S )Nr   r   r   zproductsales.sas7bdatr.   r/   zproductsales.csvZMONTHZparse_datesZACTUALZPREDICTZQUARTERZYEARr   r1   r   r   r   r   r2   r3   )r	   r"   r#   r6   Zvnr   r   r
   test_productsales   s    rk   c                 C   sL   | dddd}t |}| dddd}t |}|tj}t|| d S )Nr   r   r   ztest_12659.sas7bdatztest_12659.csvrj   r	   r"   r#   r6   r   r   r
   
test_12659   s    

rm   c                 C   sP   | dddd}t |}| dddd}t |}|tj}tj||dd d S )Nr   r   r   zairline.sas7bdatzairline.csvFr=   rj   rl   r   r   r
   test_airline   s    

rn   c                 C   sr   | dddd}t |}| dddd}t j|dddd	d
gd}|jd d df jd||jd < t|| d S )Nr   r   r   zdatetime.sas7bdatzdatetime.csvZDate1ZDate2DateTimeZ
DateTimeHiZTaiwri   rF   us)	r   r1   r   r   dtroundra   r2   r3   rl   r   r   r
   test_date_time   s    
 $rs   columnZWGTZCYLc                 C   sD   | dddd}t j|dd}|| }||  }tj||dd d S )	Nr   r   r   zcars.sas7bdatlatin-1r/   Tr=   )r   r1   rr   r2   Zassert_series_equal)r	   rt   r"   r#   resultexpectedr   r   r
   test_compact_numerical_values   s
    rx   c                 C   sH   | dddd}t j|dd}| dddd}t j|dd}t|| d S )Nr   r   r   zmany_columns.sas7bdatru   r/   zmany_columns.csvr   r1   r   r2   r3   rl   r   r   r
   test_many_columns   s
    rz   c                 C   s0   | dddd}t j|dd}t|dks,td S )Nr   r   r   zload_log.sas7bdatru   r/   i1  r   r1   lenrQ   r	   r"   r#   r   r   r
    test_inconsistent_number_of_rows   s    r~   c              	   C   s6   | dddd}t jtdd t| W 5 Q R X d S )Nr   r   r   zzero_variables.sas7bdatzNo columns to parse from filematch)r\   raisesr   r   r1   )r	   r"   r   r   r
   test_zero_variables   s    r   c                 C   sD   | dddd}t |}t dddgjd d }t|| d S )	Nr   r   r   zzero_rows.sas7bdatag      ?)Z
char_fieldZ	num_fieldr   )r   r1   	DataFramer   r2   r3   )r	   r"   rv   rw   r   r   r
   test_zero_rows   s    
r   c              	   C   s:   | dddd}d}t jt|d t| W 5 Q R X d S )Nr   r   r   zcorrupt.sas7bdatz4'SAS7BDATReader' object has no attribute 'row_count'r   )r\   r   rd   r   r1   )r	   r"   msgr   r   r
   test_corrupt_read   s    r   c                 C   sj   t | tr*| jtt| jdd d dS t | trbtjj	| d}|jtt|jdd d dS | S d S )N  )microsecond)Ztimestr)

isinstancer   replaceintrr   r   rb   dateutilparserparse)ts_tsr   r   r
   round_datetime_to_ms   s    
 
 r   c                 C   s  | dddd}t j|dd}|dd }z|d	 jd
|d	< W nJ t jjjjk
rh   |t	}Y n& t
k
r   |d	 t	|d	< Y nX t jddgddgtdddddddtdddddddgddgtdddtdddgdddd	ddgd}t|| d S ) Nr   r   r   max_sas_date.sas7bdat
iso-8859-1r/   c                 S   s   t | tr|  S | S Nr   rb   lstriprS   r   r   r
   <lambda>       z#test_max_sas_date.<locals>.<lambda>dt_as_dtrp   maxnormal`MBA'           ;   X> i     r      qgFA    @@textdt_as_floatr   date_as_floatdate_as_dater   r   r   r   ra   )r   r1   applymaprq   rr   _libstslibsnp_datetimeOutOfBoundsDatetimer   rd   applyr   r   r2   r3   r	   r"   r#   rw   r   r   r
   test_max_sas_date   s*    
r   c                 C   sR  dddddg}| dddd	}g }t j|d
ddD ]}|dd }z|d jd|d< W nJ t jjjjk
r   |t	}Y n& t
k
r   |d t	|d< Y nX |jddd || q0t jdgdgtdddddddgdgtdddgd|dt jdgdgtdgd gtd!gd|dg}t||D ]\}}t|| q6d S )"Nr   r   r   r   r   r   r   r   r   r   r   )r0   rL   c                 S   s   t | tr|  S | S r   r   r   r   r   r
   r   #  r   z,test_max_sas_date_iterator.<locals>.<lambda>rp   T)ZinplaceZdropr   r   r   r   r   r   r   r   r   r   r   r   r   z2019-08-01 23:59:59.999r   z
2019-08-01)r   r1   r   rq   rr   r   r   r   r   r   rd   r   Zreset_indexappendr   r   r   Z
datetime64rg   r2   r3   )r	   Z	col_orderr"   resultsr#   rw   rv   r   r   r
   test_max_sas_date_iterator  sD    


r   c                 C   s`   | dddd}t j|dd}t tddd	t jgtddd	d
dddt jgd}t|| d S )Nr   r   r   zdates_null.sas7bdatr.   r/   r   r   r   r   r   iQ> )ZdatecolZdatetimecol)r   r1   r   r   ZNaTr2   r3   r   r   r   r
   test_null_dateG  s    
r   c                 C   s,   | dddd}t |}t|dks(td S )Nr   r   r   ztest_meta2_page.sas7bdatr   r{   r}   r   r   r
   test_meta2_pageZ  s    
r   	test_filetest2.sas7bdatztest3.sas7bdatc              	      sR   t j  fdd}|t d| tjtdd t| ddd| W 5 Q R X d	S )
z@Errors in RLE/RDC decompression should propagate the same error.c                    s"   t | trtdn
 | |S d S )NzTest exception)r   r   	Exception)sizer   Zorig_np_zerosr   r
   _patched_zerosf  s    

zEtest_exception_propagation_rdc_rle_decompress.<locals>._patched_zeroszerosz^Test exception$r   r   r   r   N)r   r   setattrr\   r   r   r   r1   )r	   Zmonkeypatchr   r   r   r   r
   -test_exception_propagation_rdc_rle_decompressa  s
    r   c              	   C   s~   t |ddddd}t| }W 5 Q R X d}d}d}|||| < | d }|| tjtd	d
 t| W 5 Q R X dS )zMIllegal control byte in RLE decompressor should raise the correct ValueError.r   r   r   r   r8   r   i   i  zunknown control byter   N)	r>   	bytearrayr?   write_bytesr\   r   
ValueErrorr   r1   )Ztmp_pathr	   rA   r   Zinvalid_control_byteZpage_offsetZcontrol_byte_posZtmp_filer   r   r
   )test_exception_propagation_rle_decompresst  s    
r   c                 C   sH   | dddd}t j|dd}| dddd}t j|dd	}t|| d S )
Nr   r   r   z0x40controlbyte.sas7bdatasciir/   z0x40controlbyte.csvobject)r   ry   rl   r   r   r
   test_0x40_control_byte  s
    r   c                 C   s2   | dddd}t tj|dd}|jdks.td S )Nr   r   r   z0x00controlbyte.sas7bdat.bz2*  )rL   )r   rV   )nextr   r1   r   rQ   r}   r   r   r
   test_0x00_control_byte  s    r   )2re   r   r   r   pathlibr   Zdateutil.parserr   Znumpyr   r\   Zpandas.errorsr   Zpandas.util._test_decoratorsutilZ_test_decoratorsr^   Zpandasr   Zpandas._testingZ_testingr2   Zfixturer   r   r)   r]   filterwarningsr*   rh   rk   rm   rn   rs   r_   rx   rz   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   <module>   sN   


N
		
	
#.
	