U
    _{fF                     @   s2  d Z ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddl	Z	ddl
mZ ddlZddlZddlZddlZddlmZ ddlZddlZddlZddlmZmZ ddlmZ dd	lmZ ddlm  mZ  ddl!Z"dd
l!m#Z#m$Z$m%Z% ddl&m'Z( ddl)m  m*Z+ ddl,m-Z-m.Z. ej/dddd Z0dd Z1e	j23e	j24e5ddZ6ee	j23e6ddZ7ej/e7ddd Z8ej9:ddde;de<de=deddddge<d>dde<d>ddddd e?d j@d!d"d#e?d j@d!d$d#e?d j@d!d"d#dddddf gd%d& ZAd'd( ZBd)d* ZCd+d, ZDej9:d-ejEeCd.d/ejEe"jFd0d/ejEeje"jFejGd1d2d/ejEeje"jFd3d1d4d/ejEeje"jFd5d1d6d/gej9:d7e"jFeCgd8d9 ZHd:d; ZId<d= ZJej/d>d? ZKG d@dA dAZLG dBdC dCZMej9:dDdEgdFe#dGdHdIgfdJe"jNOdGdHdIgdKdd"ggfgdLdM ZPdNdO ZQej9:dPdQdRdSgdTdU ZRe SdVdWdX ZTG dYdZ dZejUZVd[d\ ZWd]d^ ZXd_d` ZYdadb ZZej9:dcdddedcedfdgdgdhgdidj Z[dkdl Z\dmdn Z]e j^dodp Z_ej9:dqej`ejGgdrds Zadtdu ZbdS )va<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    )arrayN)partial)Path)catch_warnings)get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer)IndexSeriesperiod_range)DayMonthEndmodule)scopec               
   C   s0   ddl m}  t  |  W  5 Q R  S Q R X d S )Nr   create_pickle_data)Z-pandas.tests.io.generate_legacy_storage_filesr   r   r    r   Z/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/pandas/tests/io/test_pickle.pycurrent_pickle_data8   s    r   c                 C   s   t |trt||  d S |dr2t| | nP|dkrb|tjkrT| tjks`tq| |kstn t	td| dtj
}|| | d S )NZsp_	timestampZassert_Z_equal)
isinstancer
   tmassert_index_equal
startswithZassert_equalpdZNaTAssertionErrorgetattrZassert_almost_equal)resultexpectedtypZ
comparatorr   r   r   compare_elementD   s    


r!   datalegacy_pickle*z*.pickle)paramsc                 C   s
   || j S N)param)requestdatapathr   r   r   r#   Y   s    s   123s   123456I         B)r-   r,      )r-      C)orderFc                 C   s   t | }t| d}||ks"tt| ttfr>|| kstnBt|tr|jdksVt|jdksdt|j	snt|j
|jfkstd S )NAr+   r.   )r	   
memoryviewtobytesr   r   bytes	bytearrayndimformat
contiguousshapenbytes)r"   r   r   r   r   r   test_flatten_bufferc   s    

r>   c           	      C   s  t  std t| }| D ]\}}| D ]\}}|| | }|dkr2|dkr2t|| |jj	|jj	ks~t
|jj	jrt
t|dk|dk |jj	}|td tdkst
|tjdd }t|tjst
|tjdddkst
|tjdd	 }t|tjst
|tjddd
kst
q4|dkr|dkrt|| t|j	tsdt
|j	t ksvt
|jdkst
t|d|d q4|dkr|dkrt|| q4|dkr|dkrt|| q4t||| q4q"d S )Nz"known failure on non-little endianZseriestsr   r+   r,   )hours)daysr@   )nanoseconds)rA   rB   indexZperiodM)Zdt_tzcatframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   pytestskipr   read_pickleitemsr   assert_series_equalrC   freqr   	normalizer   Z	Timedeltar   r   r   Zfreqstrshiftassert_frame_equalr!   )	r#   r"   r    dvdtr   r   rL   resr   r   r   test_pickles   s<    

rS   c              	   C   s*   t |d}tj| |dd W 5 Q R X d S Nwbprotocolopenpickledumpobjpathfhr   r   r   python_pickler   s    ra   c              
   C   s6   t | d"}|d t|W  5 Q R  S Q R X d S )Nrbr   )rZ   seekr[   load)r_   r`   r   r   r   python_unpickler   s    
re   pickle_writerpython)idZpandas_proto_defaultrW   Zpandas_proto_highestr0   Zpandas_proto_4   Zpandas_proto_5writerc                 C   s   | }|  D ]\}}|  D ]\}}t }||| t|}	t|	|| t|}	t|	|| t|dd}
||| |
d W 5 Q R X t|dd}
t|
}	|
d W 5 Q R X t|	|| W 5 Q R X qqd S )NrU   moder   rb   )	rJ   r   ensure_cleanr   rI   r!   re   rZ   rc   )r   rf   rj   r"   r    rP   rQ   r   r_   r   handler   r   r   test_round_trip_current   s     




ro   c                  C   s(   t  } t | jtj}t | | d S r&   )r   makeDataFrameround_trip_pathlib	to_pickler   rI   rO   dfr   r   r   r   test_pickle_path_pathlib   s    ru   c                  C   s(   t  } t | jtj}t | | d S r&   )r   rp   Zround_trip_localpathrr   r   rI   rO   rs   r   r   r   test_pickle_path_localpath   s    rv   c                   C   s   dt   dS )N__z	__.pickle)uuiduuid4r   r   r   r   get_random_path   s    rz   c                   @   sX   e Zd ZejZdd Zdd Zej	
ddddd	gd
d Zdd Zdd Zdd ZdS )TestCompressionc           	   
   C   s`  |d krt || d S |dkr.t|d}n|dkrDt|d}n|dkrtj|dtjd}|	|t
j| W 5 Q R X n|dkrt|d@}tj|dd&}||t
j|}||| W 5 Q R X W 5 Q R X nD|d	krt |d}n.|d
krtd|d}nd| }t||dkr\t|d$}| |	|  W 5 Q R X W 5 Q R X d S )Ngzipwbz2zipcompressiontarrb   rk   xzZzstdZ	zstandardrU   zUnrecognized compression type: )r   r   )shutilcopyfiler|   rZ   r~   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosr_   basenametarfile
gettarinfoaddfiler   r   
ValueErrorread)	selfZsrc_path	dest_pathr   fr`   r   tarinfomsgr   r   r   compress_file   s2     "


zTestCompression.compress_filec                 C   s   |}|d }|d }t |}t |t}t  }|j||d t j||d*}	t|d}
|
|	  W 5 Q R X W 5 Q R X tj	|d d}t 
|| W 5 Q R X W 5 Q R X d S )N.compressed.rawr   rU   )r   rm   rp   rr   decompress_filerZ   r   r   r   rI   rO   )r   r   rz   basepath1path2p1p2rt   r   r`   df2r   r   r   test_write_explicit  s    "z#TestCompression.test_write_explicitr    NonebadZ7zc              
   C   sJ   t jtdd2 t|}t }|j||d W 5 Q R X W 5 Q R X d S )NzUnrecognized compression type)matchr   )rG   Zraisesr   r   rm   rp   rr   )r   r   rz   r_   rt   r   r   r   test_write_explicit_bad/  s    z'TestCompression.test_write_explicit_badc                 C   s   |}|| }|d }| j | }t|}t|p}t }	|	| tj||d*}
t|d}|	|

  W 5 Q R X W 5 Q R X tj|d d}t|	| W 5 Q R X W 5 Q R X d S )Nr   r   rU   )_extension_to_compressiongetlowerr   rm   rp   rr   r   rZ   r   r   r   rI   rO   )r   compression_extrz   r   r   r   r   r   r   rt   r   r`   r   r   r   r   test_write_infer6  s    
"z TestCompression.test_write_inferc           
   
   C   s   |}|d }|d }t |\}t |F}t  }|j|d d | j|||d tj||d}	t ||	 W 5 Q R X W 5 Q R X d S )Nr   r   r   )r   rm   rp   rr   r   r   rI   rO   )
r   r   rz   r   r   r   r   r   rt   r   r   r   r   test_read_explicitL  s    z"TestCompression.test_read_explicitc              
   C   s   |}|d }|| }| j | }t|X}t|B}t }	|	j|d d | j|||d t	|}
t
|	|
 W 5 Q R X W 5 Q R X d S )Nr   r   )r   r   r   r   rm   rp   rr   r   r   rI   rO   )r   r   rz   r   r   r   r   r   r   rt   r   r   r   r   test_read_infer^  s    
zTestCompression.test_read_inferN)__name__
__module____qualname__icomZextension_to_compressionr   r   r   rG   markparametrizer   r   r   r   r   r   r   r   r{      s   
r{   c                   @   s*   e Zd Zejdddddgdd ZdS )	TestProtocolrX   rV   r   r+   r,   c              	   C   sF   t |2}t  }|j||d t|}t || W 5 Q R X d S )NrW   r   rm   rp   rr   r   rI   rO   )r   rX   rz   r_   rt   r   r   r   r   	test_readx  s
    
zTestProtocol.test_readN)r   r   r   rG   r   r   r   r   r   r   r   r   w  s   r   pickle_fileexcolsztest_py27.pklabcztest_mi_py27.pklr4   c                 C   s*   | ddd|}t |}t|j| d S )Nior"   r[   )r   rI   r   r   columns)r)   r   r   r_   rt   r   r   r   test_unicode_decode_error  s    
r   c               
   C   sl   t  Z} t  }t| d}|| W 5 Q R X t| d}t|}W 5 Q R X t || W 5 Q R X d S )NrU   rb   )r   rm   rp   rZ   rr   r   rI   rO   )r_   rt   r`   r   r   r   r   test_pickle_buffer_roundtrip  s    
r   mockurlzhttp://url.comzftp://test.comzhttp://gzip.comc              	      sp   dd }G dd d t  H fdd}t  }|| | d| t|}t || W 5 Q R X d S )Nc              	   S   s*   t |d}tj| |dd W 5 Q R X d S rT   rY   r]   r   r   r   ra     s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s:   e Zd ZddddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponseNreturnc                 S   s.   t |d| _d|kr ddi| _n
ddi| _d S )Nrb   r|   zContent-Encodingr   )rZ   fileheaders)r   r_   r   r   r   __init__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r&   r   r   r   r   r   	__enter__  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |    d S r&   )close)r   argsr   r   r   __exit__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   s
   | j  S r&   )r   r   r   r   r   r   r     s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   s
   | j  S r&   )r   r   r   r   r   r   r     s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.close)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s
   r   c                     s    S r&   r   )r   kwargsr   r_   r   r   mock_urlopen_read  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rm   rp   setattrr   rI   rO   )Zmonkeypatchr   ra   r   rt   r   r   r   r   test_pickle_generalurl_read  s    


r   Zfsspecc               	   C   sD   t  2 d} t  }||  t| }t || W 5 Q R X d S )Nzmemory://afiler   )r   rt   r   r   r   r   test_pickle_fsspec_roundtrip  s    


r   c                   @   s   e Zd ZddddZdS )MyTzNr   c                 C   s   d S r&   r   r   r   r   r   r     s    zMyTz.__init__)r   r   r   r   r   r   r   r   r     s   r   c                  C   sD   t tdt f} t| }t|d | d  t|d ts@td S )N)Zdtyper   r+   )r   objectr   r   round_trip_picklerK   r   r   )r   r   r   r   r   test_read_pickle_with_subclass  s    
r   c              	   C   s   t  }t   }|j|| d t| }W 5 Q R X t }|j|| d |d |	 |ksn| dksnt
tj|| d}|d t || dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r   r   )r|   r   r   N)r   rp   rm   rr   r   
read_bytesr   BytesIOrc   getvaluer   r   rI   rO   )r   rt   r_   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression  s    


r   c                 C   s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S   s   t | }t | | d S r&   )r   r   rO   )rF   	unpickledr   r   r   _test_roundtrip  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdrF   r   r   r   r   +test_pickle_dataframe_with_multilevel_index  s    
r   c                  C   sB   t dddd} ttjt| | }t|}|jj	dks>t
d S )Nz1/1/2011z1/1/2012rD   )rL   )r   r   nprandomZrandnlenr   r   rC   rL   r   )Zprngr?   Znew_tsr   r   r   "test_pickle_timeseries_periodindex  s    
r   namei	  g     H@i     )r+   r,   c                 C   s$   t t j| d}|j| ks td S )N)r   )r   r   ZmakeTimeSeriesr   r   )r   r   r   r   r   test_pickle_preserve_name  s    r   c                 C   s   t | }t ||  d S r&   r   r   rK   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimes%  s    
r   c                 C   s   t | }t ||  d S r&   r   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_strings*  s    
r   c                  C   sf   t tddjdg } t| }|jjd jdks:t	|jjd j
dksPt	t|dg |  d S )Nabccategoryr   r+   )r+   T)r   listZastypeZilocr   r   Z_mgrblocksr9   r   r<   rK   )ZserrR   r   r   r    test_pickle_preserves_block_ndim/  s
    
r   rX   c                 C   s@   t td}tt|j| |dtt j|d}t|| d S )Ni )rX   r   r   )	r   	DataFrameranger   rq   r   rr   rI   rO   )rX   r   rt   r   r   r   r   %test_pickle_big_dataframe_compression<  s    r   c               	   C   sN   t jtdd} t| d}t|}W 5 Q R X tjg g d}t	
|| d S )Nz1.2.4zempty_frame_v1_2_4-GH#42345.pklrb   )rC   r   )r   r_   joinlegacy_dirnamerZ   r[   rd   r   r   r   rO   )r_   fdrt   r   r   r   r   #test_pickle_frame_v124_unpickle_130G  s
    r   )c__doc__r   r~   datetime	functoolsr   globr|   r   r   pathlibr   r[   r   r   rx   warningsr   r   numpyr   rG   Zpandas.compatr   r   Zpandas.compat._optionalr   Zpandas.compat.compressorsr	   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r
   r   r   Zpandas._testingZ_testingr   Zpandas.io.commoncommonr   Zpandas.tseries.offsetsr   r   Zfixturer   r!   r_   r   dirname__file__r   filesr#   r   r   r8   r5   PickleBuffercastZarangeZreshaper>   rS   ra   re   r'   rr   HIGHEST_PROTOCOLro   ru   rv   rz   r{   r   Z
MultiIndexZfrom_arraysr   r   r   Z
skip_if_nor   tzinfor   r   r   r   r   r   r   r   Zskip_array_manager_invalid_testr   DEFAULT_PROTOCOLr   r   r   r   r   r   <module>   s   



	$
.
~


 
%
		 



