U
    襡cG                     @   s4  d dl Z d dlZd dlZd dlZd dlmZmZm	Z	 zd dl
Z
W n ek
rX   e	Z
Y nX zd dlmZ W n ek
r   dd ZY nX dd Zdd	 Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Z d,d- Z!d.d/ Z"dS )0    N)SerializationContext	py_bufferbuiltin_pickle)descr_to_dtypec                 C   s  t | trt| S t | tr<t| d }t|| d fS g }d}| D ]z}t|dkrj|\}}t|}n|\}}}tt||f}|dko|jtjko|j	dk}|s|
|||f ||j7 }qHt| \}	}
}dd |	D }t| \}}	t|	|
|||dS )	ad  
        descr may be stored as dtype.descr, which is a list of (name, format,
        [shape]) tuples where format may be a str or a tuple.  Offsets are not
        explicitly saved, rather empty fields with name, format == '', '|Vn'
        are added as padding.  This function reverses the process, eliminating
        the empty padding fields.
        r          Nc                 s   s$   | ]}t |tr|nd |fV  qd S N)
isinstancetuple).0n r   9/tmp/pip-unpacked-wheel-qoi2rb4q/pyarrow/serialization.py	<genexpr>I   s     z!descr_to_dtype.<locals>.<genexpr>)namesformatstitlesoffsetsitemsize)r
   strnpdtyper   r   lentypevoidr   appendr   zip)descrdtfieldsoffsetfieldnameZ	descr_strshapeZis_padr   r   r   Znametupsr   r   r   r   r   %   s0    





 r   c                 C   s   d | }tj|tdd d S )Nz'pyarrow.{}' is deprecated as of 2.0.0 and will be removed in a future version. Use pickle or the pyarrow IPC functionality instead.   )
stacklevel)formatwarningswarnFutureWarning)r#   msgr   r   r   _deprecate_serializationO   s
    r,   c                 C   sT   | j jdkr8| jjst| } | dtjj	| j fS | 
 tjj	| j fS d S N|Ouint8)r   r   flagsc_contiguousr   ascontiguousarrayviewlibr'   dtype_to_descrtolistobjr   r   r   _serialize_numpy_array_list\   s
    
r9   c                 C   sV   | d dkr6| d j tjks t| d t| d S tj| d t | d dS d S )Nr   r.   r   )r   )r   r   r/   AssertionErrorr3   r   arraydatar   r   r   _deserialize_numpy_array_listg   s    r>   c                 C   sZ   | j jdkr<| jjs t| j} | jdtjj	
| j fS | j tjj	
| j fS d S r-   )r   r   r0   r1   r   r2   Ar3   r4   r'   r5   r6   r7   r   r   r   _serialize_numpy_matrixo   s
    r@   c                 C   sb   | d dkr@| d j tjks ttj| d t| d ddS tj| d t | d ddS d S )Nr   r.   r   F)copy)r   rA   )r   r   r/   r:   matrixr3   r   r<   r   r   r   _deserialize_numpy_matrixz   s    rC   c              	   C   s6   t  }t j|| jd}||  W 5 Q R X | S N)schema)paBufferOutputStreamRecordBatchStreamWriterrE   Zwrite_batchgetvalue)batchoutput_streamwrr   r   r   _serialize_pyarrow_recordbatch   s    rM   c              
   C   s*   t | }| W  5 Q R  S Q R X d S r	   )rF   RecordBatchStreamReaderZread_next_batchbufreaderr   r   r    _deserialize_pyarrow_recordbatch   s    rR   c                 C   s   t j| gdg}t|S )Nr   )rF   RecordBatchZfrom_arraysrM   )r;   rJ   r   r   r   _serialize_pyarrow_array   s    rT   c                 C   s   t | }|jd S Nr   )rR   columns)rP   rJ   r   r   r   _deserialize_pyarrow_array   s    rW   c              	   C   s6   t  }t j|| jd}||  W 5 Q R X | S rD   )rF   rG   rH   rE   Zwrite_tablerI   )tablerK   rL   r   r   r   _serialize_pyarrow_table   s    rY   c              
   C   s*   t | }| W  5 Q R  S Q R X d S r	   )rF   rN   Zread_allrO   r   r   r   _deserialize_pyarrow_table   s    rZ   c                 C   s   t j| t jd}t|S )N)protocol)r   dumpsHIGHEST_PROTOCOLr   )xZpickledr   r   r   _pickle_to_buffer   s    r_   c                 C   s   t | }t|S r	   )
memoryviewr   loads)r=   Zas_memoryviewr   r   r   _load_pickle_from_buffer   s    rb   c                    s6  zdd l W n tk
r"   Y d S X dd lm dfddfdd fdd} fd	d
}| jjd||d | jjdttd t	j
drt	j
jdr| jj
jjjdttd t	j
jdr| jj
jjjdttd t	j
jdr| jj
jjjdttd | jjd d d S )Nr   z{0} serialization is not supported.
Note that {0} is planned to be deprecated in pandas future releases.
See https://github.com/pandas-dev/pandas/issues/19239 for more information.c                    s,   j jr"t|  jr"td| S )NSparseDataFrame)_pandas_api
has_sparser
   rc   NotImplementedErrorr'   Zdataframe_to_serialized_dictr7   )pdpdcompatsparse_type_error_msgr   r   _serialize_pandas_dataframe   s    
zE_register_custom_pandas_handlers.<locals>._serialize_pandas_dataframec                    s
     | S r	   )Zserialized_dict_to_dataframer<   )rh   r   r   _deserialize_pandas_dataframe   s    zG_register_custom_pandas_handlers.<locals>._deserialize_pandas_dataframec                    s6   j jr"t| jr"td | j| iS )NSparseSeries)rd   re   r
   rl   rf   r'   	DataFramer#   r7   )rj   rg   rh   ri   r   r   _serialize_pandas_series   s    
zB_register_custom_pandas_handlers.<locals>._serialize_pandas_seriesc                    s    | }||j d  S rU   )rV   )r=   Zdeserialized)rk   r   r   _deserialize_pandas_series   s    zD_register_custom_pandas_handlers.<locals>._deserialize_pandas_seriesz	pd.SeriesZcustom_serializerZcustom_deserializerzpd.Indexarraysintervalz%pd.core.arrays.interval.IntervalArrayperiodz!pd.core.arrays.period.PeriodArray	datetimesz&pd.core.arrays.datetimes.DatetimeArrayzpd.DataFrame)ZpandasImportErrorZpyarrow.pandas_compatZpandas_compatregister_typeZSeriesZIndexr_   rb   hasattrcorerq   rr   ZIntervalArrayrs   ZPeriodArrayrt   ZDatetimeArrayrm   )contextrn   ro   r   )rk   rj   rg   rh   ri   r    _register_custom_pandas_handlers   sf    		  


 rz   c              	      s   t d zfdd l dd } fdd} j j j j j j j j	 j
f	D ]}| j|d|j ||d qNW n tk
r   Y nX d S )	N%register_torch_serialization_handlersr   c                 S   sH   | j r8tjj|    |    jt	| j
dS |   S d S )N)r$   )Z	is_sparserF   SparseCOOTensor
from_numpyZ_valuesdetachnumpyZ_indicesTlistr$   r7   r   r   r   _serialize_torch_tensor  s    zFregister_torch_serialization_handlers.<locals>._serialize_torch_tensorc                    sJ   t | tjr< j|  d j|  d d d df | jdS  | S d S )Nr   r   )indicesvaluessize)r
   rF   r|   Zsparse_coo_tensorto_numpyr   r$   r}   r<   torchr   r   _deserialize_torch_tensor  s    zHregister_torch_serialization_handlers.<locals>._deserialize_torch_tensorztorch.rp   )r,   r   ZFloatTensorZDoubleTensorZ
HalfTensorZ
ByteTensorZ
CharTensorZShortTensorZ	IntTensorZ
LongTensorZTensorrv   __name__ru   )serialization_contextr   r   tr   r   r   r{     s*    		     r{   c           	      C   s   dd }dd }| j tjd||d dd }d	d
 }| j tjd||d dd }dd }| j tjd||d dd }dd }| j tjd||d d S )Nc                 S   s   t | S r	   )r   r7   r   r   r   _serialize_deque5  s    zF_register_collections_serialization_handlers.<locals>._serialize_dequec                 S   s
   t | S r	   )collectionsdequer<   r   r   r   _deserialize_deque8  s    zH_register_collections_serialization_handlers.<locals>._deserialize_dequezcollections.dequerp   c                 S   s   t |  t |  fS r	   r   keysr   r7   r   r   r   _serialize_ordered_dict@  s    zM_register_collections_serialization_handlers.<locals>._serialize_ordered_dictc                 S   s   t t| d | d S Nr   r   )r   OrderedDictr   r<   r   r   r   _deserialize_ordered_dictC  s    zO_register_collections_serialization_handlers.<locals>._deserialize_ordered_dictzcollections.OrderedDictc                 S   s   t |  t |  | jfS r	   )r   r   r   default_factoryr7   r   r   r   _serialize_default_dictK  s    zM_register_collections_serialization_handlers.<locals>._serialize_default_dictc                 S   s   t | d t| d | d S )Nr   r   r   )r   defaultdictr   r<   r   r   r   _deserialize_default_dictN  s    zO_register_collections_serialization_handlers.<locals>._deserialize_default_dictzcollections.defaultdictc                 S   s   t |  t |  fS r	   r   r7   r   r   r   _serialize_counterV  s    zH_register_collections_serialization_handlers.<locals>._serialize_counterc                 S   s   t tt| d | d S r   )r   Counterdictr   r<   r   r   r   _deserialize_counterY  s    zJ_register_collections_serialization_handlers.<locals>._deserialize_counterzcollections.Counter)rv   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r   r   r   r   ,_register_collections_serialization_handlers4  s@        r   c                    s   zxddl m}m}m}mmmm   fdd}dd }| j|d||d | j|d	||d | j|d
||d W n t	k
r   Y nX d S )Nr   )
csr_matrix
csc_matrix
coo_matrixisspmatrix_cooisspmatrix_csrisspmatrix_csc
isspmatrixc                    sz   | rdt j| fS | r0dt j| fS | rHdt j| fS  | rddt j|  fS td| d d S )Ncoocsrcsc%Serialization of {} is not supported.r   )rF   r|   Z
from_scipyZSparseCSRMatrixZSparseCSCMatrixZto_coorf   r'   r7   r   r   r   r   r   r   _serialize_scipy_sparsel  s    z9_register_scipy_handlers.<locals>._serialize_scipy_sparsec                 S   sX   | d dkr| d   S | d dkr0| d   S | d dkrH| d   S | d   S d S )Nr   r   r   r   r   )Zto_scipyr<   r   r   r   _deserialize_scipy_sparse}  s    z;_register_scipy_handlers.<locals>._deserialize_scipy_sparsezscipy.sparse.coo.coo_matrixrp   zscipy.sparse.csr.csr_matrixzscipy.sparse.csc.csc_matrix)
Zscipy.sparser   r   r   r   r   r   r   rv   ru   )r   r   r   r   r   r   r   r   r   _register_scipy_handlersf  s0    $   
r   c                    sR   z8dd l   fdd} fdd}| j jd||d W n tk
rL   Y nX d S )Nr   c                    s0   t |  jrdtj| fS td jd S )Nr   r   )r
   COOrF   r|   Zfrom_pydata_sparserf   r'   r7   sparser   r   _serialize_pydata_sparse  s
    
zB_register_pydata_sparse_handlers.<locals>._serialize_pydata_sparsec                    sD   | d dkr@| d   \}} j|d d df |j| d jdS d S )Nr   r   r   )r=   coordsr$   )r   r   r   r$   )r=   Z
data_arrayr   r   r   r   _deserialize_pydata_sparse  s     zD_register_pydata_sparse_handlers.<locals>._deserialize_pydata_sparsez
sparse.COOrp   )r   rv   r   ru   )r   r   r   r   r   r    _register_pydata_sparse_handlers  s     
r   c                 C   s   | j tddd dd d | j tdd ddd	 | j td
dd	 | j tjdttd | j tjdtt	d | j t
jdttd | j t
jdttd | j t
jdttd t|  t|  t|  t|  d S )Nintc                 S   s   t | S r	   )r   r7   r   r   r   <lambda>      z:_register_default_serialization_handlers.<locals>.<lambda>c                 S   s   t | S r	   )r   r<   r   r   r   r     r   rp   c                   S   s   dS rU   r   r   r   r   r   r     r   functionT)pickler   z	np.matrixznp.arrayzpyarrow.Arrayzpyarrow.RecordBatchzpyarrow.Table)rv   r   r   r   rB   r@   rC   Zndarrayr9   r>   rF   ArrayrT   rW   rS   rM   rR   ZTablerY   rZ   r   rz   r   r   r   r   r   r   (_register_default_serialization_handlers  s\     
      r   c                 C   s   t d t|  d S )N'register_default_serialization_handlers)r,   r   r   r   r   r   r     s    r   c                  C   s   t d t } t|  | S )Ndefault_serialization_context)r,   r   r   )ry   r   r   r   r     s    r   )#r   r(   r   r   ZpyarrowrF   Zpyarrow.libr   r   r   Zcloudpickleru   Znumpy.lib.formatr   r,   r9   r>   r@   rC   rM   rR   rT   rW   rY   rZ   r_   rb   rz   r{   r   r   r   r   r   r   r   r   r   r   <module>   sB   
*	Q&2;2