U
    饡c4                     @   sp  d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlZdd Zejdd Zdd Zd	d
 Zdd Zdd Zd3ddZdd Zdd Zejdd Zejdd Zejdd Z dd  Z!G d!d" d"Z"G d#d$ d$ej#j$Z%d%d& Z&ejd'd(d)d*Z'd+d, Z(d-d. Z)d/d0 Z*d1d2 Z+dS )4z
Utility functions for testing
    Nc                   C   s
   t dS )zLRandomly choose either 1 or -1.

    Returns
    -------
    sign : int
    )   )randomchoice r   r   6/tmp/pip-unpacked-wheel-qoi2rb4q/pyarrow/tests/util.pyrandsign+   s    r   c              	   c   s.   t  }t |  z
dV  W 5 t | X dS )a  Set the random seed inside of a context manager.

    Parameters
    ----------
    seed : int
        The seed to set

    Notes
    -----
    This function is useful when you want to set a random seed but not affect
    the random state of other functions using the random module.
    N)r   getstateseedsetstate)r
   Zoriginal_stater   r   r   random_seed5   s
    

r   c              	   C   s   d|   krdksn t d|dk r.tdd| |  d }t| |}|sZt|S d| d }td|}td|t||dS )	a  Generate a random decimal value with specified precision and scale.

    Parameters
    ----------
    precision : int
        The maximum number of digits to generate. Must be an integer between 1
        and 38 inclusive.
    scale : int
        The maximum number of digits following the decimal point.  Must be an
        integer greater than or equal to 0.

    Returns
    -------
    decimal_value : decimal.Decimal
        A random decimal.Decimal object with the specified precision and scale.
    r   &   z,precision must be between 1 and 38 inclusiver   zHranddecimal does not yet support generating decimals with negative scale
   z{}.{}0)	AssertionError
ValueErrorr   randintdecimalDecimalformatstrrjust)Z	precisionZscaleZmax_whole_valueZwholeZmax_fractional_valueZ
fractionalr   r   r   randdecimalK   s    
r   c                 C   s   t tjjdd| ddS )NA   {   i1)sizedtype)bytesnpr   r   )lengthr   r   r   random_asciip   s    r!   c                 C   s4   t jttjtj t jdfd}dt j	|| S )z%
    Generate one random string.
    r   )r    )
r   arrayliststringascii_lettersdigitsZstr_joinr   r   )ZncharsZRANDS_CHARSr   r   r   randst   s
     r)   c                     sJ   dd l } d | j fddtjd d D | dd t D d}|S )	Nr      c                    s   i | ]}|t j qS r   )r   r   Zrandn).0colNr   r   
<dictcomp>   s      z"make_dataframe.<locals>.<dictcomp>   c                 S   s   g | ]}t d qS )r   )r)   )r+   _r   r   r   
<listcomp>   s     z"make_dataframe.<locals>.<listcomp>)index)ZpandasZ	DataFramer%   ascii_uppercaseZIndexrange)pdZdfr   r-   r   make_dataframe}   s    r7   rss   r   r   c                    s`   ddl }| fdd    fdd}t|D ]|   | dkr>|  q>dS )a  
    Execute the function and try to detect a clear memory leak either internal
    to Arrow or caused by a reference counting problem in the Python binding
    implementation. Raises exception if a leak detected

    Parameters
    ----------
    f : callable
        Function to invoke on each iteration
    metric : {'rss', 'vms', 'shared'}, default 'rss'
        Attribute of psutil.Process.memory_info to use for determining current
        memory use
    threshold : int, default 128K
        Threshold in number of bytes to consider a leak
    iterations : int, default 10
        Total number of invocations of f
    check_interval : int, default 1
        Number of invocations of f in between each memory use check
    r   Nc                      s   t   t  S N)gcZcollectgetattrZmemory_infor   )metricprocr   r   _get_use   s    z#memory_leak_check.<locals>._get_usec                     s*     } |  kr&t d|  d S )NzDMemory leak detected. Departure from baseline {} after {} iterations)	Exceptionr   )Zcurrent_use)r?   baseline_usei	thresholdr   r   _leak_check   s     z&memory_leak_check.<locals>._leak_check)psutilProcessr5   )fr=   rC   Z
iterationsZcheck_intervalrE   rD   r   )r?   rA   rB   r=   r>   rC   r   memory_leak_check   s    rH   c                  C   sZ   t j } | dd}t jt jt jtj}|rJt j	
||f}n|}|| d< | S )N
PYTHONPATHr"   )osenvironcopygetpathabspathdirnamepa__file__pathsepr(   )envZexisting_pythonpathmodule_pathZnew_pythonpathr   r   r    get_modified_env_with_pythonpath   s    
rV   c                 G   sN   t  }tjtjt}tj|| }tj|g}|	| t
j||d d S )N)rT   )rV   rJ   rN   rP   realpathrR   r(   sys
executableextend
subprocess
check_call)script_nameargsZsubprocess_envZdir_pathZpython_filecmdr   r   r   invoke_script   s    

r`   c              	   c   sD   t j| }|t j| < z
dV  W 5 |dkr4t j| = n
|t j| < X dS )zA
    Temporarily set environment variable *name* to *value*.
    N)rJ   rK   rM   )namevalueZ
orig_valuer   r   r   changed_environ   s    


rc   c              	   c   s2   t  }t t|  z
d V  W 5 t | X d S r:   )rJ   getcwdchdirr   )rN   curdirr   r   r   
change_cwd   s
    
rg   c                   c   s"   t   z
d V  W 5 t   X d S r:   )r;   disableenabler   r   r   r   disabled_gc   s    
rj   c                 C   s$   t jdkrd| }n
d| }|S )Nntz
file:///{}z	file://{})rJ   ra   r   )rN   urir   r   r   _filesystem_uri   s    

rm   c                   @   s   e Zd Zdd Zdd ZdS )FSProtocolClassc                 C   s
   || _ d S r:   )_pathselfrN   r   r   r   __init__   s    zFSProtocolClass.__init__c                 C   s
   t | jS r:   )r   ro   rq   r   r   r   
__fspath__   s    zFSProtocolClass.__fspath__N)__name__
__module____qualname__rr   rt   r   r   r   r   rn      s   rn   c                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&S )'ProxyHandlerz
    A dataset handler that proxies to an underlying filesystem.  Useful
    to partially wrap an existing filesystem with partial changes.
    c                 C   s
   || _ d S r:   )_fs)rq   fsr   r   r   rr   	  s    zProxyHandler.__init__c                 C   s   t |tr| j|jkS tS r:   
isinstancerx   ry   NotImplementedrq   otherr   r   r   __eq__  s    
zProxyHandler.__eq__c                 C   s   t |tr| j|jkS tS r:   r{   r~   r   r   r   __ne__  s    
zProxyHandler.__ne__c                 C   s   d| j j S )Nzproxy::)ry   	type_namers   r   r   r   get_type_name  s    zProxyHandler.get_type_namec                 C   s   | j |S r:   )ry   normalize_pathrp   r   r   r   r     s    zProxyHandler.normalize_pathc                 C   s   | j |S r:   ry   get_file_info)rq   pathsr   r   r   r     s    zProxyHandler.get_file_infoc                 C   s   | j |S r:   r   )rq   selectorr   r   r   get_file_info_selector  s    z#ProxyHandler.get_file_info_selectorc                 C   s   | j j||dS )N)	recursive)ry   
create_dir)rq   rN   r   r   r   r   r   "  s    zProxyHandler.create_dirc                 C   s   | j |S r:   )ry   
delete_dirrp   r   r   r   r   %  s    zProxyHandler.delete_dirc                 C   s   | j j||dS )N)missing_dir_okry   delete_dir_contents)rq   rN   r   r   r   r   r   (  s    z ProxyHandler.delete_dir_contentsc                 C   s   | j jdddS )Nr"   T)Zaccept_root_dirr   rs   r   r   r   delete_root_dir_contents,  s    z%ProxyHandler.delete_root_dir_contentsc                 C   s   | j |S r:   )ry   delete_filerp   r   r   r   r   /  s    zProxyHandler.delete_filec                 C   s   | j ||S r:   )ry   moverq   srcdestr   r   r   r   2  s    zProxyHandler.movec                 C   s   | j ||S r:   )ry   	copy_filer   r   r   r   r   5  s    zProxyHandler.copy_filec                 C   s   | j |S r:   )ry   open_input_streamrp   r   r   r   r   8  s    zProxyHandler.open_input_streamc                 C   s   | j |S r:   )ry   open_input_filerp   r   r   r   r   ;  s    zProxyHandler.open_input_filec                 C   s   | j j||dS N)metadata)ry   open_output_streamrq   rN   r   r   r   r   r   >  s    zProxyHandler.open_output_streamc                 C   s   | j j||dS r   )ry   open_append_streamr   r   r   r   r   A  s    zProxyHandler.open_append_streamN)ru   rv   rw   __doc__rr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rx     s&   rx   c                  C   s6   t jdkrtjS tjdkr&td ndd } | S d S )N)      rk   z$test requires Python 3.8+ on Windowsc                 S   s   t t  |  d S r:   )rJ   killgetpid)signumr   r   r   raise_signalN  s    z&get_raise_signal.<locals>.raise_signal)rX   version_infosignalr   rJ   ra   pytestskip)r   r   r   r   get_raise_signalE  s    

r   Fwarn_on_full_bufferc              	   c   sj   t  \}}d }z0|d |d tj| | d}|V  W 5 |d k	rTt| |  |  X d S )NFr   )socket
socketpairr   set_wakeup_fdclosesetblockingfileno)r   rwZold_fdr   r   r   signal_wakeup_fdS  s    

 

r   c                 C   st   | dg}t j|t jt jdd}|ddkr0dS |j }| d }t||}|rh|d}t	||kS t
d	d S )
Nz	--versionutf-8stdoutstderrencodingr   r   Fz version RELEASE\.(\d+)-.*r   z+minio component older than the minimum year)r[   PopenPIPEwaitr   readresearchgroupintFileNotFoundError)	componentZminimum_year	full_argsr>   r   patternZversion_matchZversion_yearr   r   r   _ensure_minio_component_versionf  s    
 

r   c                 C   sf   t   }t   | dk rZz t| dddd| || W d S  tk
rV   t d Y qX qtdd S )Nr   aliassetmyminiozhttp://r   z+mc command could not connect to local minio)time_run_mc_commandChildProcessErrorsleepr@   )mcdiraddress
access_key
secret_keystartr   r   r   _wait_for_minio_startupv  s    
  r   c                 G   s   dd| gt | }tj|tjtjdd}|d}d|}td|  td|  td	|j   td
|j	   |dkrt
dd S )Nmcz-Cr   r   r    zCmd: z
  Return: z
  Stdout: z
  Stderr: r   zCould not run mc)r$   r[   r   r   r   r(   printr   r   r   r   )r   r^   r   r>   retvalZcmd_strr   r   r   r     s    
 

r   c              	   C   s,  t jdkrtd ztdd tdd | d }| d \}}}}d||}tj|d}tj	|rrt
| t| tj|d	}	t|	d
d}
|
| W 5 Q R X t|||| t|ddddd|	 t|dddddd t|dddddd t|ddd W n  tk
r&   td Y nX dS )ab  
    Attempts to use the mc command to configure the minio server
    with a special user limited:limited123 which does not have
    permission to create buckets.  This mirrors some real life S3
    configurations where users are given strict permissions.

    Arrow S3 operations should still work in such a configuration
    (e.g. see ARROW-13685)
    win32z*The mc command is not installed on Windowsr   i  Zminiotempdir
connectionz{}:{}zlimited-buckets-policy.jsonr   )modeZadminpolicyaddzmyminio/zno-create-bucketsuserZlimitedZ
limited123r   r   zuser=limitedmbzmyminio/existing-bucketz--ignore-existingz"Configuring limited s3 user failedN)rX   platformr   r   r   r   rJ   rN   r(   existsshutilrmtreemkdiropenwriter   r   r   )Z	s3_serverr   r   hostportr   r   r   r   Zpolicy_pathZpolicy_filer   r   r   _configure_s3_limited_user  sH    






  
  
  r   )r8   r9   r   r   ),r   
contextlibr   r;   Znumpyr   rJ   r   r   r   r   r   r%   r[   rX   r   r   ZpyarrowrQ   Z
pyarrow.fsr   contextmanagerr   r   r!   r)   r7   rH   rV   r`   rc   rg   rj   rm   rn   rz   ZFileSystemHandlerrx   r   r   r   r   r   r   r   r   r   r   <module>   sZ   

%	  
+

	
	B