U
    -eg                     @   sj  d dl m Z mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZmZmZ d dlmZmZmZmZmZmZmZmZmZmZmZ G dd deZe	j dd	 Z!e	j d
d Z"e	j dd Z#e	j dd Z$e	j dd Z%e	j dd Z&e	j dd Z'e	j dd Z(e	j dd Z)dZ*e	j dd Z+e	j dd Z,e	j dd  Z-e	j d!d" Z.e	j e	j/e	0d	d#d$e	j/e	0dd%d$e	j/e	0dd&d$e	j/e	0dd'e	j1j2d(e	j/e	0dd)e	j1j3d(e	j/e	0dd*e	j1j+d(e	j/e	0dd+d$e	j/e	0dd,d$e	j/e	0dd-d$e	j/e	0dd.d$e	j/e	0d d/d$e	j/e	0d"d0e	j1j2d(gd1d2d3 Z4e	j d4d5 Z5e	j d6d7 Z6e	j d8d9 Z7e	j d:d; Z8d<d= Z9d>d? Z:d@dA Z;dBdC Z<e	j1j2dDdE Z=dFdG Z>dHdI Z?dJdK Z@dLdM ZAdNdO ZBdPdQ ZCdRdS ZDdTdU ZEdVdW ZFdXdY ZGdZd[ ZHd\d] ZId^d_ ZJd`da ZKdbdc ZLddde ZMdfdg ZNdhdi ZOdjdk ZPdldm ZQdndo ZRe	j1je	j1SdpddeRfddqeRfdrdejTfdrdsejTfgdtdu ZUdvdw ZVdxdy ZWe	j1je	j1SdzddeRfddqeRfdrdejXfdrdsejXfgd{d| ZYe	j1je	j1Sd}ddeReRfddqeReRfdrdejTejXfdrdsejTejXfgd~d ZZdd Z[dd Z\dd Z]dd Z^dd Z_e	j1j3dd Z`e	j1j2dd Zae	j1j2dd Zbe	j1j2dd Zce	j1j+dd Zde	j1Sddedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfgdd Zee	j1Sdddddefgdd Zge	j1j2dd Zhe	j1j3dd Zidd Zjdd Zkdd Zldd Zmdd Zndd Zodd Zpdd Zqdd Zrdd ZsddĄ Zte	j1j2ddƄ Zue	j1j2ddȄ Zve	j1j2ddʄ Zwe	j1j2dd̄ Zxdd΄ Zye	j1j2ddЄ Zze	j1j2dd҄ Z{dS )    )datetimetimezone	timedeltaN)assert_file_not_found)_filesystem_uriProxyHandler_configure_s3_limited_user)FileTypeFileInfoFileSelector
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystemFSSpecHandler
copy_filesc                   @   s   e Zd Zd'ddZdd Zdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&S )(DummyHandler*   c                 C   s
   || _ d S N)_value)selfvalue r   V/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/test_fs.py__init__)   s    zDummyHandler.__init__c                 C   s   t |tr| j|jkS tS r   
isinstancer   r   NotImplementedr   otherr   r   r   __eq__,   s    
zDummyHandler.__eq__c                 C   s   t |tr| j|jkS tS r   r   r    r   r   r   __ne__1   s    
zDummyHandler.__ne__c                 C   s   dS )Ndummyr   r   r   r   r   get_type_name6   s    zDummyHandler.get_type_namec                 C   s   |S r   r   r   pathr   r   r   normalize_path9   s    zDummyHandler.normalize_pathc                 C   s   g }|D ]r}d|kr(| t|tj qd|krD| t|tj qd|kr`| t|tj qd|krv| t  qtq|S )Nfiledirnotfoundbadtype)appendr
   r	   File	DirectoryNotFoundobjectIOError)r   pathsinfor(   r   r   r   get_file_info<   s    zDummyHandler.get_file_infoc                 C   sZ   |j dkr|jrg S t|j tdtjddtdtjg}|jrV|tdtjddg7 }|S )Nsomedirsomedir/file1{   )sizesomedir/subdir1somedir/subdir1/file2i  )base_dirallow_not_foundFileNotFoundErrorr
   r	   r/   r0   	recursive)r   selectorinfosr   r   r   get_file_info_selectorL   s    


z#DummyHandler.get_file_info_selectorc                 C   s4   |dkr|dks0t n|dkr,|dks0t ntd S )Nr@   Tnon-recursiveF)AssertionErrorr3   )r   r(   r@   r   r   r   
create_dir\   s
    zDummyHandler.create_dirc                 C   s   |dkst d S )N
delete_dirrE   r'   r   r   r   rG   d   s    zDummyHandler.delete_dirc                 C   s   | dst|dkstd S )N/delete_dir_contents)strip
ValueErrorrE   )r   r(   missing_dir_okr   r   r   rJ   g   s    
z DummyHandler.delete_dir_contentsc                 C   s   d S r   r   r%   r   r   r   delete_root_dir_contentsl   s    z%DummyHandler.delete_root_dir_contentsc                 C   s   |dkst d S )Ndelete_filerH   r'   r   r   r   rO   o   s    zDummyHandler.delete_filec                 C   s   |dkst |dkst d S )N	move_frommove_torH   r   srcdestr   r   r   mover   s    zDummyHandler.movec                 C   s   |dkst |dkst d S )Ncopy_file_fromcopy_file_torH   rR   r   r   r   	copy_filev   s    zDummyHandler.copy_filec                 C   s*   d|krt |d|d}t|S )Nr,   z{0}:input_streamutf8r?   formatencodepaZBufferReaderr   r(   datar   r   r   open_input_streamz   s    zDummyHandler.open_input_streamc                 C   s*   d|krt |d|d}t|S )Nr,   z{0}:input_filerY   rZ   r^   r   r   r   open_input_file   s    zDummyHandler.open_input_filec                 C   s   d|krt |t S Nr,   r?   r]   ZBufferOutputStreamr   r(   metadatar   r   r   open_output_stream   s    zDummyHandler.open_output_streamc                 C   s   d|krt |t S rb   rc   rd   r   r   r   open_append_stream   s    zDummyHandler.open_append_streamN)r   )__name__
__module____qualname__r   r"   r#   r&   r)   r6   rC   rF   rG   rJ   rN   rO   rU   rX   r`   ra   rf   rg   r   r   r   r   r   (   s$   
r   c                    s   t t  fdddddS )Nc                    s    |    S r   as_posixptempdirr   r   <lambda>       zlocalfs.<locals>.<lambda>Tfspathfnallow_move_dirallow_append_to_filedictr   requestrp   r   ro   r   localfs   s    
r|   c                    s"   t ttt  fdddddS )Nc                    s    |    S r   rk   rm   ro   r   r   rq      rr   zpy_localfs.<locals>.<lambda>Trs   )ry   r   r   r   rz   r   ro   r   
py_localfs   s    
r}   c                 C   s   t t dd dddS )Nc                 S   s   | S r   r   rm   r   r   r   rq      rr   zmockfs.<locals>.<lambda>Trs   )ry   r   r{   r   r   r   mockfs   s    r   c                 C   s   t ttt dd dddS )Nc                 S   s   | S r   r   rm   r   r   r   rq      rr   zpy_mockfs.<locals>.<lambda>Trs   )ry   r   r   r   r~   r   r   r   	py_mockfs   s    r   c                    s   t tdd fdddddS )NTZuse_mmapc                    s    |    S r   rk   rm   ro   r   r   rq      rr   z#localfs_with_mmap.<locals>.<lambda>rs   rx   rz   r   ro   r   localfs_with_mmap   s    
r   c                 C   s"   t tt||d dd dddS )Nrt   c                 S   s   | S r   r   rm   r   r   r   rq      rr   z!subtree_localfs.<locals>.<lambda>Trs   )ry   r   str)r{   rp   r|   r   r   r   subtree_localfs   s    r   c              
   c   s   | j jd ddlm} |d \}}d}|| d| ddtd	d
dd}z|| W n8 tk
r } zt	d| d|  W 5 d }~X Y nX t
||jdddV  || d S )Ngcsr   GcsFileSystem
connectionpyarrow-filesystem/:httpT-   )secondstest-project-id)endpoint_overridescheme	anonymousZretry_time_limit
project_idzCould not create directory in z: Frs   )configpyarrowrequires
pyarrow.fsr   r   rF   OSErrorpytestskipry   __add__rG   )r{   
gcs_serverr   hostportbucketrt   er   r   r   gcsfs   s,    (r   c           	      c   sv   | j jd ddlm} |d \}}}}d}|||d||dddd	}|| t||jd
d
dV  |	| d S )Ns3r   S3FileSystemr   r   {}:{}r   T)
access_key
secret_keyr   r   allow_bucket_creationallow_bucket_deletionFrs   )
r   r   r   r   r   r[   rF   ry   r   rG   )	r{   	s3_serverr   r   r   r   r   r   rt   r   r   r   s3fs   s(    

r   c                 C   s    d}t t||d |jdddS )Nzpyarrow-filesystem/prefix/rt   Frs   )ry   r   r   )r{   r   prefixr   r   r   subtree_s3fs  s    r   a  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}c                 C   sX   | j jd t s td ddlm} |\}}}||||d}t	|dd ddd	S )
NhdfsCannot locate libhdfsr   HadoopFileSystem)r   userc                 S   s   | S r   r   rm   r   r   r   rq   4  rr   zhdfs.<locals>.<lambda>Trs   )
r   r   r   r]   have_libhdfsr   r   r   r   ry   )r{   hdfs_connectionr   r   r   r   rt   r   r   r   r   '  s    

r   c                    s4   t d}|d}ttt| fdddddS )Nfsspecr*   c                    s    |    S r   rk   rm   ro   r   r   rq   @  rr   z#py_fsspec_localfs.<locals>.<lambda>Trs   )r   importorskip
filesystemry   r   r   r{   rp   r   rt   r   ro   r   py_fsspec_localfs:  s    



r   c                 C   sH   t jddd}|jdkr"t d |d}ttt|dd d	d	d
S )Nr   z0.7.5)Z
minversionz0.8.5z,Bug in fsspec 0.8.5 for in-memory filesystemZmemoryc                 S   s   | S r   r   rm   r   r   r   rq   O  rr   z$py_fsspec_memoryfs.<locals>.<lambda>Trs   )r   r   __version__r   r   ry   r   r   r   r   r   r   py_fsspec_memoryfsF  s    



r   c           	      c   st   t d}|d \}}}}d}|j||td||dd}tt|}|| t||jddd	V  |	| d S )
Nr   r   r   zhttp://{}:{})Zendpoint_url)keysecretZclient_kwargsFTrs   )
r   r   r   ry   r[   r   r   rF   r   rG   )	r{   r   r   r   r   r   r   r   rt   r   r   r   py_fsspec_s3fsU  s"    

r   zLocalFileSystem())idzLocalFileSystem(use_mmap=True)z$SubTreeFileSystem(LocalFileSystem())r   )r   Zmarksr   r   z_MockFileSystem()z-PyFileSystem(ProxyHandler(LocalFileSystem()))z-PyFileSystem(ProxyHandler(_MockFileSystem()))z5PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))z8PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))z0PyFileSystem(FSSpecHandler(s3fs.S3FileSystem())))paramsc                 C   s   | j S r   )paramr~   r   r   r   filesystem_configl  s    7r   c                 C   s   |d S Nrt   r   r{   r   r   r   r   rt     s    rt   c                 C   s   |d S )Nru   r   r   r   r   r   ru     s    ru   c                 C   s   |d S )Nrv   r   r   r   r   r   rv     s    rv   c                 C   s   |d S )Nrw   r   r   r   r   r   rw     s    rw   c                 C   sz   t | jtstt | jts t| jdks.t| jt| j d ksLt| jj	}|d k	s`t|
d tdksvtd S )Nr   g    eA)r   mtimer   rE   mtime_nsintr   Zapprox	timestamptzinfo	utcoffsetr   )	file_infor   r   r   r   check_mtime  s    
r   c                 C   s    | j d kst| jd kstd S r   )r   rE   r   r   r   r   r   check_mtime_absent  s    r   c                 C   s    | j d krt|  nt|  d S r   )r   r   r   r   r   r   r   check_mtime_or_absent  s    

r   c                 C   s   | j dkrtjdd d S )Npy::fsspec+s3zNot working with fsspec's s3fsreason)	type_namer   xfailrt   r   r   r   skip_fsspec_s3fs  s    
r   c              	   C   s   ddl m} t| t | d \}}}}|ddd||dd}|d	 tjtj	d
d |d W 5 Q R X tjtj	dd |
d W 5 Q R X d S )Nr   r   r   ZlimitedZ
limited123r   r   )r   r   r   r   zexisting-bucket/testzBucket 'new-bucket' not foundmatchz
new-bucketzWould delete bucketzexisting-bucket)r   r   r   _minio_limited_policyr[   rF   r   raisesr]   ArrowIOErrorrG   )r   r   r   r   _rt   r   r   r   +test_s3fs_limited_permissions_create_bucket  s    


r   c                  C   s2  t dtj} td}|jdks$t|jdks2t|jt	j
ksBt|jd ksPtt| tdt	jddd}|jdksxt|jdkst|jt	jkst|jdkst|jd	kstt| td
t	j| d}|jd
kst|jd
kst|jt	jkst|jd kst|j| kst|jdks&tt| d S )NR]foo/barbarzfoo/baz.txtr9   g  `A)typer:   r   zbaz.txtl    t<|+. foo)r   r   l    4 |+. )r   fromtimestampr   utcr
   r(   rE   	base_namer   r	   Unknownr:   r   r/   r   r   r0   r   )dtr5   r   r   r   test_file_info_constructor  s0    
r   c                	   C   s    t t t  W 5 Q R X d S r   )r   r   	TypeErrorr   r   r   r   r   'test_cannot_instantiate_base_filesystem	  s    r   c               	   C   s   t  } t  }t }| | s t| |s.ttt | d W 5 Q R X | |   krb|kshn t| dkstt||kst|t ksttd| td| ksttd| td|ksttd| td| kstd S )Nstring   /basez/other)r   r   equalsrE   r   r   r   r   )Zfs0fs1fs2r   r   r   test_filesystem_equals  s    r   c                  C   s|   t  } td| }|jdkst|j| ks,tt|ds>ttdt  }|jdksXt|j| ksftt|dsxtd S )Nr   z/base/zHSubTreeFileSystem(base_path=/base/, base_fs=<pyarrow._fs.LocalFileSystemz/another/base/zPSubTreeFileSystem(base_path=/another/base/, base_fs=<pyarrow._fs.LocalFileSystem)r   r   	base_pathrE   Zbase_fsrepr
startswith)r|   Zsubfsr   r   r   test_subtree_filesystem"  s    
r   c                 C   sT   | j dd dkr tjdd t| }t|}t|tsBt	|
| sPt	d S )N::mock"MockFileSystem is not serializabler   )r   splitr   r   pickledumpsloadsr   r   rE   r   )rt   
serializedrestoredr   r   r   test_filesystem_pickling2  s    

r   c           
   	   C   s   | j dd dkr tjdd t|  |d}|d}|d}| | | | W 5 Q R X | |}|d	 W 5 Q R X t	t
| }||||g\}}}	|jtjkst|jtjkst|	jtjkstd S )
Nr   r   r   r   r   	a/aa/aaa/a/bbc.txt   test)r   r   r   r   r   rF   rf   writer   r   r   r6   r   r	   r0   rE   r/   )
rt   ru   aaabbcfpr   aaa_infobb_infoc_infor   r   r   ,test_filesystem_is_functional_after_pickling<  s     

r  c                  C   s,   t  } | jdkstt } | jdks(td S )Nlocalr   )r   r   rE   r   r   r   r   r   test_type_nameR  s    r  c                 C   s   |  ddkstd S )Nr   )r)   rE   r   r   r   r   test_normalize_pathY  s    r  c              
   C   s\   G dd d}dd| t  i g dd t g}|D ]$}tt | | W 5 Q R X q2d S )Nc                   @   s   e Zd ZdS )z-test_non_path_like_input_raises.<locals>.PathN)rh   ri   rj   r   r   r   r   Path`  s   r     g?c                   S   s   dS )Nr  r   r   r   r   r   rq   c  rr   z1test_non_path_like_input_raises.<locals>.<lambda>)tuplepathlibr  r   r   r   rF   )rt   r  Zinvalid_pathsr(   r   r   r   test_non_path_like_input_raises_  s    r  c              	   C   s  |d}|d}|d}|d}|  | | | W 5 Q R X | |}|d W 5 Q R X | ||||g\}}}	}
|j|kstdt|kst|jdkst| jdkr|j	t
jkstn |j	t
jkstd	t|kst|jd kstt| |jt|kst|jd
kst|jdks,t|j	t
jks>tdt|ksPt|jdks`t| jdkrtt| |	jt|kst|	jdkst|	jdkst|	j	t
jkstdt|	kst|	jdkst| jdkrt|	 |
jt|kst|
jdkst|
jdks$t|
j	t
jks6t|
jd ksFt|
jd ksVtdt|
kshtt|
 | |}|j|jkst|j	|j	kstd S )Nr   r  r  zzzr  r   r   zFileType.Directoryr  zFileType.Filer   )zpy::fsspec+memoryr   txtr   zFileType.NotFound)rF   rf   r  r6   r(   rE   r   	extensionr   r   r	   r1   r0   r:   r   r   r   r/   r   r   r   )rt   ru   r  r  r  r  r  r	  r
  r  Zzzz_infoZ	aaa_info2r   r   r   test_get_file_infoj  s\    



r  c              	   C   s,  |d}|d}|d}|d}|d}|d}z| | | | W 5 Q R X | | W 5 Q R X | | | | W 5 Q R X | | t|ddd	}|j|kst| |}	| jd
krt|	dkst|	dkstnt|	dkst|	D ]}
|
j	
|s$|
j	
|s$|
j	
|r8|
jtjkstn|
j	d
|s`|
j	d
|rt|
jtjkstnD| jd
kr|
j	d
dr|
jtjkstntd|
j	t|
 qt|dd}| |}	| jd
krt|	dkst|	dkstnt|	dkstW 5 |  | X d S )Nzselector-dir/zselector-dir/test_file_azselector-dir/test_file_bzselector-dir/test_dir_az#selector-dir/test_dir_a/test_file_czselector-dir/test_dir_bFT)r>   r@   r   r      rI   zselector-dirzunexpected path {}r@         )rG   rF   rf   r   r=   rE   r6   r   lenr(   endswithr   r	   r/   rstripr0   rL   r[   r   )rt   ru   r=   Zfile_aZfile_bZdir_aZfile_cZdir_brA   rB   r5   r   r   r    test_get_file_info_with_selector  sZ    










"r!  c              	   C   sj   t |  |d}ttj | | W 5 Q R X | | | | |d}| j|dd | | d S )Nztest-directory/zdeeply/nested/test-directory/Tr  )r   r   r   r]   r   rG   rF   )rt   ru   dr   r   r   test_create_dir  s    

r#  c              	   C   st   t |  |d}|d}| | | | ttj | | W 5 Q R X ttj | | W 5 Q R X d S )N
directory/directory/nested/)r   rF   rG   r   r   r]   r   rt   ru   r"  ndr   r   r   test_delete_dir  s    

r(  c              	   C   s   t |  |d}|d}| | | | ttj | | W 5 Q R X | j|dd ttj | | W 5 Q R X | | ttj | | W 5 Q R X d S )Nr$  r%  T)rM   )r   rF   rJ   r   r   r]   r   rG   r&  r   r   r   test_delete_dir_contents  s    


r)  c              	   C   s   | d }| d }|d}|d}| | ttj |d W 5 Q R X ttj |d W 5 Q R X ttj |d W 5 Q R X |jddd	 |jddd	 |jddd	 ttj || W 5 Q R X d S )
Nrt   ru   r$  r%  r  rI   //TZaccept_root_dir)rF   r   r   r]   ArrowInvalidrJ   r   rG   )r   rt   ru   r"  r'  r   r   r   _check_root_dir_contents  s     
r-  c                 C   s   t |  t | d S r   )r-  )r   r   r   r   r   test_delete_root_dir_contents-  s    r.  c              	   C   sJ   |d}|d}|  | W 5 Q R X | || | | | | d S )Nztest-copy-source-fileztest-copy-target-file)rf   rX   rO   rt   ru   str   r   r   test_copy_file2  s    

r2  c              	   C   s   |d}|d}|  | |rX| || ttj | | W 5 Q R X | | n$ttj | || W 5 Q R X d S )Nzsource-dir/ztarget-dir/)rF   rU   r   r   r]   r   rG   )rt   ru   rv   r0  r1  r   r   r   test_move_directory>  s    
r3  c              	   C   sj   t |  |d}|d}| | W 5 Q R X | || ttj | | W 5 Q R X | | d S )Nztest-move-source-fileztest-move-target-file)r   rf   rU   r   r   r]   r   rO   r/  r   r   r   test_move_fileO  s    
r4  c              	   C   s   |d}|  | W 5 Q R X | | ttj | | W 5 Q R X |d}| | |d}|  |}|d W 5 Q R X | | d S )Nztest-delete-target-fileztest-delete-nestedztest-delete-nested/target-file   data)	rf   rO   r   r   r]   r   rF   r  rG   )rt   ru   rn   r"  fr0  r   r   r   test_delete_file`  s    


r7  c                 C   s   | S r   r   )vr   r   r   identityr  s    r9  )compressionbuffer_size
compressor@   gzip   c           	   	   C   sf   |d}dd }|  |}||| W 5 Q R X | |||}| }W 5 Q R X ||ksbtd S )Nzopen-input-streams   some data for reading
   )rf   r  r`   readrE   )	rt   ru   r:  r;  r<  rn   r_   r0  resultr   r   r   test_open_input_streamv  s    rC  c              	   C   s   |d}dd }|  |}|| W 5 Q R X tdd }| |}| }W 5 Q R X ||ksft| |}|| | }W 5 Q R X |||d  kstd S )Nzopen-input-file	   some data   r@  )rf   r  r  ra   rA  rE   seek)rt   ru   rn   r_   r0  Z	read_fromr6  rB  r   r   r   test_open_input_file  s    
rG  c              	   C   s,   |d}t t | | W 5 Q R X d S )Nzopen-input-stream-not-found)r   r   r?   r`   )rt   ru   rn   r   r   r    test_open_input_stream_not_found  s    rH  )r:  r;  decompressorc              	   C   sh   |d}dd }|  |||}|| W 5 Q R X | |||}|t||ksZtW 5 Q R X d S )Nzopen-output-streams   some data for writingrE  )rf   r  r`   rA  r  rE   )rt   ru   r:  r;  rI  rn   r_   r6  r   r   r   test_open_output_stream  s    rJ  )r:  r;  r<  rI  c              	   C   s   |d}|d}|  |}	|	| W 5 Q R X |r| j|||d}
|
d W 5 Q R X | |}
|
 }W 5 Q R X ||}|dkstn(ttj	 | j|||d W 5 Q R X d S )Nzopen-append-streams   already existing)r:  r;  s   
newly addeds   already existing
newly added)
rf   r  rg   r`   rA  rE   r   r   r]   ZArrowNotImplementedError)rt   ru   r:  r;  r<  rI  rw   rn   initialr0  r6  rB  r   r   r   test_open_append_stream  s$    rL  c           	   	   C   s   |d}ddi}d}| j ||d}|| W 5 Q R X | |}| |ksTt| }W 5 Q R X | jdkszd| jkr| D ]\}}|| | kstqn|i kstd S )Nzopen-output-stream-metadataContent-Typezx-pyarrow/testrD  )re   )r   r   r   )	rf   r  r`   rA  rE   re   r   itemsr\   )	rt   ru   rn   re   r_   r6  Zgot_metadatakr8  r   r   r    test_open_output_stream_metadata  s    rP  c                	   C   s.   t dd tt t dd W 5 Q R X d S )NFr   )xxx)r   r   r   r   r   r   r   r   test_localfs_options  s    
rR  c              	   C   s   | d }t   |d W 5 Q R X t   |d W 5 Q R X t   |jddd W 5 Q R X t   |d W 5 Q R X t   |d W 5 Q R X t   |dd W 5 Q R X t   |dd W 5 Q R X d S )Nrt   z/non/existent/filez/non/existent/dirFr  z/non/existentz/xxx)r   r`   rf   rF   rG   rO   rU   rX   )r|   rt   r   r   r   test_localfs_errors  s    rS  c                 C   s   | d }t t}|j}|| | g\}}|j| jksFt	|j
| jksZt	t| |j
| jksvt	t| d S r   )r  r  __file__parentr6   rl   r:   statst_sizerE   r   st_mtime_nsr   )r|   rt   	file_pathZdir_pathr   dir_infor   r   r   test_localfs_file_info  s    


r[  c              	   C   sL   t dtj}t|}|d W 5 Q R X |dg\}|j|ksHtd S )Nr   r   )	r   r   r   r   r   rf   r6   r   rE   )r   r   rt   r5   r   r   r   test_mockfs_mtime_roundtrip  s    
r\  c               	   C   s  ddl m}  t }| dd|ddddd	}t|| s8t|jdksFt|jdksTtt	t
||kslt|  }t|| stt	t
||kst| d
d}t|| stt	t
||kst| dddd}t|| stt	t
||ksttt | dd W 5 Q R X tt | d
dd W 5 Q R X tt | d
dd W 5 Q R X tt | t d W 5 Q R X d S )Nr   r   abczservice_account@apachezus-west2httpslocalhost:8999r   )access_tokentarget_service_accountcredential_token_expirationdefault_bucket_locationr   r   r   Tr   authenticated-read
text/plainZACLrM  default_metadataaccess)r`  r   )r   r`  Zacct)r   ra  )rb  )r   r   r   nowr   rE   rc  r   r   r   r   r   r   rL   )r   r   rt   r   r   r   test_gcs_options&  sB     

rl  c               	   C   s  ddl m} m}m}m} |ddddddd	}t||s:t|jdksHtt	t
||ks`t|d
dddd}t||s~tt	t
||kst||ddd}t||st|| ddd}t||st|d
d}t||stt	t
||kst||kst|dd}t||s,tt	t
||ksFt|dd}t||s`tt	t
||kszt|ddddd}t||stt	t
||kst||kst|ddd}t||stt	t
||kst|ddd}t||stt	t
||ks0t|ddd}t||sLtt	t
||ksft||kstttt |dd W 5 Q R X tt |dd W 5 Q R X tt |ddd  W 5 Q R X tt |ddd! W 5 Q R X tt |ddd"d# W 5 Q R X tt |dddd$ W 5 Q R X tt |d"dd% W 5 Q R X tt |d&d'gd( W 5 Q R X tt || d W 5 Q R X d S ))Nr   )AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyr   S3RetryStrategyrj  r   token	us-east-2r^  r_  )r   r   session_tokenregionr   r   Zrolesessionr   d   )role_arnZsession_nameZexternal_idZload_frequencyr  )Zmax_attempts)Zretry_strategy)rv  Trd  )background_writesre  rf  rg  )rw  ri  )r   r   g      ?g      ?)request_timeoutconnect_timeout)r   )r   )r   rr  )r   rr  Zarn)r   r   rv  )r   r   r   )rv  r   r   r   rh  )r   rm  rn  r   ro  r   rE   rs  r   r   r   r   r   rL   )rm  rn  r   ro  rt   r   r   r   r   test_s3_optionsL  s       


    rz  c           	   	   C   sr  ddl m} dddd}d}ddd	d}d
}||d}t||sDttt||ks\t||d}t||stttt||kst||d}t||sttt||kst||d}t||sttt||kst||d}||d}||ksttt||ks(ttt||ksBt||d}||d}||ksdttt||ks~ttt||kst||d}||d}||ksttt||ksttt||kst||d}||d}||ksttt||ks*ttt||ksDt||d}||d}||ksfttt||ksttt||kst||d}||d}||ksttt||ksttt||kst||d}||d}||ksttt||ks,ttt||ksFt||d}||d}||kshttt||ksttt||kst||d}||d}||ksttt||ksttt||kst||d}||d}||ksttt||ks.ttt||ksHt||d}| }||ksfttt||ksttt||kst||d}| }||ksttt||ksttt||kst||d}| }||ks
ttt||ks$ttt||ks>t||d}| }||ks\ttt||ksvttt||ksttt	 |dd W 5 Q R X tt
 |dddd W 5 Q R X tt
 |dddd W 5 Q R X tt
 |dddd W 5 Q R X ttj |dd W 5 Q R X ttj |ddddd W 5 Q R X d S )Nr   r   r   	localhosti'#  )r   r   r   zhttp://localhost:8999r^  i  zhttps://localhost:8080)Zproxy_options)r   r{  #  r|  )r   r   )r   r   )r   r   zhtttps://localhost:9000Zhtttp)r   r   r   rE   r   r   r   r   r   r   KeyErrorr]   r,  )	Zmonkeypatchr   Zproxy_opts_1_dictZproxy_opts_1_strZproxy_opts_2_dictZproxy_opts_2_strrt   r   r   r   r   r   test_s3_proxy_options  s    



























r~  c               	   C   sl   ddl m}  | ddd}d}tjt|d}|d W 5 Q R X d	t|jkrRd S | d
dd}|d d S )Nr   r   z
eu-north-1T)rs  r   a  When getting information for bucket 'voltrondata-labs-datasets': AWS Error UNKNOWN \(HTTP status 301\) during HeadBucket operation: No response body. Looks like the configured region is 'eu-north-1' while the bucket is located in 'us-east-2'.|NETWORK_CONNECTIONr   voltrondata-labs-datasetsZNETWORK_CONNECTIONrq  )r   r   r   r   r   r6   r   r   )r   rt   msgexcr   r   r   test_s3fs_wrong_region-  s    r  c                 C   s  ddl m} t std | \}}}d}d}d}d}|||d|||d	}	||||d|||}
||||d
|||}||||d
|d ||}|||}|d||}|||dd}|||ddd}|||dt	dd}|||ddd}|||ddddid}|	|
ks"t
||ks0t
||ks>t
|
|ksLt
||ksZt
||ksht
|
|ksvt
||kst
||kst
||kst
||kst
||kst
tt |  W 5 Q R X tt |d W 5 Q R X |	|
|||||||||fD ] }tt||kst
q| \}}}||||d}|tds`t
|d|||}|tdst
d S )Nr   r   r   r  i   i   zIhdfs://{}:{}/?user={}&replication={}&buffer_size={}&default_block_size={}Zlibhdfs)r   replicationr;  default_block_sizemer  zhdfs://{}:{}Z	localuser)r   
cache_path)r   kerb_ticketZcache_path2Z
hdfs_tokenabcd)r   r  Z
extra_confr  rI   zhdfs://{}:{}/?user={})r   r   r]   r   r   r   from_urir[   r  r  rE   r   r   r   r   r   r6   r   )r   r   r   r   r   r  r;  r  uriZhdfs1Zhdfs2Zhdfs3Zhdfs4Zhdfs5Zhdfs6Zhdfs7Zhdfs8Zhdfs9Zhdfs10Zhdfs11rt   r   r   r   r   test_hdfs_optionsE  s    

                
  
r  )r  expected_klassexpected_pathzmock:r  zmock:foo/barr   zmock:/foo/barzmock:///foo/barzmock:///some%20path/%C3%A9u   some path/ézfile:/rI   zfile:///zfile:/foo/barz/foo/barzfile:///foo/barzfile:///some%20path/%C3%A9u   /some path/éu   /some path/%20éc                 C   s,   t | \}}t||st||ks(td S r   )r   r  r   rE   )r  r  r  rt   r(   r   r   r   test_filesystem_from_uri  s    r  r(   c                 C   sB   t | }t|\}} t|ts&t| |  	 ks>td S r   )
r  r  r   r  r   r   rE   resolveabsoluterl   )r(   rn   rt   r   r   r    test_filesystem_from_path_object  s    
r  c           
      C   s   ddl m} | d \}}}}d||||}t|\}}t||sHt|dksTt|| ||g\}	|	j	|kszt|	j
tjkstd S )Nr   r   r   zZs3://{}:{}@mybucket/foo/bar?scheme=http&endpoint_override={}:{}&allow_bucket_creation=Truemybucket/foo/bar)r   r   r[   r   r  r   rE   rF   r6   r(   r   r	   r0   )
r   r   r   r   r   r   r  rt   r(   r5   r   r   r   test_filesystem_from_uri_s3  s       
r  c                 C   s   ddl m} | d \}}dd| d| d d }t|\}}t||sNt|d	ksZt|| ||g\}|j|kst|j	t
jkstd S )
Nr   r   r   zgs://anonymous@z/mybucket/foo/bar?scheme=http&endpoint_override=r   &z0retry_limit_seconds=5&project_id=test-project-idr  )r   r   r   r  r   rE   rF   r6   r(   r   r	   r0   )r   r   r   r   r  rt   r(   r5   r   r   r   test_filesystem_from_uri_gcs  s    
r  c               	   C   sZ   t  } t| }t|tst|jdks*t|j| ks8ttt td  W 5 Q R X d S Nz	py::dummy)	r   r   r   rE   r   handlerr   r   r   )r  rt   r   r   r   test_py_filesystem  s    r  c                  C   s   t d} t d}t d}t| }t| }t|}t|}||k	sDt||k	sPt||k	s\t||ksht||kstt||kst|t kst|t kstd S )Nr  r  )r   r   rE   r   r2   )Zhandler1Zhandler2Zhandler3r   r   Zfs3Zfs4r   r   r   test_py_filesystem_equality  s    r  c                  C   s\   t  } t| }t|}t|}t|ts0t||ks<t|j| ksJt|j	dksXtd S r  )
r   r   r   r   r   r   r   rE   r  r   )r  rt   r   r   r   r   r   test_py_filesystem_pickling  s    

r  c                  C   s   t  } t| }t|tstt| }d } | d k	s8td }| d ksJtt  } t| }t| }d } | |jksvt| d k	std }| d kstd S r   )r   r   r   rE   weakrefrefr  )r  rt   wrr   r   r   test_py_filesystem_lifetime  s     

r  c               	   C   s   t  } t| }|dg\}|jdks*t|jtjks:t|dg\}|jdksVt|jtjksft|dg\}|jdkst|jtj	kstt
t |dg W 5 Q R X t
t |dg W 5 Q R X d S )Nzsome/dirz	some/filer,   r-   rQ  )r   r   r6   r(   rE   r   r	   r0   r/   r1   r   r   r   r3   )r  rt   r5   r   r   r    test_py_filesystem_get_file_info  s    r  c               	   C   sL  t  } t| }tdd}||}t|dks2t|d jdksDt|d jtj	ksXt|d j
dksjt|d jdks|t|d jtjkst|d j
d ksttdd	d
}||}t|dkst|d jdkst|d jdkst|d jdksttdd}tt || W 5 Q R X tdd	d}||g ksHtd S )Nr7   )r=   r  r   r8   r9   r  r;   T)r=   r@   r  r<   r,   )r=   r>   )r   r   r   r6   r  rE   r(   r   r	   r/   r:   r0   r   r   r?   )r  rt   rA   rB   r   r   r   )test_py_filesystem_get_file_info_selector/  s,    



r  c               
   C   s   t  } t| }|jddd |jddd tt |d W 5 Q R X |d |d d	D ]2}tt || W 5 Q R X |j|dd
 qb|	d |
dd |dd d S )Nr@   Tr  rD   FZfoobarrG   rJ   )r  rI   r*  r+  rO   rP   rQ   rV   rW   )r   r   rF   r   r   r3   rG   rJ   rL   rO   rU   rX   )r  rt   r(   r   r   r   test_py_filesystem_opsL  s    


r  c               	   C   sT   t t } | d}| dks&tW 5 Q R X tt | d W 5 Q R X d S )Nsomefiles   somefile:input_streamr,   )r   r   r`   rA  rE   r   r   r?   rt   r6  r   r   r   test_py_open_input_stream`  s
    
r  c               	   C   sT   t t } | d}| dks&tW 5 Q R X tt | d W 5 Q R X d S )Nr  s   somefile:input_filer,   )r   r   ra   rA  rE   r   r   r?   r  r   r   r   test_py_open_input_filei  s
    
r  c               	   C   s.   t t } | d}|d W 5 Q R X d S Nr  r5  )r   r   rf   r  r  r   r   r   test_py_open_output_streamr  s    
r  c               	   C   s.   t t } | d}|d W 5 Q R X d S r  )r   r   rg   r  r  r   r   r   test_py_open_append_streamy  s    
r  c               	   C   s   ddl m}  tjdpd}| dd}|j|ks4t| ddd}|td	}t	|dks^td
}|
|:}| }d|kst|d dkst|d dkstW 5 Q R X d S )Nr   r   ZPYARROW_TEST_S3_REGIONz	us-east-1Trd  rq  )r   rs  z"voltrondata-labs-datasets/nyc-taxizCvoltrondata-labs-datasets/nyc-taxi/year=2019/month=6/part-0.parquetrM  zLast-Modifieds   2022-07-12T23:32:00ZETags%   "4c6a76826a695c6ac61592bc30cda3df-16")r   r   osenvirongetrs  rE   r6   r   r  r`   re   )r   Zdefault_regionrt   entriesr   r6  mdr   r   r   test_s3_real_aws  s"    
r  c               	   C   s   t d\} }| jdkst| |}d|dks:tW 5 Q R X t d\} }| jdks`ttjtdd t d	 W 5 Q R X t d
\} }| jdkstd S )Nzs3://mf-nwp-models/README.txt	eu-west-1s&   Meteo-France Atmospheric models on AWS2   z.s3://mf-nwp-models/README.txt?region=us-east-2rq  zBucket '.*' not foundr   z s3://x-arrow-non-existent-bucketz1s3://x-arrow-non-existent-bucket?region=us-east-3z	us-east-3)	r   r  rs  rE   r`   rA  r   r   r3   )rt   r(   r6  r   r   r   !test_s3_real_aws_region_selection  s    r  c               	   C   st   ddl m}  | ddkst| ddks,ttjtdd | d	 W 5 Q R X tjtdd | d
 W 5 Q R X d S )Nr   resolve_s3_regionr  rq  zmf-nwp-modelsr  zNot a valid bucket namer   r   z	s3:bucket)r   r  rE   r   r   rL   r  r   r   r   test_resolve_s3_region  s    r  c              
   C   s  |d }|d }|d}| |}|d W 5 Q R X | \}}}	}
d|	 d|
 d| d| d| 
}t|d	 }t|| t }||}| dkstW 5 Q R X t|d
 }t|||d ||}| dkstW 5 Q R X t|d }t|}t|| ||}| dks$tW 5 Q R X t|d }t|||d ||}| dksftW 5 Q R X t|d }t||ddd ||}| dkstW 5 Q R X d S )Nrt   ru   r  r  zs3://r   @z?scheme=http&endpoint_override=zc_copied1.txtzc_copied2.txt)source_filesystemzc_copied3.txtzc_copied4.txt)destination_filesystemzc_copied5.txtr  F
chunk_sizeZuse_threads)	rf   r  r   r   r   r`   rA  rE   r   )Zs3_connectionr   rp   rt   ru   r(   r6  r   r   r   r   
source_uriZlocal_path1r|   Zlocal_path2Zlocal_path3destination_uriZlocal_path4Zlocal_path5r   r   r   test_copy_files  s<     

r  c              	      sT  t   | d }|   t|d }|d W 5 Q R X  t|d }|d W 5 Q R X  fdd}| d }|  tt|t| || | d	 }|  tt|t|  d
 || | d }|  tt|}tt|}t|| || | d }	|	  t||	 ||	 | d }
|
  t||
ddd ||
 d S )Nsourcefile1   test1file2   test2c              	      s`     t| d }| dks$tW 5 Q R X   t| d }| dksRtW 5 Q R X d S )Nr  r  r  r  )r`   r   rA  rE   )destination_dirr6  r|   r   r   check_copied_files  s    z5test_copy_files_directory.<locals>.check_copied_filesZdestination1Zdestination2)r  r  Zdestination3Zdestination4Zdestination5r  Fr  )r   mkdirrf   r   r  r   r   )rp   
source_dirr6  r  Zdestination_dir1Zdestination_dir2Zdestination_dir3r  r  Zdestination_dir4Zdestination_dir5r   r  r   test_copy_files_directory  sB     

r  c                  C   s   d} t tjd| g d S )Na  if 1:
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)

        fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        assert fs.region == 'eu-west-1'
        f = fs.open_input_stream(path)
        f.read(50)

        finalize_s3()

        with pytest.raises(ValueError, match="S3 .* finalized"):
            f.read(50)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            fs.open_input_stream(path)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            S3FileSystem(anonymous=True)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        -c
subprocess
check_callsys
executablecoder   r   r   test_s3_finalize$  s    r  c                  C   s   d} t tjd| g d S )Na  if 1:
        import pytest
        from pyarrow.fs import resolve_s3_region, ensure_s3_initialized, finalize_s3

        resolve_s3_region('mf-nwp-models')

        finalize_s3()

        # Testing both cached and uncached accesses
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('mf-nwp-models')
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('voltrondata-labs-datasets')
        r  r  r  r   r   r    test_s3_finalize_region_resolver@  s    r  )|r   r   r   r>  r  r  r   r  r  r   r  r   r]   Zpyarrow.tests.test_ior   Zpyarrow.tests.utilr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   Zfixturer|   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   Zlazy_fixturemarkr   r   r   rt   ru   rv   rw   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r!  r#  r(  r)  r-  r.  r2  r3  r4  r7  r9  ZparametrizecompressrC  rG  rH  
decompressrJ  rL  rP  rR  rS  r[  r\  rl  rz  r~  r  r  r  rT  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r   <module>   s  4i
	
	
	
	
	
	







6






;A

	

	

	

%
S
 

G



		



44
