U
    -ewm                     @   s   d Z ddlZddlZddlZddlZddlZedZ	e
 ZedkZejje de dZdZdZdd Zd	d
 Zdd Zejddddgdd Zdd ZdDddZejddddgdd Zejddddgdd Zdd Zejddddgdd Zejddddgdd  Zejddddgd!d" Zejddddgd#d$ Zejd%d&d'gejddddgd(d) Z ejddddgd*d+ Z!d,d- Z"d.d/ Z#d0d1 Z$d2d3 Z%d4d5 Z&d6d7 Z'd8d9 Z(d:d; Z)d<d= Z*d>d? Z+d@dA Z,eejddddgdBdC Z-dS )Ez
UNTESTED:
read_message
    Nzpyarrow.cudazlinux-x86_64z'CUDA IPC not supported in platform `%s`)reasonc                 C   s&   t d| _t t j d | _d S )Nr      )cudaContextglobal_contextget_num_devicesglobal_context1module r   X/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/test_cuda.pysetup_module.   s    r   c                 C   s   | ` d S N)r   r	   r   r   r   teardown_module3   s    r   c                	   C   sf   t j dksttjdks ttjt j d ks8ttjt	dd t t j  W 5 Q R X d S )Nr   r   z5device_number argument must be non-negative less thanmatch)
r   r   r   AssertionErrorr   device_numberr   pytestraises
ValueErrorr   r   r   r   test_Context7   s    r   sizer     c                 C   sj   t | }tj|tjd}d|| d d|  d < | }tj|tjd}tj|| |j| ksft	d S )Ndtyper         )
r   new_host_buffernp
frombufferuint8copytestingassert_equalr   r   )r   bufarrZarr_cparr2r   r   r   test_manage_allocate_free_hostB   s    
r(   c                  C   s6   t j} t d}t j| d ks"t~t j| ks2td S )N   )r   bytes_allocated
new_bufferr   )r*   cudabufr   r   r   test_context_allocate_delM   s
    
r-   hostc                 C   s   |dkr| dkst t| }|j| ks,t tj|tjd}|j| ksJt tjjdd| tjd|dd< |	 dks| dkst tj|tjd}tj
|| ||fS |dkrt| dd	\}}t| }|j| kst |j|d| d
 ||fS tddS )z5Return a host or device buffer with random data.
    r.   r   r   r      )lowhighr   r   Ndevicetargetpositionnbyteszinvalid target value)r   paallocate_bufferr   r   r    r!   randomrandintsumr#   r$   make_random_bufferr   r+   copy_from_hostr   )r   r4   r%   r&   arr_dbufr   r   r   r=   U   s$    

r=   c           
   	   C   s  t | \}}t|}|j| ks$ttj| tjd}tj	
|| tt t| W 5 Q R X t|}|j| ks|ttj| tjd}tj	
|| t| }|j| ksttj| tjd}tj	
|| |d|j}|j| ksttj| tjd}tj	
|| | dkr||| d d   tj| tjd}tj	
t|| d d  || d d  f| ||d | d   t|}|j| ksttj| tjd}tj	
|| ||| d d   tj| tjd}tj	
|| |d|jd }|j| ks*ttj| tjd}tj	
|| || d | d }|j| | d  ksxttj| tjd}tj	
|| d d  | | d }d|  d }tj|||d}|j|ksttj| tjd}tj	
||||  | t|j||d}|j|ks,ttj| tjd}tj	
||||  | tj|||d}|j|kszttj| tjd}tj	
||||  | t||||  }|j|ksttj| tjd}tj	
||||  | tj| ||d}|j|ks ttj| tjd}tj	
||||  | t| }|j| kshtt|}|||}|j|ksttj| tjd}tj	
||||  | t| }tj|tjd}	||	d d < t|}|j| ksttj| tjd}tj	
|| tj|||d}|j|ks>ttj| tjd}tj	
||||  | t|j||d}|j|ksttj| tjd}tj	
||||  | d S )	Nr   r   r      
   r   )offsetr   rC   length)r=   r   buffer_from_datar   r   r   r    copy_to_hostr!   r#   r$   r   r   BufferError
memoryviewtobytesslicer>   Zconcatenater+   r   r   )
r   r&   r%   r,   r'   Zcudabuf2Zarr3ZsoffsetZssizer?   r   r   r   test_context_device_bufferl   s    


*






rL   c              	   C   s  t }t| dd\}}|j}t| |jj }|tj||dd d < ||}|j	|j	ks\t
tj| |d}tj|| ||}|j	|j	kst
tj| |d}tj|| tjtjdd |td W 5 Q R X tjtjdd |tdd	d
g W 5 Q R X d S )Nr2   r3   r   z$buffer is not backed by a CudaBufferr   s   123z8cannot create device buffer view from .* 'numpy.ndarray'r   rA   r   )r   r=   r   r   r   itemsizer   r    Zbuffer_from_objectr   r   rG   r#   r$   r   r   r8   ZArrowTypeError	py_bufferarray)r   ctxr&   cbufr   hbufZcbuf2r'   r   r   r   test_context_from_object   s*    

rS   c               	   C   s   t } ttj}d}t||j }t|}| 	|j
|j|}t||d ksVt~t||ksjt| 	|j
|j|}~|  t||j }| 	|j
|j}~tjtjdd |  W 5 Q R X d S )NrB   r   zCuda error r   )r   r   r   r!   r   r   rM   sysgetrefcountZforeign_bufferaddressr   r   rG   r   r   r8   ZArrowIOError)rP   r   r   rR   rcZfbufr   r   r   test_foreign_buffer  s&    
rX   c              	   C   s  t | \}}| | ks tt|}|j| ks8t|jrBt| | ksVt| dkrl|jdksltt	| D ]}|| || ksttqtt
d t
| d | d fD ] }||  ||  kstq|
| d | d }|j|ksttjtdd t  W 5 Q R X d S )Nr   r   rA   z-Do not call CudaBuffer's constructor directlyr   )r=   rJ   
to_pybytesr   r   rF   r   is_cpurV   rangerK   parentr   r   	TypeErrorr   
CudaBuffer)r   r&   r%   rQ   issbufr   r   r   test_CudaBuffer'  s(    

rb   c              	   C   s  t | \}}| | ks tt| }|tj|tjdd d < |j	| ksPt|j
sZt| | ksntt| D ]}|| || ksvtqvtd t| d | d fD ] }||  ||  kstq|| d | d }|j|kst~tjtdd t  W 5 Q R X d S )Nr   r   rA   z-Do not call HostBuffer's constructor directlyr   )r=   rJ   rY   r   r   r   r   r    r!   r   rZ   r[   rK   r\   r   r   r]   Z
HostBuffer)r   r&   r%   rR   r_   r`   ra   r   r   r   test_HostBufferC  s(    

rc   c           	      C   s
  t d}| |j }tj|dd}t|tjs2tt|tj	rBtt j
||d}|j| ks^tt| |d d < t j
||d}t j|| t|}t|tj	stt|tjst|j|kst|jrt|j|d|d |jd|d}t j
||d}t j|| d S )NZuint16T)Z	resizabler   r   r5   )r   r   rM   r8   r9   
isinstanceBufferr   r   r^   r    r   r[   r#   r$   r   r+   rZ   r>   rG   )	r   dtr7   r%   r&   r?   Zdevice_bufferbuf2r'   r   r   r   test_copy_from_to_host^  s&    



rh   c              
   C   s  t | dd\}}| }|js"ttj|tj|tjd |j| d d}|jsVttj|| d d  tj|tjd |j| d | d d}|jsttj|| d | d | d   tj|tjd |j| d dd}|jst|j	dkst| d	 d
fd| d dfdfD ]2\}}t
jtdd |j||d W 5 Q R X qd| d f| d	 | d d	 d f| dffD ]2\}}t
jtdd |j||d W 5 Q R X qrt| d }|j|d tj|d | d  tj|tjd | dk rd S |j|dd tj|dd| d   tj|tjd |j|dd tj|d d tj|tjdd d  |j|ddd tj|dd tj|tjdd d  d| d fd| d fd| d	 f| d | d d ffD ]4\}}t
jtdd |j|||d W 5 Q R X qd S )Nr2   r3   r   r   r6      r5   r   rA   rk   r   r   !position argument is out-of-ranger   8requested more to copy than available from device buffer)r%      )r%   r6   )r%   r7      )r%   r7   r6      rB      z,requested copy does not fit into host buffer)r%   r6   r7   )r=   rG   rZ   r   r   r#   r$   r    r!   r   r   r   r   r8   r9   )r   r&   r@   r%   r6   r7   r   r   r   test_copy_to_hostz  sp    

&
 

  
 
  &
*
 

 rv   dest_ctxZsameanotherc              
      s  t |dd\} | }| dkr>t}  jj| jkrDtd n j} | | fdd}| |ksjt|dkr||d d|d |d  |d | d   kst| |kst|d	|d
 d|d d	 |d |d
   |||d
  d	  d   kst|d
 dfd|d	 dfdfD ]0\}}tj	t
dd |||d W 5 Q R X qd|d	 ffD ]0\}}tj	t
dd |||d W 5 Q R X qZ|dk rd S |d
 |d	 d
 d	 ffD ]0\}}tj	t
dd |||d W 5 Q R X qd S )Nr2   r   r4   rx   znot a multi-GPU systemc                     s.   j  f| |  }tj|tjd S Nr   )Zcopy_from_devicerG   r   r    r!   tolistargskwargsZrbufr%   r@   r   r   put  s    z"test_copy_from_device.<locals>.putr   ri   r   rA   r5   rk   rl   r   rn   rp   r   rq   6requested more to copy than available in device buffer)r=   r{   r   contextr   r   skipr+   r   r   r   )rw   r   r&   lstr   r6   r7   r   r   r   test_copy_from_device  sN    
44
  
 

r   c              
      s  t | dd\} | }t|  fdd}| |ks>t| dkr|| d d|d | d  |d |  d   kszt| |kst|d| d d	|d d |d | d   || | d  d  d   kst| d d
fd| d dfdfD ].\}}tjtdd |||d	 W 5 Q R X qd| d ffD ]0\}}tjtdd |||d	 W 5 Q R X q,| dk rld S | d | d d d ffD ]0\}}tjtdd |||d	 W 5 Q R X qd S )Nr.   ry   c                     s.   j  f| |  }tj|tjd S rz   )r>   rG   r   r    r!   r{   r|   r   r   r   r     s    z test_copy_from_host.<locals>.putr   ri   r   rA   r5   rk   rl   r   rn   rp   r   z6requested more to copy than available from host bufferr   )r=   r{   r   r+   r   r   r   r   )r   r&   r   r   r6   r7   r   r   r   test_copy_from_host  sD    
44
  
 

r   c                     s  dd  d fdd	} d\}}| || | |||d   d\}}| tjdtjd	 |d
tjdtjd	 | tjdtjd	 |  t| tj}tj	|d d
 tjd
tjd	 tj	|d
d tjdtjd	 tj	|dd  tjdtjd	 d S )Nc                 S   s   t | }t|}||fS r   )r   r+   r   BufferWriter)r   rQ   writerr   r   r   allocate  s    

z#test_BufferWriter.<locals>.allocater   c                    s   | \}}t | dd\}}|dkr*||_| }|dks>t||j|d | |ks`t|d | }|dks~t|| k rt|| | }||j||d ||7 }q~|  |j	| kst|j
  | }	|j
  |	j	| ksttj|	tjd}
tj||
 d S )Nr.   ry   r   rE   rD   r   )r=   buffer_sizetellr   writerK   seekminflushr   r   synchronizerG   r   r    r!   r#   r$   )
total_size	chunksizer   rQ   r   r&   r%   r6   Zbytes_to_writerg   r'   r   r   r   test_writes  s.    



z&test_BufferWriter.<locals>.test_writes)i   r      d   r   2      K   )r   )
r   r   Zaranger!   Zwriteatr   r    rG   r#   r$   )r   r   
chunk_sizerQ   r   r&   r   r   r   test_BufferWriter  s    
""r   c                  C   sN  d} t | }t|}t| dd\}}|jdks6td|_|jdksJt||jdd |	 dkslt||jdd d|_|jdkst|j
dkst||jdd	d
 |j
dkst||jddd
 |j
dkst||jddd
 ||jddd
 |  | }|j| ks,ttj|tjd}tj|| d S )Nr   r.   ry   r   r   r   rB      i,  rD   i6  i  i  i  r   )r   r+   r   r   r=   r   r   r   rK   r   Znum_bytes_bufferedr   rG   r   r   r    r!   r#   r$   )r   rQ   r   r&   r%   rg   r'   r   r   r   test_BufferWriter_edge_casesL  s0    

r   c                  C   s  d} t | dd\}}t|}|d | dks8t|d}t|dksRt| dksbt|d tj	dtj
d}||}|d	kst| dksttj|dd  |d d	  |d
 | d
kst| }tj| tj
d}tj|| d S )Nr   r2   ry   i  r   r   i  r   r   r   )r=   r   BufferReaderr   r   r   readlenr   Zzerosr!   readintor#   r$   read_bufferr    rG   )r   r&   rQ   readerdatar'   nrg   r   r   r   test_BufferReaderp  s&    





r   c                  C   sz   t ddd\} }t|}|d | }t|dks<t| dksLt| }t	j
| t	jd}t	j| | d S )Nr   r2   ry   r   )r=   r   r   r   r   r   r   r   r   r   r    rG   r!   r#   r$   )r&   rQ   r   r   rg   r'   r   r   r   test_BufferReader_zero_size  s    

r   c                 C   st   t t dt  t dt  g}t tjjdd| tjd}t tjjdd| tjd}t j||g|d}|S )NZf0f1r   r/   )r   r   schema)	r8   r   fieldint16rO   r   r:   r;   record_batch)rE   r   a0a1batchr   r   r   make_recordbatch  s    r   c                  C   s   t d} |  }t| t}t|| j}t|tj	s:t
| j|jksJt
| j|jksZt
| j|jksjt
| }||st
tj|| j}|| st
| j|jkst
| j|jkst
| d|dst
| |st
d S )NrB   r   )r   	serializer   Zserialize_record_batchr   Zread_record_batchr   rd   r8   ZRecordBatchr   num_columnsnum_rowsrG   equalsipccolumn)r   rR   rQ   Zcbatchr%   Zbatch2r   r   r   test_batch_serialize  s     r   c            
      C   s
  t jdddd gt  d} t jddgdgg d gt t  d}t jdddd gt d	t  fd
t  fgd}t jj	dddd gt ddgd}t jj	dddd g|d}t jj	dddd g|d}| |||||g}t 
dd t|D }t j||d}t j|g}	|	S )Nr   r   *   )typerA   )abT)ZcdeF)NNstrsZboolsZfoobar)indices
dictionaryc                 S   s    g | ]\}}d  ||jfqS )zf{})formatr   ).0r_   r&   r   r   r   
<listcomp>  s   zmake_table.<locals>.<listcomp>r   )r8   rO   r   list_Zint32structutf8Zbool_ZDictionaryArrayfrom_arraysr   	enumerater   TableZfrom_batches)
r   r   Za2a3Za4Za5Zarraysr   r   tabler   r   r   
make_table  s4    &




r   c               	   C   s   t  } t }tj|| j}||  W 5 Q R X t| 	 }t
t|}|j|t|d tjt| }|| ||fS )N)r7   )r   r8   ZBufferOutputStreamr   Z
new_streamr   Zwrite_tablerN   getvaluerY   r   r+   r   r>   open_streamr   r   read_all)htableZsinkoutrR   r@   dtabler   r   r   make_table_cuda  s    r   c                  C   sr   t  \} }}}|j|jkst|j|jks.t|j|jks>t| | sPt|tj	| 
 sntd S r   )r   r   r   r   r   r   rG   r8   r   r   r   )rR   r   r@   r   r   r   r   test_table_deserialize  s    
r   c                  C   s   t  \} }}}tj|j|j}|j|jks0t|j|jks@t|j	|j	ksPt| 
| sbt|
tj|  std S r   )r   r8   r   r   columnsZcolumn_namesr   r   r   r   r   rG   r   r   r   )rR   r   r@   r   Zdtable2r   r   r   %test_create_table_with_device_buffers  s    
r   c                 C   sx   t jd}t jj| }||}|j  | }|j	|j	ksVt
t|j	|j	ftj||jd}tj|| d S )Nr   r   )r8   r   r   ZIpcMemHandleZfrom_bufferZopen_ipc_bufferr   r   rG   r   r   reprr   r    r   r#   r$   )handle_bufferZexpected_arrZother_context
ipc_handleZipc_bufr%   r&   r   r   r   other_process_for_test_IPC  s    

 r   c                 C   sf   dd l }|d}t| dd\}}| }| }|jt||fd}|  |  |j	dksbt
d S )Nr   spawnr2   ry   )r4   r}   )multiprocessingZget_contextr=   Zexport_for_ipcr   Processr   startjoinexitcoder   )r   r   rP   r&   rQ   r   r   pr   r   r   test_IPC  s    
r   )r.   ).__doc__rT   	sysconfigr   Zpyarrowr8   numpyr   Zimportorskipr   get_platformplatformZhas_ipc_supportmarkZskipifZcuda_ipcr   r   r   r   r   Zparametrizer(   r-   r=   rL   rS   rX   rb   rc   rh   rv   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   sj   









?1
+3$	