U
    -evg                     @   s  d dl Z d dlZd dlZd dlmZ e jjZ	dddZ
zd dlmZ W n ek
r^   dZY nX dzddZG dd	 d	eZe jd
ddd Ze jd
ddd Ze jd
dd{ddZe jd
dd|ddZe jd
ddd Ze jd
ddd Ze jd
ddd Ze jd
ddd Ze jd
ddd Ze jd
ddd Ze jd
dd d! Ze jd
dd"d# Zd$dd%d&d'Zd(d) Zd*d+ Z d,d- Z!d.d/ Z"d0d1 Z#d2d3 Z$d4d5 Z%d6d7 Z&d8d9 Z'd:d; Z(d<d= Z)d>d? Z*d@dA Z+dBdC Z,dDdE Z-dFdG Z.dHdI Z/dJdK Z0dLdM Z1dNdO Z2dPdQ Z3dRdS Z4dTdU Z5dVdW Z6dXdY Z7dZd[ Z8d\d] Z9d^d_ Z:d`da Z;dbdc Z<ddde Z=dfdg Z>dhdi Z?djdk Z@dldm ZAdndo ZBdpdq ZCdrds ZDdtdu ZEdvdw ZFdxdy ZGdS )}    N)compute summarydescription
   c                 C   s   ddl m} |t | S )Nr   )_get_udf_context)Zpyarrow._computer   padefault_memory_pool)batch_lengthr    r   W/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/tests/test_udf.pymock_udf_context&   s    r   c                   @   s   e Zd ZdS )MyErrorN)__name__
__module____qualname__r   r   r   r   r   +   s   r   session)scopec                  C   s6   dd } d}t }t| ||dt it  | |fS )z4
    Register a unary aggregate function (mean)
    c                 W   s   t t|S N)r	   scalarnpZnansum)ctxxargsr   r   r   func4   s    z"sum_agg_func_fixture.<locals>.funcsum_udfr   )empty_udf_docpcregister_aggregate_functionr	   float64r   	func_namefunc_docr   r   r   sum_agg_func_fixture/   s     r$   c                  C   s6   dd } d}t }t| ||dt it  | |fS )Nc                 S   s   t dtt|S )NOops)RuntimeErrorr	   r   lenr   r   r   r   r   r   G   s    z(exception_agg_func_fixture.<locals>.funcy=exception_len(x)r   r   r   r   r	   int64r!   r   r   r   exception_agg_func_fixtureE   s     r,   c                 C   s6   dd }d}t }t|||dt it  ||fS )Nc                 S   s   t t|t  S r   )r	   r   r'   int32r(   r   r   r   r   [   s    z1wrong_output_dtype_agg_func_fixture.<locals>.funcy=wrong_output_dtype(x)r   r*   r   r   r"   r#   r   r   r   #wrong_output_dtype_agg_func_fixtureY   s     r0   c                 C   s6   dd }d}t }t|||dt it  ||fS )Nc                 S   s   t |S r   )r'   r(   r   r   r   r   n   s    z0wrong_output_type_agg_func_fixture.<locals>.funcy=wrong_output_type(x)r   r*   r/   r   r   r   "wrong_output_type_agg_func_fixturel   s     r2   c                  C   sB   dd } d}ddd}t | ||t t dt  | |fS )z,
    Register a binary scalar function.
    c                 S   s   t jd||g| jdS )Nmultiplymemory_poolr   call_functionr5   )r   mr   r   r   r   binary_function   s    z,binary_func_fixture.<locals>.binary_functionzy=mxzfind y from y = mxr   )r8   r   r   register_scalar_functionr	   r+   )r9   r"   Z
binary_docr   r   r   binary_func_fixture   s    r<   c               	   C   sH   dd } ddd}d}t | ||t t t dt  | |fS )z-
    Register a ternary scalar function.
    c                 S   s,   t jd||g| jd}t jd||g| jdS )Nr3   r4   addr6   )r   r8   r   cZmxr   r   r   ternary_function   s    z.ternary_func_fixture.<locals>.ternary_functionzy=mx+czfind y from y = mx + cr   )array1array2Zarray3r:   )r?   Zternary_docr"   r   r   r   ternary_func_fixture   s    	rB   c                  C   sB   dd } d}ddd}t | ||t t dt  | |fS )zI
    Register a varargs scalar function with at least two arguments.
    c                 W   s(   |}|D ]}t jd||g| jd}q|S )Nr=   r4   r6   )r   firstvaluesaccvalr   r   r   varargs_function   s    z.varargs_func_fixture.<locals>.varargs_functionz	z=ax+by+czfind z from z = ax + by + cr   )r@   rA   r:   )rG   r"   Zvarargs_docr   r   r   varargs_func_fixture   s    rH   c                  C   s4   dd } ddd}d}t | ||i t  | |fS )z-
    Register a nullary scalar function.
    c                 S   s   t jdg| j t  | jdS )N*   typer5   )r	   arrayr   r+   r5   contextr   r   r   nullary_func   s    z*nullary_func_fixture.<locals>.nullary_funczrandom functionzgenerates a random valuer   Ztest_nullary_funcr:   )rO   r#   r"   r   r   r   nullary_func_fixture   s    rP   c                  C   s<   dd } d}i }t  }ddd}t| |||| | |fS )zi
    Register a scalar function which returns something that is neither
    a Arrow scalar or array.
    c                 S   s   dS )NrI   r   r   r   r   r   wrong_output_type   s    z9wrong_output_type_func_fixture.<locals>.wrong_output_typetest_wrong_output_typezreturn wrong output typer   r   r	   r+   r   r;   )rR   r"   in_typesout_typedocr   r   r   wrong_output_type_func_fixture   s    
 rX   c                  C   sD   dd } d}dt  i}t  }ddd}t| |||| | |fS )zq
    Register a scalar function whose actual output DataType doesn't
    match the declared output DataType.
    c                 S   s   t d|dgS Nr=      r   r7   )r   rL   r   r   r   wrong_output_datatype   s    zAwrong_output_datatype_func_fixture.<locals>.wrong_output_datatypetest_wrong_output_datatyperL   zreturn wrong output datatyper   r   )r	   r+   Zint16r   r;   )r\   r"   rU   rV   rW   r   r   r   "wrong_output_datatype_func_fixture   s    
 r^   c                  C   s<   dd } d}i }t  }ddd}t| |||| | |fS )z>
    Register a scalar function with the wrong signature.
    c                   S   s   t jdt  dS )NrZ   rK   )r	   r   r+   r   r   r   r   wrong_signature  s    z5wrong_signature_func_fixture.<locals>.wrong_signaturetest_wrong_signaturezUDF with wrong signaturer   r   rT   )r`   r"   rU   rV   rW   r   r   r   wrong_signature_func_fixture  s    
 rb   c                  C   s4   dd } d}ddd}t | ||i t  | |fS )zE
    Register a scalar function which raises a custom exception.
    c                 S   s   t dd S )Nerror raised by scalar UDF)r   rQ   r   r   r   raising_func"  s    z*raising_func_fixture.<locals>.raising_funcZ
test_raisezraising functionr   r   r:   )rd   r"   rW   r   r   r   raising_func_fixture  s    
 re   Trun_in_datasetr   c                C   s   | \}}|d kr>d}|D ]}t |tjrd}t|}q|r>d}t|}|j|ksVttj|||d}	|t	|f| }
|	|
kst|rdd |D }tj
||}t|}dd |D }|jdtd	||id
}|djd |
kstd S )NTFrZ   lengthc                 S   s   g | ]\}}d | qS )fieldr   ).0indexZin_arrr   r   r   
<listcomp>E  s     z)check_scalar_function.<locals>.<listcomp>c                 S   s   g | ]}t |qS r   )dsrj   )rk   
field_namer   r   r   rm   H  s     resultr   )columnsr   )
isinstancer	   ZArrayr'   r   Zget_functionnameAssertionErrorr7   r   Tablefrom_arraysrn   datasetZto_tablerj   _callcolumnchunks)Zfunc_fixtureinputsrg   r   functionrs   Z
all_scalarargr   rp   Zexpected_outputfield_namestablerw   	func_argsZresult_tabler   r   r   check_scalar_function.  s.    


r   c                 C   s    t | tddgt g d S )Nr      r   r	   rL   r+   )unary_func_fixturer   r   r   test_udf_array_unaryN  s    r   c                 C   s2   t | tddgt tddgt g d S )Nr   r         r   )r<   r   r   r   test_udf_array_binaryV  s
    r   c              	   C   sD   t | tddgt tddgt tddgt g d S )Nr   r   r   r      r   )rB   r   r   r   test_udf_array_ternary_  s    r   c                 C   sh   t | tddgt tddgt tddgt tddgt tddgt g d S )Nr      r   r         r   r   )rH   r   r   r   test_udf_array_varargsi  s    r   c               	   C   s   ddd} dt  i}t  }dd }tt t|d | || W 5 Q R X tjtdd td d	| || W 5 Q R X d
}tjt|d t|d| |d  W 5 Q R X d}tjt|d t|d| d | W 5 Q R X t|d| i | d}tjt|d t|d| i | W 5 Q R X d S )Nztest udf inputzparameters are validatedr   r   c                 S   s   t dgS )Nr   )r	   rL   rM   r   r   r   test_reg_function~  s    z3test_registration_errors.<locals>.test_reg_functionzfunc must be a callablematchZtest_none_function)DataType expected, got <class 'NoneType'>Ztest_output_functionz)in_types must be a dictionary of DataTypeZtest_input_functionr   z?Already have a function registered with name: test_reg_function)r	   r+   pytestraises	TypeErrorr   r;   KeyError)rW   rU   rV   r   expected_exprr   r   r   test_registration_errorsu  s^              r   c              	   C   s8   | \}}d}t jt|d t|dg W 5 Q R X d S )Nz9VarArgs function 'z=ax\+by\+c' needs at least 2 argumentsr   rI   )r   r   
ValueErrorr   r7   )rH   _r"   	error_msgr   r   r    test_varargs_function_validation  s    r   c               	   C   s   dt  i} t  }ddi}dd }tjtdd t|d|| | W 5 Q R X d	d
i}tjtdd t|d|| | W 5 Q R X d S )Nr   r   descc                 S   s   t d|dgS rY   r[   )r   r   r   r   r   	add_const  s    z/test_function_doc_validation.<locals>.add_constz#Function doc must contain a summaryr   Ztest_no_summaryr   ztest summaryz'Function doc must contain a descriptionZtest_no_desc)r	   r+   r   r   r   r   r;   )rU   rV   r#   r   r   r   r   test_function_doc_validation  s2        r   c                 C   s   t | g ddd d S )NFrZ   rf   )r   )rP   r   r   r   test_nullary_function  s    r   c              	   C   s6   | \}}t jtdd tj|g dd W 5 Q R X d S )NzUnexpected output type: intr   rZ   rh   r   r   r   r   r7   )rX   r   r"   r   r   r   rS     s
    rS   c              	   C   sB   | \}}d}t jt|d t|tddgg W 5 Q R X d S )NzDExpected output datatype int16, but function returned datatype int64r   r   r   )r   r   r   r   r7   r	   rL   )r^   r   r"   r   r   r   r   r]     s    r]   c              	   C   s:   | \}}d}t jt|d tj|g dd W 5 Q R X d S )Nz@wrong_signature\(\) takes 0 positional arguments but 1 was givenr   rZ   rh   r   )rb   r   r"   r   r   r   r   ra     s    ra   c               	   C   sV   dd } d}dt  i}i }ddd}tjtdd	 t| |||| W 5 Q R X d S )
Nc                 S   s   |S r   r   r   rF   r   r   r   identity  s    z1test_wrong_datatype_declaration.<locals>.identitytest_wrong_datatype_declarationrL   ztest output valueztest outputr   z%DataType expected, got <class 'dict'>r   r	   r+   r   r   r   r   r;   r   r"   rU   rV   rW   r   r   r   r     s      r   c               	   C   sV   dd } d}dd i}t  }ddd}tjtdd	 t| |||| W 5 Q R X d S )
Nc                 S   s   |S r   r   r   r   r   r   r     s    z3test_wrong_input_type_declaration.<locals>.identity!test_wrong_input_type_declarationrL   ztest invalid input typezinvalid input functionr   r   r   r   r   r   r   r   r     s    
 r   c                 C   s   t t  }| \}}tj|t jdgd t  dg|d}|t jdgd t  dks\t| dksltd }| dkstd S )NrZ     r_   r4   r   @  r   )	r	   proxy_memory_poolr
   r   r7   rL   r+   rt   bytes_allocated)r   
proxy_poolr   r"   resr   r   r   test_scalar_udf_context  s     r   c              	   C   s6   | \}}t jtdd tj|g dd W 5 Q R X d S )Nrc   r   rZ   rh   )r   r   r   r   r7   )re   r   r"   r   r   r   test_raising_func#  s    r   c                 C   s2   | \}}t |tdg}|tdks.td S )Nr      )r   r7   r	   r   rt   )r   r|   r"   r   r   r   r   test_scalar_input)  s    r   c                 C   s   | \}}t t  }| dks&tt jdgd t  |d}| dksPtt||g | dksntd }| dkstd S )Nr   rZ   r   rJ   r   )	r	   r   r
   r   rt   rL   r+   r   r7   )r   r|   r"   r   vr   r   r   test_input_lifetime/  s    r   c                    s&    fddt |D }tjj| dS )Nc                    s(   g | ] \}}t jt| | jd qS r_   )r	   rL   listrK   )rk   ir   schemar   r   rm   ?  s   z,_record_batch_from_iters.<locals>.<listcomp>arraysr   )	enumerater	   RecordBatchrv   )r   Zitersr   r   r   r   _record_batch_from_iters>  s    
r   c                 C   s$   t | t||d t|d |d S )Nr   rZ   r   )r   range)r   nr   r   r   _record_batch_for_rangeD  s    r   c                    s    fdd}|S )Nc                    s   G  fddd}| S )Nc                       s"   e Zd Zdd Z fddZdS )z,make_udt_func.<locals>.udf_func.<locals>.UDTc                 S   s
   d | _ d S r   )callerselfr   r   r   __init__M  s    z5make_udt_func.<locals>.udf_func.<locals>.UDT.__init__c                    sf   z*| j d kr |jd  | _ }|  |}W n2 tk
r\   dd D }tjj|d}Y nX | S )Nc                 S   s   g | ]}t jg |jd qS r   )r	   rL   rK   )rk   rj   r   r   r   rm   V  s   zImake_udt_func.<locals>.udf_func.<locals>.UDT.__call__.<locals>.<listcomp>r   )r   sendStopIterationr	   r   rv   to_struct_array)r   r   batchr   	batch_genr   r   r   __call__P  s    
 z5make_udt_func.<locals>.udf_func.<locals>.UDT.__call__Nr   r   r   r   r   r   r   r   r   UDTL  s   r   r   )r   r   r   r   r   udf_funcK  s    zmake_udt_func.<locals>.udf_funcr   )r   r   r   r   r   r   make_udt_funcJ  s    r   c                      s$   t  G fddd  fddS )zA short datasetc                       s    e Zd Zdd Z fddZdS )z%datasource1_direct.<locals>.Generatorc                 S   s
   d| _ d S )Nr   )r   r   r   r   r   r   d  s    z.datasource1_direct.<locals>.Generator.__init__c                    s:   | j dkrt g g }n|  j d8  _ t | j }| S )Nr   rZ   )r   r   r   r   )r   r   r   r   r   r   r   g  s
    
z.datasource1_direct.<locals>.Generator.__call__Nr   r   r   r   r   	Generatorc  s   r   c                    s     S r   r   rQ   )r   r   r   <lambda>n      z$datasource1_direct.<locals>.<lambda>)datasource1_schemar   r   )r   r   r   datasource1_direct_  s    r   c                     s   t    fdd} t | S )Nc                 3   s&   t dddD ]}t |d V  qd S )Nr   r   rZ   )r   r   r   r   r   r   r   r   t  s    z(datasource1_generator.<locals>.batch_genr   r   r   r   r   r   datasource1_generatorq  s    r   c                     s   t    fdd} t | S )Nc                 3   s.   t dddD ]}t |d V  qtdd S )Nr   r   r   rZ   datasource1_exception)r   r   r&   r   r   r   r   r   ~  s    z(datasource1_exception.<locals>.batch_genr   r   r   r   r   r   {  s    r   c                   C   s   t dt  fdt  fgS )Nr   )r	   r   r-   r   r   r   r   r     s    r   c                 C   s@   | ddd}i }t dt  fdt  fg}| ||||fS )Nz UDTztest {func_name} UDTr   r   )r	   structr-   )r   r"   r#   rU   rV   r   r   r   datasource1_args  s    r   c                 C   sX   t  }|  }| j}t||}tj|  d}t|D ]}|d8 }|t||ks4tq4d S )Nr   rZ   )r   r   r   r   Zregister_tabular_functionZcall_tabular_functionr   rt   )Z
func_makerr   r   r"   r   r   itemr   r   r   _test_datasource1_udt  s    

r   c                   C   s   t t d S r   )r   r   r   r   r   r   test_udt_datasource1_direct  s    r   c                   C   s   t t d S r   )r   r   r   r   r   r   test_udt_datasource1_generator  s    r   c                	   C   s&   t jtdd tt W 5 Q R X d S )Nr   r   )r   r   r&   r   r   r   r   r   r   test_udt_datasource1_exception  s    r   c                 C   sB   t dddddgt  }td|g}t d}||ks>td S )N      $@      4@      >@      D@      I@mean_udf)r	   rL   r    r   r7   r   rt   )unary_agg_func_fixturearrrp   expectedr   r   r   test_scalar_agg_basic  s    
r   c              	   C   s>   t g t  }tjt jdd td|g W 5 Q R X d S )Nzempty inputsr   r   )r	   rL   r    r   r   ZArrowInvalidr   r7   )r   emptyr   r   r   test_scalar_agg_empty  s    r   c              	   C   sH   t dddddgt  }tjt jdd td|g W 5 Q R X d S )	Nr   r   r   (   2   output datatyper   r.   r	   rL   r+   r   r   ArrowTypeErrorr   r7   )r0   r   r   r   r   "test_scalar_agg_wrong_output_dtype  s    r   c              	   C   sH   t dddddgt  }tjt jdd td|g W 5 Q R X d S )	Nr   r   r   r   r   output typer   r1   r   )r2   r   r   r   r   !test_scalar_agg_wrong_output_type  s    r   c                 C   s^   t dddddgt  }t dddd	d
gt  }td||g}t d}||ksZtd S )Nr   r   r   r   r   g      ?g       @g      @g      @g      @Zsum_meang     @@)r	   rL   r+   r    r   r7   r   rt   )Zvarargs_agg_func_fixturearr1arr2rp   r   r   r   r   test_scalar_agg_varargs  s     
r   c              	   C   sH   t ddddddgt  }tjtdd td	|g W 5 Q R X d S )
Nr   r   r   r   r   <   r%   r   r)   )r	   rL   r+   r   r   r&   r   r7   )r,   r   r   r   r   test_scalar_agg_exception  s    r   c           
      C   s   t dddddgt  }t dddddgt  }t d	d
dddgt  }t dddddgt  }t j||gddgd}t j||gddgd}t ||g}|ddg}|ddgddg}	|	d|		dkst
d S )Nr   r   r   r   r   r   r   rZ   g      N@g     Q@g      T@g     V@g      Y@r   idvaluenamesr   r   )r   Zmeanvalue_mean_udf)r	   rL   r    r-   r   Zconcat_tablesgroup_by	aggregaterename_columnssort_byrt   )
r   r   r   Zarr3Zarr4Ztable1Ztable2r   rp   r   r   r   r   test_hash_agg_basic  s    
r  c                 C   s   t g t  }t g t  }t j||gddgd}|ddg}t jt g t  t g t  gddgd}||kstd S )Nr   r   r   r   r   )r	   rL   r    r-   r   r  r  rt   )r   r   r   r   rp   r   r   r   r   test_hash_agg_empty  s     r  c              	   C   s|   t dddddgt  }t dddddgt  }t j||gd	d
gd}tjt jdd |d		dg W 5 Q R X d S )Nr   r   r   r   r   r   r   rZ   r   r   r   r   r   )r   r.   
r	   rL   r+   r-   r   r   r   r   r  r  )r0   r   r   r   r   r   r    test_hash_agg_wrong_output_dtype  s
    r  c              	   C   s|   t dddddgt  }t dddddgt  }t j||gd	d
gd}tjt jdd |d		dg W 5 Q R X d S )Nr   r   r   r   r   r   r   rZ   r   r   r   r   r   )r   r1   r  )r2   r   r   r   r   r   r   test_hash_agg_wrong_output_type  s
    r	  c              	   C   sz   t dddddgt  }t dddddgt  }t j||gd	d
gd}tjtdd |d		dg W 5 Q R X d S )Nr   r   r   r   r   r   r   rZ   r   r   r   r%   r   )r   r)   )
r	   rL   r+   r-   r   r   r   r&   r  r  )r,   r   r   r   r   r   r   test_hash_agg_exception  s
    r
  c                 C   s   d}d}t td|t  }t tj||t  }t j||gddgd}|	d
dg}|	d
dgdd	g}|d|dkstd
S )z2Test hash aggregate udf with randomly sampled datai@B r   rZ   r   r   r   )r   r   )r   sumZvalue_sum_udfN)r	   rL   r   repeatr    randomchoicer-   r   r  r  r  r  rt   )r$   Z	value_numZ	group_numr   r   r   rp   r   r   r   r   test_hash_agg_random  s    
r  )r   )r   )r   )Hr   numpyr   Zpyarrowr	   r   r   markrw   Z
pytestmarkr   Zpyarrow.datasetrn   ImportErrorr   r&   r   Zfixturer$   r,   r0   r2   r<   rB   rH   rP   rX   r^   rb   re   r   r   r   r   r   r   r   r   r   rS   r]   ra   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r	  r
  r  r   r   r   r   <module>   s   
























 	
2	


			