U
    d"                    @   s  d dl Zd dlZd dlZd dlmZmZ d dlmZm	Z	 d dl
mZmZmZmZ d dlmZ d dlZd dlZd dlmZmZmZmZ d dlm  mZ d dlmZ ed ej reZ!dd	 Zd
d Z"ej#dddZ$ddej%dfddZ&ej%ej'j(ej)ej'j*ej+ej'j,ej-ej'j.ej/ej'j0ej1ej'j2ej3ej'j4ej5ej'j6ej7ej'j8i	Z9dddZ:dej;dde:fddZ<dej;de:fddZ=G dd dej>Z?e@dkreA  dS )    N)partialreduce)	viewitemsviewkeys)assumegivensettingsHealthCheck)core	workspacett_coredyndep)
caffe2_pb2z*@/caffe2/caffe2/fb/optimizers:sgd_simd_opsc                     s,   d krd  d<   dd  fdd}|S )Ndeadlinemax_examples2   c                    s   t f  | S N)_hypothesis_settings)fkwargs A/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/hypothesis_test.pywrapped   s    zsettings.<locals>.wrapped)
setdefault)r   r   r   r   r   r      s
    r   c                 C   s   ddt |    S )N      ?npexpxr   r   r   sigmoid!   s    r!         c                 K   sx   | t jtjf |||d}| t jtjf |||d}tt|t| |k  | t|| ||| t|||fS )Nmin_sizemax_size)stlistshudimsr   lenarrays)Zdrawdtypeelementsmin_dimmax_dimr   Zdims_Zextra_r   r   r   _tensor_and_prefix%   s    r1   c                    sB   t jtjf || |d}t jddd}t || fddS )zF generates a tensor and a list of indices of larger tensor of same dimr$      
   	min_value	max_valuec                    sN   t t | d t| d  tj| d d tjt jd| d d ddS )Nr"   r   r4   r-   r.   )r'   tuplesjustr)   r,   r   int64integers)pairr7   r   r   <lambda>5   s    
z%_tensor_and_indices.<locals>.<lambda>)r'   r(   r)   r*   r;   r8   flatmap)r/   r0   r-   r.   r   Z
data_dims_Zoriginal_dimr   r7   r   _tensor_and_indices0   s    r?   c                 C   s"   | r| nt jt jt jg} t| S r   )r   int32r:   float32r'   sampled_fromdtypesr   r   r   _dtypesJ   s    rE   Fc                    sR   t f |  fddt|r"dndd|tdd dfdd	}|S )
Nc                    s   t jd| t j|  ddS )Nr2   filter_)nr-   r.   )r)   tensorselements_of_typer-   rF   r   r   r=   S   s    z_test_binary.<locals>.<lambda>)YX1X2)rL   )inputsout   r   r   c                    sr   t  ddg|g}|\}}| ||||gdg rZ|jtjkrZ| ||||gddg | ||||g d S )NrM   rN   r   )r
   CreateOperatorassertDeviceChecksr-   r   rA   assertGradientChecksassertReferenceChecks)selfrO   rP   gcdcoprM   rN   )namereftest_gradientr   r   test_binaryQ   s    	z!_test_binary.<locals>.test_binary)r   r>   r'   rB   r   )r[   r\   rG   gcsr]   allow_inplacerD   r^   r   )rG   r[   r\   r]   r   _test_binaryO   s    

	ra   c                    sT   t f |  fdd|r"t ntdd|tdddfdd	}|S )
Nc                    s   t | tj|  ddS )NrF   r7   )r1   r)   rJ   rK   rF   r   r   r=   h   s   z(_test_binary_broadcast.<locals>.<lambda>FrO   in_place   d   rR   c           	         sb   t j ddg|rdndgdd}|\}}| ||||gdg fdd}| ||||g| d S )	NrM   rN   rL   r"   )	broadcastr   c                    s   t j | |d | jdfS )Nr   rK   )r   arrayr-   r    y)r\   r   r   cast_reft   s    zG_test_binary_broadcast.<locals>.test_binary_broadcast.<locals>.cast_refr
   rS   rT   rV   )	rW   rO   rc   rX   rY   rZ   rM   rN   rj   )r[   r\   r   r   test_binary_broadcastg   s       z5_test_binary_broadcast.<locals>.test_binary_broadcast)r   r>   r'   booleansr9   r   )r[   r\   rG   r_   r`   rD   rl   r   )rG   r[   r\   r   _test_binary_broadcaste   s    
rn   c                   @   sN  e Zd Zdd Zef ejdde dej	e
dddd	 Zef d
ejddddieje
dddd Zef d
ejddieje
dddd Zdd Zdd Zdd Zdd Zef e e dej	e
dddd Zef de iej	e
dddd Zef de iej	e
dddd  Zef e e d!ej	e
ddd"d# Zeejdd$eejd%e d&e
ddd'd( Zef ejd)d$d*ejd$d+d*ed,d-gd.ej	e
ddd/d0 Zee j! d1e
d2deejd)d3d*ejd)d3d*e ed4ged5gej"d6d6d*ejdd7d*ejd)d$d*ejd)d$d*d8	d9d: Z#ef ed)d$ed;d3ed;d)edd$d<ej	e
d2d=d>d?d@ Z$ef edd$edAdBgdCej	e
dddDdE Z%ef dej&dFdgej"d6dGd*dHieje
dddIdJ Z'eee(j)e(j*e(j+e(j,gdKe
dddLdM Z-ef ejdde ej"dNdOd*e ej"dNdOd*dPej	e
dddQdR Z.ef ejd3de ej"dNdOd*ej"dNdOd*ej"dNdOd*ej"dSdTd*dUej	e
dddVdW Z/e0dXdY Z1ef ejd$de ej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gd\eje
ddd]d^ Z2e0d_d` Z3ef ejd$de ej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gd\eje
dddadb Z4ef ejd$dej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gdceje
ddddde Z5e0dfdg Z6ef ejd$de ej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gd\eje
dddhdi Z7ef ejd$dej"dTdNd*ej"dNdOd*ej"dZdNd*ej"dZdNd*ed2d[gdceje
dddjdk Z8ef ejdld)e(j+ejd;dmd*dne doej9e
dddpdq Z:ef ej&dmd3gej"drdrd;d)dsdHej&dmge(j+ejd;dd*dtejd)d3d*duej	e
dddvdw Z;ef dxej&dmgej"drdrdTd)dsdHiej	e
dddydz Z<ef d{ejejd;dmd*d;dmd|ieje
ddd}d~ Z=ef d{ejejd;dmd*d;dmd|ieje
dddd Z>ef eje
dddd Z?ef d{ejejd;dmd*d;dmd|ieje
dddd Z@ef ejejd;dmd*d;dmd|e deje
d2ddd ZAef ej&dmd3gej"drdrd;d)dsdHej&dmge(j+ejd;dd*dtdej	e
dddd ZBef d{ejejd;dmd*d;dmd|ieje
dddd ZCef ejejd)dmd*d;dmd|edd6ddgdeje
dddd ZDef dej&dmgej"drdrddHiej	e
dddd ZEef dej&dmgej"ddmd*dHiej	e
dddd ZFef dej&dmgej"ddmd*dHiej	e
dddd ZGef dej&dmgej"drdrddHiej	e
dddd ZHef dej&dmgej"d)dd*dHieje
dddd ZIdd ZJeed)dmed)ded)dFed)d3eejKde
dddd ZLeed)dmed)dmed)dFed)d3eejKde
d2d=d>dd ZMeed)dFedFdmed)dFed)d3de
d2d=d>dd ZNef de ieje
dddd ZOef eje
dddd ZPef ejdd ZQef ejdd ZRef ejdd ZSef eje
dddd ZTef ejdd ZUef ejdd ZVef ed)dmeddge j!rddgng  dej	e
dddd ZWef ejdd7dÍe e e e dĜeje
d2d=d>ddƄ ZXef de ieje
ddddȄ ZYef de ieje
ddddʄ ZZef e e d˜eje
dddd̈́ Z[eed;ded;dd΍e
ddddЄ Z\dd҄ Z]eed;ded;ddӍe
ddddՄ Z^eed;ded;ded;dFd֍e
d2d=d>dd؄ Z_ef e ee`eaebee`eaebe dٜej	e
ddddۄ Zcef e ej"ddTd*ejej"dTdd*dލej"ddZd*dߜej	e
dddd Zdef ejej"dddލej"ddmd*dej	e
dddd Zeef efe(j+e(jge(j)e(j,gdhdd e e ejd)dFed)dFd%dej	e
dddd Zief defe(j+e(jge(j)e(j,gdhdd iej	e
dddd Zjeed)dFed)dFed)dFde
dddd Zke
eljmgddef ed)dFed)dFed)dFed)dFed;dedd3dej	dd Zne
eljmgddef ed)dFed)dFed)dFed)dFed;dedd3dej	dd Zoef e e ej"ddd*dej	e
dddd Zpeeq ddd Zref ed;d3ed)dmed)dFed)dFed dgedddgdej	e
d2d=d>dd Zsef ed;d3ed;d3ed;d3ed;d3ed)dmed)dFed)dFed dgedddgd	ej	e
d2d=d>d	d
 Ztef eddmed)dmed)d3ed dgej"ddTd*deje
dddd Zuef def hdd iej	e
dddd Zvef d
ejddddieje
dddd Zwef ejddmd*ejddmd*ejddmd*ej"dNd6d*deje
dddd Zxef ejddmd*ejddmd*ej"dNd6d*deje
dddd Zyef ejddmd*ejddmd*deje
dddd Zzef ejdmdd*ejddmd*ejd)dFd*deje
dddd  Z{e
d)d2d!eeddedFdmed)d)eddd"d#d$ Z|d2S (%  TestOperatorsc                 C   s\   dd dd dd dd d}t |D ]0\}}t||tjd|  t||tjd|  q&d S )Nc                 S   s
   | |k gS r   r   x1Zx2r   r   r   r=          z3TestOperators.test_comparison_ops.<locals>.<lambda>c                 S   s
   | |kgS r   r   rp   r   r   r   r=      rr   c                 S   s
   | |kgS r   r   rp   r   r   r   r=      rr   c                 S   s
   | |kgS r   r   rp   r   r   r   r=      rr   )LTZLEGTGE)r_   )r   ra   r)   gcs_cpu_onlyrn   )rW   opsr[   r\   r   r   r   test_comparison_ops   s    z!TestOperators.test_comparison_opsr2   rH   rb   i'  )r   c                 C   sV   t dddg|sdndg}|\}}| ||||gdg | ||||gddg d S )NSumrM   rN   rL   r   r
   rS   rT   rU   )rW   rO   rc   rX   rY   rZ   rM   rN   r   r   r   test_sum   s    zTestOperators.test_sumrO   )rH   r/   r0   c           
      C   s   t dddgdg}|\}}|d d df }dd }| ||||gdg tdD ]}	| ||||g|	dg qR| ||||g| d S )	NZRowMulrM   rN   rL   r   c                 S   s@   t j| j| jd}t|jD ]}| |f ||  ||f< q|gS )N)shaper-   )r   zerosr}   r-   rangesize)r    ri   retir   r   r   r\      s    z'TestOperators.test_row_mul.<locals>.refr2   )r
   rS   rT   r   rU   rV   )
rW   rO   rX   rY   rZ   rM   ZXtmprN   r\   r   r   r   r   test_row_mul   s    zTestOperators.test_row_mulc           	      C   s   t dddgdg}|\}}|t||d k||k  d8  < |t||d k||k  d7  < | ||||gdg tdD ]}| ||||g|dg qdd	 }| ||||g| d S )
NZMaxrM   rN   rL   g?r   r2   c                 S   s   t | |gS r   )r   maximumXrL   r   r   r   elementwise_max   s    z/TestOperators.test_max.<locals>.elementwise_max)r
   rS   r   logical_andrT   r   rU   rV   )	rW   rO   rX   rY   rZ   rM   rN   r   r   r   r   r   test_max   s    $$zTestOperators.test_maxc                 C   s:   dd }dd }t d||dd|  td||d|  d S )	Nc                 S   s   t | tst| dk S dS )Ni?T
isinstancefloatabsr   r   r   r   not_overflow   s    
z,TestOperators.test_add.<locals>.not_overflowc                 S   s
   | | fS r   r   rh   r   r   r   r\      s    z#TestOperators.test_add.<locals>.refAddTrG   r]   rF   ra   rn   rW   r   r\   r   r   r   test_add   s    zTestOperators.test_addc                 C   s,   dd }t d|dd|  td||  d S )Nc                 S   s
   | | fS r   r   rh   r   r   r   r\      s    z#TestOperators.test_sub.<locals>.refZSubT)r]   r   )rW   r\   r   r   r   test_sub   s    zTestOperators.test_subc                 C   s:   dd }dd }t d||dd|  td||d|  d S )	Nc                 S   s   t | tst| dk S dS )Ni  Tr   r   r   r   r   r      s    
z,TestOperators.test_mul.<locals>.not_overflowc                 S   s
   | | fS r   r   rh   r   r   r   r\      s    z#TestOperators.test_mul.<locals>.refZMulTr   rF   r   r   r   r   r   test_mul   s    zTestOperators.test_mulc                 C   s`   dd }dd }dd }t d||d|tjd	|  t d||d
|d|  td|||d|  d S )Nc                 S   s
   | | fS r   r   rh   r   r   r   r\      s    z#TestOperators.test_div.<locals>.refc                 S   s   t | dkS )N{Gz?)r   r   r   r   r   non_zero   s    z(TestOperators.test_div.<locals>.non_zeroc                   S   s   t tjtjgS r   )r'   rB   r   rA   float64r   r   r   r   
div_dtypes   s    z*TestOperators.test_div.<locals>.div_dtypesZDivT)rG   r]   rD   r_   F)rG   r]   rD   )rG   rD   )ra   r)   rv   rn   )rW   r\   r   r   r   r   r   test_div   s<              zTestOperators.test_div)r   rc     c                 C   sH   t ddg|sdndg}| |||gdg | |||gddg d S )NZNegativer   rL   r   r{   )rW   r   rc   rX   rY   rZ   r   r   r   test_negative   s
    
zTestOperators.test_negativer   c                 C   s<   t ddd}| |||gdg | |||gddg d S )NZTanhr   rL   r   r{   rW   r   rX   rY   rZ   r   r   r   	test_tanh   s    zTestOperators.test_tanhc                 C   s@   t ddgdg}| |||gdg | |||gddg d S )NAveragedLossr   lossr   r{   r   r   r   r   test_averaged_loss   s    z TestOperators.test_averaged_loss)r   inplacec              	   C   s   t ddg|rdndg}dd }| |||gdg | |||g| |rz| t | |||gddg W 5 Q R X n| |||gddg d S )NZSoftsignr   rL   c                 S   s   | dt |   fS Nr"   r   r   )r   r   r   r   softsign  s    z-TestOperators.test_softsign.<locals>.softsignr   )r
   rS   rT   rV   assertRaises	ExceptionrU   )rW   r   r   rX   rY   rZ   r   r   r   r   test_softsign   s    "zTestOperators.test_softsignr#   )r%   r&   r.   )device_optionsset_seedc              
      s   t |}ttdd |D dk |r8|D ]
}d|_q,dd   fdd|D }|dd  D ]D}|rztj|d	 | q^| t	 tj|d	 | W 5 Q R X q^d S )
Nc                 S   s   h | ]
}|j qS r   )device_type.0dor   r   r   	<setcomp>  s     z;TestOperators.test_random_seed_behaviour.<locals>.<setcomp>r"   r   c                 S   sB   t   t j }tjdg dg| dgd}|| |jd  S )NZ
XavierFillrL   r2   )device_optionr}   )	r   ZResetWorkspaceC	Workspacer
   rS   runblobsfetch)r   wsrZ   r   r   r   r     s    
  
z5TestOperators.test_random_seed_behaviour.<locals>.runc                    s   g | ]} |qS r   r   r   r   r   r   
<listcomp>*  s     z<TestOperators.test_random_seed_behaviour.<locals>.<listcomp>r   )
copydeepcopyr   r+   Zrandom_seedr   testingassert_array_equalr   AssertionError)rW   r   r   r   ysri   r   r   r   test_random_seed_behaviour  s    

z(TestOperators.test_random_seed_behaviourr"   r4       ZPACKED)axis
num_outputenginec                 C   sL  t jd t jdddddt j}dd }|t|j|d  }|}	t j|	|t j}
t j|	t j}tj	dddd	gd
g||d}d|fd|
fd	|ffD ]\}}| j
|| q| j
| | j
jd
  }| t|jt|jd | |	g  ||
|g}| |||dg t|D ]\}}| ||||dg q(d S )Nr"   r2   rd   c                 S   s   d}| D ]}||9 }q|S r   r   )xspr    r   r   r   prod;  s    
z5TestOperators.test_fully_connected_axis.<locals>.prodZFCr   WbrL   )r   r   r   )r   randomseedrandnastyperA   listr}   r
   rS   r   create_blobfeedr   r   r   assertEqualrT   	enumeraterU   )rW   r   r   r   rX   rY   r   r   KNr   r   rZ   r[   paramrL   rO   _r   r   r   test_fully_connected_axis2  s.    &
z'TestOperators.test_fully_connected_axisz$Skipping test due to no gpu present.Nrd   lstmZlinearr      )	hidden_size
num_layersbidirectionalrnn_mode
input_modedropoutTr   Dc
                 C   s  t jrt|dk d}
tj|
 t jr4tj}d}n
tj}d}||	 }|| }|rZ|d9 }|| }|}|}|rrdnd}|| | | }|| | | }d||d |   }||9 }tj	|
tj}| jdj||d tjd	d
dddgdddddg|||||||
|d}tj|||	
tj}| jd
j||d | jjd  }tj|||| 
tj}|dkrtj|||| 
tjntd
tj}||||g}|dkrdd t|D ndddg}|D ] }| j||||dgddd qd S )Nr"     ZMIOPENZCUDNNr2   r#   ZWEIGHTr   Z	RecurrentZINPUTZHIDDEN_INPUTZ
CELL_INPUTZOUTPUTZHIDDEN_OUTPUTZCELL_OUTPUTZRNN_SCRATCHZDROPOUT_STATES)r   r   r   r   r   r   r   r   r   r"   c                 S   s   g | ]\}}|qS r   r   )r   r   r   r   r   r   r     s     z0TestOperators.test_recurrent.<locals>.<listcomp>r   rd   r   )stepsize	threshold)r   Zhas_hip_supportr   r   r   r   r)   Zhip_doZgpu_dorandr   rA   r   r   r   r
   rS   r   r   r   emptyr   rU   )rW   r   r   r   r   r   r   r   r   r   r   r   r   Zinput_weight_sizeZupper_layer_input_weight_sizeZrecurrent_weight_sizeZinput_bias_sizeZrecurrent_bias_sizeZnum_directionsZfirst_layer_szZupper_layer_szZtotal_szr   rZ   r   Hr   rO   Z
input_idxsZ	input_idxr   r   r   test_recurrentW  s    
   
       zTestOperators.test_recurrentr   )ndimr   add_axis
num_inputsr   )r   r   c                    s   t |k  ddddgd | }ddddgd | }d	ddd
dgd | }	g }
t|D ]2} dkrn|	| |< |
tjj| tj qVtj	d|ddg d}| 
|||
dg t|D ]}| |||
|dg q fdd}| |||
| d S )NX0rM   rN   X3r2   rd         r"   r#   r   ConcatrL   Y_dims)r   r   c                     s\   t | }  r2tt| D ]}t| | | |< qtfdd| D }tj| d|gS )Nc                    s   g | ]}t |  qS r   r   r}   r   r    r   r   r   r     s     zITestOperators.test_depth_concat.<locals>.depth_concat.<locals>.<listcomp>r   )r   r   r+   r   expand_dimsrg   concatenate)rO   r   
input_dimsr   r   r   r   depth_concat  s    z5TestOperators.test_depth_concat.<locals>.depth_concat)r   r   appendr   r   r   r   rA   r
   rS   rT   rU   rV   )rW   r   r   r   r   rX   rY   input_namesr}   individual_dimsrO   r   rZ   r   r   r   r   test_depth_concat  s$     zTestOperators.test_depth_concat)NCHWr"   )NHWCrd   )r   orderc                    s   ddddgd | }ddddg}d	ddd
gd | }g }t |D ].}	||	 | d	 < |tjj| tj q@tjd|ddg d d}
| 	||
|dg t |D ]}	| 
||
||	dg q fdd}| ||
|| d S )Nr   rM   rN   r   r2   rd   r   r   r"   r#   r   rL   r   r   r  c                     s:   t | } d  t fdd| D }tj|  d|gS )Nr"   c                    s   g | ]}t |  qS r   r   r   r   r   r   r     s     z_TestOperators.test_depth_concat_with_order.<locals>.depth_concat_with_order.<locals>.<listcomp>r   )r   r   rg   r   )rO   r   r  r   r   depth_concat_with_order  s    zKTestOperators.test_depth_concat_with_order.<locals>.depth_concat_with_order)r   r   r   r   r   r   rA   r
   rS   rT   rU   rV   )rW   r   r  rX   rY   r   r}   r   rO   r   rZ   r  r   r  r   test_depth_concat_with_order  s    z*TestOperators.test_depth_concat_with_orderr   g      $@)r*   r.   c                 C   s   t d| t dtjdtjd t d td}|jdddgddgdd t	d	}|
tjd	|gd
d t | t d}t d}td|jd g}t|jd d
 D ]}	||	|jd   ||	d < qdd lm}
 |
j||dd d S )Ninputnextr   rK   outputcollect_netr   )Znum_to_collectZcollect_datar2   num_iterr"   r   )decimal)r   FeedBlobr   rg   r@   Z
CreateBlobr
   NetZLastNWindowCollectorPlanAddStepexecution_stepZRunPlan	FetchBlobr~   r}   r   Znumpy.testingr   Zassert_almost_equal)rW   r   rX   rY   r	  planr  rO   Z
new_outputr   Znptr   r   r   test_last_n_windows  s,    



 


z!TestOperators.test_last_n_windowsrK   c                 C   sB   t jd|}| jd| tddg }| j	| d S )Nr   dataZPrint)
r   r   Zpermutationr   r   r   r   r
   rS   r   )rW   r-   r  rZ   r   r   r   
test_print  s    zTestOperators.test_print皙??)rO   rc   momentumnesterovlrc                    s   |\}}	t j|gt jd}tjddddg|r0dnd|r:dndg t|d}
| ||
||	|gd	g  fd
d}| ||
||	|g| d S )NrK   ZMomentumSGDgradmr  grad_oZm_o)r  r  r   r   c                    sP   |d }s$||   |  }||fS  | ||   }d  |  |  |fS d S )Nr   r"   r   )r  r  r  Zadjusted_gradientZm_newr  r  r   r   momentum_sgd  s    z5TestOperators.test_momentum_sgd.<locals>.momentum_sgd)r   asarrayrA   r
   rS   intrT   rV   )rW   rO   rc   r  r  r  rX   rY   r  r  rZ   r   r   r  r   test_momentum_sgd  s(    	

   	zTestOperators.test_momentum_sgdgh㈵>r   )rO   rc   decayr  r  epsilonc	              	      s   |\}	}
}t |
d }
t j|gt jd}tjdddddg|rBdnd|rLdnd	|rVdnd
g |d}| |||	|
||gdg  fdd}| |||	|
||g| d S )Nr   rK   ZRmsPropr  msmomr  r  ms_omom_o)r  r$  r%  r   r   c                    sN   |d }|d  t | |   }| ||  t |   }|}|||fS )Nr   r   )r   squaresqrt)r  r&  r'  r  r(  r)  r  r$  r%  r  r   r   rmsprop>  s
    z/TestOperators.test_rmsprop_sgd.<locals>.rmsprop)r   r   r!  rA   r
   rS   rT   rV   )rW   rO   rc   r$  r  r  r%  rX   rY   r  r&  r'  rZ   r-  r   r,  r   test_rmsprop_sgd(  s$    





   zTestOperators.test_rmsprop_sgdc                 C   s   t | tjrt| } tj|ddd}tj|ddd}|| }	t||	 t| |  }
|||
|  7 }||	7 }t|| | |t| |  |  }d|t||k< |tj||gddfS )Nr   r   r"   )	r   r   ndarrayasscalartaker+  signr   stack)alphabetalambda1lambda2wnzgrH   zg2sigmar   r   r   _dense_ftrlG  s    
zTestOperators._dense_ftrlgMbP?ZSIMD)rO   rc   r5  r6  r7  r8  r   c
                 C   s   |\}
}}}t |}t j||gdd}tjddddg|r>dnd|rHdndg||||||d		}| |	||
||gd
g | |||
||gt| j|||| d S )Nr/  r   Ftrlvarr:  r  var_onz_or5  r6  r7  r8  r   r   r   )	r   r   r4  r
   rS   rT   rV   r   r?  rW   rO   rc   r5  r6  r7  r8  r   rX   rY   rA  rH   r<  r  r:  rZ   r   r   r   test_ftrl_sgdW  s8    


        zTestOperators.test_ftrl_sgdc                 C   sr  t | tjrt| } |j}tj|ddd}tj|ddd}	|jd }
||
d}||
d}||
d}|	|
d}	|jd }|| }t|| t| |  }|	|||  7 }	||7 }tjj	|	ddd}|d }|	|t|
 | d  |t| |  |  }t
|D ].}|| |t|
 krd|d d |f< q||}||}|	|}	|tj||	gddfS )Nr   r/  r   r"   r2   gư>)r   r   r0  r1  r}   r2  reshaper+  linalgnormr   r4  )r5  r6  r7  r8  r9  r:  r;  Z	old_shaperH   r<  Z
output_dimZ	input_dimr=  r>  Zz_normsr   r   r   r   _dense_gftrlu  s6    





zTestOperators._dense_gftrlc
                 C   s   |\}
}}}t |}t j||gdd}tjddddg|r>dnd|rHdndg||||||d		}| |	||
||gd
g | |||
||gt| j|||| d S )Nr/  r   ZGFtrlrA  r:  r  rB  rC  rD  r   )	r   r   r4  r
   rS   rT   rV   r   rJ  rE  r   r   r   test_gftrl_sgd  s8    


        zTestOperators.test_gftrl_sgd)rO   r5  r6  r7  r8  r   c	                    s   |\}	}
}}t |	jd }||d dk }|| }t |
}
t j|
|gdd}tjddddd	gddg ||d
	}|||	|||gdg  fdd}|||	|||g| d S )Nr   r2   r/  r   
SparseFtrlrA  r:  indicesr  rD  c              	      s:     | | || |\}}|| |< |||< | |fS r   )r?  )r9  r:  r   r;  swsnzr5  r6  r7  r8  rW   r   r   ftrl  s      z0TestOperators.test_sparse_ftrl_sgd.<locals>.ftrl)	r   aranger}   r   r4  r
   rS   rT   rV   rW   rO   r5  r6  r7  r8  r   rX   rY   rA  rH   r<  r  rM  r:  rZ   rQ  r   rP  r   test_sparse_ftrl_sgd  s2    


     
 z"TestOperators.test_sparse_ftrl_sgdc              	   C   s   t || |||||S r   )ro   r?  )r6  r7  r8  r9  r:  r;  r5  r   r   r   _dense_ftrl_send_alpha_by_input  s    z-TestOperators._dense_ftrl_send_alpha_by_inputc
              
   C   s   |\}
}}}t |}t j||gdd}t |t j}tjdddddg|rRdnd|r\dnd	g|||||d
}| |	||
|||gdg | 	|||
|||gt
| j||| d S )Nr/  r   r@  rA  r:  r  r5  rB  rC  r6  r7  r8  r   r   r   )r   r   r4  rg   r   rA   r
   rS   rT   rV   r   rU  rE  r   r   r   !test_ftrl_sgd_send_alpha_by_input  s8    



    
   
z/TestOperators.test_ftrl_sgd_send_alpha_by_inputc	              
      s   |\}	}
}}t |	jd }||d dk }|| }t |
}
t j|
|gdd}t |t j}tj	ddddd	d
gddg ||d}
|||	||||gdg  fdd}|||	||||g| d S )Nr   r2   r/  r   rL  rA  r:  rM  r  r5  rV  c              	      s:     | | || ||\}}|| |< |||< | |fS r   )rU  )r9  r:  r   r;  r5  rN  rO  r6  r7  r8  rW   r   r   rQ     s       zDTestOperators.test_sparse_ftrl_sgd_send_alpha_by_input.<locals>.ftrl)r   rR  r}   r   r4  rg   r   rA   r
   rS   rT   rV   rS  r   rX  r   (test_sparse_ftrl_sgd_send_alpha_by_input  s6    
     z6TestOperators.test_sparse_ftrl_sgd_send_alpha_by_inputrQ   r3   )r6   r0   r-   r.   )r  with_remappingc                    s^   t jddgdgrdgng  |d} |||gdg d	 fdd	} |||g| d S )
NZUniquer  unique	remappingr   r   c                    sp    jtt|   ttt|  rl  j| j  fddtt| D }tj	||  d S )Nc                    s   g | ]} |  qS r   r   r   r   r\  r[  r   r   r   @  s     zCTestOperators.test_unique.<locals>.unique_valid.<locals>.<listcomp>)
r   r   r+   setsortedr}   r   r   r   r   )r  r[  r\  ZremappedrW   rZ  r^  r   unique_valid;  s    z/TestOperators.test_unique.<locals>.unique_valid)N)r
   rS   rT   ZassertValidationChecks)rW   r  rZ  rX   rY   rZ   rb  r   ra  r   test_unique+  s    zTestOperators.test_uniqueF)	allow_nanallow_infinityr5   r6   )r*   r-   r.   )
predictionlabelstop_kc                 C   sL   |dkrt j}tjdddgdg||d}dd }| j|||||g|d	 d S )
Nr"   ZAccuracyrf  rg  accuracy)rh  r   c           
      S   s   | j d }d}tdt| D ]\}tdd t| | D dd dd}dd	 |d| D }|D ]}||| kr^|d
7 }q^q|| }	|	fS )Nr   c                 s   s   | ]\}}||gV  qd S r   r   )r   jitemr   r   r   	<genexpr>b  s     z>TestOperators.test_accuracy.<locals>.op_ref.<locals>.<genexpr>c                 S   s   | d S Nr   r   r   r   r   r   r=   c  rr   z=TestOperators.test_accuracy.<locals>.op_ref.<locals>.<lambda>T)keyreversec                 S   s   g | ]}|d  qS r   r   r   r   r   r   r   f  s     z?TestOperators.test_accuracy.<locals>.op_ref.<locals>.<listcomp>r"   )r}   r   r+   r`  r   )
rf  rg  rh  r   correctr   Zpred_sortedmax_idsr  ri  r   r   r   op_ref]  s    
z+TestOperators.test_accuracy.<locals>.op_refr   rZ   rO   	reference)r)   cpu_dor
   rS   rV   )rW   rf  rg  rh  rX   rY   rZ   rr  r   r   r   test_accuracyE  s     zTestOperators.test_accuracytarget_probabilitiesc                 C   s2   t ddgdg}dd }| j|||g|d d S )NZ
Perplexityrw  
perplexityc                 S   s.   | j d }t| d| }tdd |}|fS )Nr   g      c                 S   s   | | S r   r   rh   r   r   r   r=     rr   z?TestOperators.test_perplexity.<locals>.op_ref.<locals>.<lambda>)r}   r   powerr   )rw  r   Zperplexitiesrx  r   r   r   rr    s    
z-TestOperators.test_perplexity.<locals>.op_refrs  r
   rS   rV   )rW   rw  rX   rY   rZ   rr  r   r   r   test_perplexitys  s    zTestOperators.test_perplexitylengthsr$   c                 C   s>   t ddgdg}dd }| j||tj|tjdg|d d S )NZLengthsToSegmentIdsr|  segment_idsc                 S   s8   g }t | D ]\}}|||g  qtj|tjdfS NrK   r   extendr   rg   r@   r|  sidsr   lr   r   r   rr    s    z9TestOperators.test_lengths_to_segment_ids.<locals>.op_refrK   rs  r
   rS   rV   r   rg   r@   rW   r|  rX   rY   rZ   rr  r   r   r   test_lengths_to_segment_ids  s    z)TestOperators.test_lengths_to_segment_idsc                 C   s>   t ddgdg}dd }| j||tj|tjdg|d d S )NZLengthsRangeFillr|  Zincreasing_seqc                 S   s:   g }t | D ]\}}|tt| qtj|tjdfS r~  )r   r  r   r   r   rg   r@   )r|  r  r   r  r   r   r   rr    s    z5TestOperators.test_lengths_range_fill.<locals>.op_refrK   rs  r  r  r   r   r   test_lengths_range_fill  s    z%TestOperators.test_lengths_range_fillc                 C   sX   ddddddg}t ddgdg}d	d
 }dd }| j||tj||tjd|d d S )Nr#   r   rd   r2   r   ZSegmentIdsToRangesr}  rangesc                 S   s~   t jddgt jdg}d}t| D ]H\}}||krV|d7 }|t j|dgt jd q*|d d  d7  < q"t j|t jdfS )Nr   rK   r"   r/  )r   rg   r@   r   r   )r}  r  prevr   Zsidr   r   r   rr    s    z8TestOperators.test_segment_ids_to_ranges.<locals>.op_refc                 S   s8   g }t | D ]\}}|||g  qtj|tjdfS r~  r  r  r   r   r   lengths_to_segment_ids  s    zHTestOperators.test_segment_ids_to_ranges.<locals>.lengths_to_segment_idsrK   rs  r  )rW   rX   rY   r|  rZ   rr  r  r   r   r   test_segment_ids_to_ranges  s    
z(TestOperators.test_segment_ids_to_rangesc                 C   s>   t ddgdg}dd }| j||tj|tjdg|d d S )NZLengthsToRangesr|  r  c                 S   s<   | j s| dfS ttdgt| d d f| ffS )N)r   r2   r   r/  )r   rG  r   Zcolumn_stackr   cumsumr   r   r   r   rr    s
     z4TestOperators.test_lengths_to_ranges.<locals>.op_refrK   rs  r  r  r   r   r   test_lengths_to_ranges  s    z$TestOperators.test_lengths_to_ranges)r|  include_last_offsetc                    sF   t jddgdg d} fdd}| j||tj|tjdg|d d S )	NZLengthsToOffsetsr|  r  r  c                    sd   | j s| dg}n tdgt| d d fg} r\t|d tt| gf|d< t|S )Nr   r/  )r   rG  r   r   r  rg   sumtuple)r    arrr  r   r   rr    s     $z5TestOperators.test_lengths_to_offsets.<locals>.op_refrK   rs  r  )rW   r|  r  rX   rY   rZ   rr  r   r  r   test_lengths_to_offsets  s    		z%TestOperators.test_lengths_to_offsets)rf  rg  c                 C   s8   t dddgddg}dd }| j||||g|d d S )	NZMultiClassAccuracyrf  rg  
accuraciesamountsc                 S   s   | j d }| j d }tj|td}|d tj|td}|d tj| dd}td|D ]<}|| }|| }	||	kr||	  d7  < ||	  d7  < q\td|D ] }|| }
|
r||  |
  < q||fS )Nr   r"   rK   r   )r}   r   r   r   fillr"  Zargmaxr   )rf  rg  r   r   r  r  rq  r   Zmax_idZlabel_idamountr   r   r   rr  #  s$    



z7TestOperators.test_multi_class_accuracy.<locals>.op_refrs  rz  )rW   rf  rg  rX   rY   rZ   rr  r   r   r   test_multi_class_accuracy  s    z'TestOperators.test_multi_class_accuracyc                 C   sN   t ddgdg}dd }||}dd }| j||tj|tjdg|d	 d S )
NZSegmentIdsToLengthsr}  r|  c                 S   s*   g }t | D ]\}}|||g  q|S r   )r   r  r  r   r   r   lengths_to_idsH  s    zATestOperators.test_segment_ids_to_lengths.<locals>.lengths_to_idsc                 S   s   t | }|dkr"tjg tjdfS g }d}d}t|D ]V}| | }||kr|dkr\|| |d |kr||d |d7 }q\|}d}|d7 }q6|| tj|tjdfS )Nr   rK   r/  r"   )r+   r   rg   r@   r   r   )idsZ
ids_lengthr|  Zprev_idZ
tmp_lengthidxZcur_idr   r   r   ids_to_lengthsP  s&    




zATestOperators.test_segment_ids_to_lengths.<locals>.ids_to_lengthsrK   rs  r  )rW   r|  rX   rY   rZ   r  r}  r  r   r   r   test_segment_ids_to_lengths=  s    z)TestOperators.test_segment_ids_to_lengths      ?g      ?g       @)r|  ry  c                    sF   t jddgdg d} fdd}| j||tj|tjdg|d d S )	NZLengthsToWeightsr|  weightsry  c                    s8   g }| D ]}| |dt|  g  qtj|tdfS Nr"   rK   )r  powr   rg   r   )r|  Zweighted_lengthr  r  r   r   lengths_to_weightsz  s    zATestOperators.test_lengths_to_weights.<locals>.lengths_to_weightsrK   rs  r  )rW   r|  ry  rX   rY   rZ   r  r   r  r   test_lengths_to_weightsm  s    z%TestOperators.test_lengths_to_weightsinput_tensor)rd  re  c                 C   s2   t ddgdg}dd }| j|||g|d d S )NZAbsr  r  c                 S   s   t | fS r   r   r  r   r   r   abs_ref  s    z'TestOperators.test_abs.<locals>.abs_refrs  rz  )rW   r  rX   rY   rZ   r  r   r   r   test_abs  s    zTestOperators.test_absic                 C   s2   t ddgdg}dd }| j|||g|d d S )NZCosr  r  c                 S   s   t | fS r   )r   cosr  r   r   r   cos_ref  s    z'TestOperators.test_cos.<locals>.cos_refrs  rz  )rW   r  rX   rY   rZ   r  r   r   r   test_cos  s    zTestOperators.test_cosc                 C   s2   t ddgdg}dd }| j|||g|d d S )NZSinr  r  c                 S   s   t | fS r   )r   sinr  r   r   r   sin_ref  s    z'TestOperators.test_sin.<locals>.sin_refrs  rz  )rW   r  rX   rY   rZ   r  r   r   r   test_sin  s    zTestOperators.test_sinc                 C   s2   t ddgdg}dd }| j|||g|d d S )NZExpr  r  c                 S   s   t | fS r   r   r  r   r   r   exp_ref  s    z'TestOperators.test_exp.<locals>.exp_refrs  rz  )rW   r  rX   rY   rZ   r  r   r   r   test_exp  s    zTestOperators.test_expc                 C   sH   t ddgdg}dd }| j|||g|d | |||gddg d S )NLogr  r  c                 S   s   t | fS r   )r   logr  r   r   r   log_ref  s    z'TestOperators.test_log.<locals>.log_refrs  r   )r
   rS   rV   rU   )rW   r  rX   rY   rZ   r  r   r   r   test_log  s    zTestOperators.test_logc                    sr   t jdg dgddd j  t }t jddgdgdd	 t fd
d t | }|d d S )NCreateBlobsQueuequeuer   r"   capacity	num_blobsDequeueBlobsrP   g?)Ztimeout_secsc                      s   j  S r   )r   r   r   rZ   rW   r   r   r=     rr   z:TestOperators.test_blobs_dequeue_timeout.<locals>.<lambda>gRQ?)r
   rS   r   r   timer   RuntimeErrorZassertGreater)rW   tr   r  r   test_blobs_dequeue_timeout  s$    z(TestOperators.test_blobs_dequeue_timeoutre   )num_threadsnum_elementsr  r  r   c              	      s  ddl 
ddltjdg dg| d}	j| fddtD  tD ]fddD  qZ 	fd	d
t|D ]&}tD ]	j	d
| qq
fddt|D }|D ]}	|	  qtD ]:fddtD }
tjddg|
 d}	j| q|D ]}	|	  q.tddgg }	j| 	fddtD tD ]T	 j j tD ],	tfddtD  qqzdS )a  
        - Construct matrices of size N x D
        - Start K threads
        - Push all N rows into the queue of capacity C
        - Pull all N rows out of the queue.
        - Verify that the output matrices are permutation of the rows of the
          original matrices.
        r   Nr  r  )r  r  r   c                    s"   g | ]}t j d t jqS )r   )r   r   r   r   rA   r   r   )r  r   r   r     s   z<TestOperators.test_blobs_queue_threading.<locals>.<listcomp>c                    s   g | ]}|  qS r   r   r   r   r   r   r   "  s     c                    s    fddt D }tjddg| |d}zB }t||D ]\}}j|j|d qBj| W q  j	k
r   Y d S X q d S )Nc                    s   g | ]}d  | qS )x_{}_{}formatr]  r  r   r   r   &  s     zMTestOperators.test_blobs_queue_threading.<locals>.enqueue.<locals>.<listcomp>ZEnqueueBlobsr  r   )
r   r
   rS   
get_nowaitzipr   r   r   r   Empty)r  Z
feed_blobsrZ   elemselemZ	feed_blob)r   r  qr  rW   r  r   enqueue$  s"     z9TestOperators.test_blobs_queue_threading.<locals>.enqueuer  c                    s   g | ]}j  |fd qS )targetargs)Thread)r   r  )r  	threadingr   r   r   ;  s   c                    s   g | ]}d  | qS zy_{}_{}r  r]  ry   r   r   r   A  s     r  r   CloseBlobsQueuec                    s*   g | ]" t  fd dtD qS )c                    s$   g | ]}j jd  |  qS r  )r   r   r  r   )r   rH   )r   rW   r   r   r   L  s   zGTestOperators.test_blobs_queue_threading.<locals>.<listcomp>.<listcomp>)r   Zvstackr   )r   )r  rW   r  r   r   L  s   c                 3   s*   | ]"}t      | V  qd S r   )r   Zarray_equal)r   k)r   rj  r   r   r   r   rl  V  s   z;TestOperators.test_blobs_queue_threading.<locals>.<genexpr>)r  r  r
   rS   r   r   r   Queueputr   r  startjoinr   r}   
assertTrueany)rW   r  r  r  r  r   rZ   r  threadsthreadZdequeue_blobsr   )r   r  r   rj  rH   r  r  r  r  rW   r  r   r   r   test_blobs_queue_threading  sd    

z(TestOperators.test_blobs_queue_threading)num_producersnum_consumersr  r  r   c                    s   t d}|jg d||d}g }d}	t|D ]z}
d|
 }t |  fddt|D }  } |g| ||g  |
d d }t j| |d	}|	|7 }	|| q,t d
}||gd t dt jd|ddt d|g}g }g }|j	g ddd}t|D ]}
d|
 }t |}|
|g|d }|d }t |d }|j	g ddd}|| |||g| |t j|||g|d qt jd|dd}t d|}t jd||gdd}t d}|| || | j| d}|D ]"}|| jjt|   7 }q| ||	 d S )Ninit_netr"   r  r   producer_%dc                    s   g | ]} j g d dddqS r"   r   F)valueZrun_onceConstantFill)r   timesnetr   r   r   i  s   z7TestOperators.test_safe_blobs_queue.<locals>.<listcomp>r3   r
  producer_exit_netproducer	producersTZconcurrent_substepsproducer_exitr   r  zconsumer_%dr/  _counter        Zshould_stop_blobconsumerinitworkertest)r
   r  r  r   NextNameSafeEnqueueBlobsr  r   r  r  SafeDequeueBlobsr   r  r  r   r   r   strr   tolistr   )rW   r  r  r  r  r   r  r  producer_stepstruthr   r[   r   statuscountstepr  producer_stepZconsumer_stepsZcountersZconst_1Znet1Znet2counterconsumer_step	init_stepworker_stepr  vr   r  r   test_safe_blobs_queueY  s    
   


  


      


 z#TestOperators.test_safe_blobs_queue)
num_queuesr  r  r  c              	      s  t d| td fddt|D }g }g }t|D ]}d| }	t|	fddtD }
 }|| g|
 |
|g  td| }||| d || t|	tjd| g|d	td| |gg}|| q>td
tjd|ddg}g fdd}td}t	j
jddd}|j|d t	j
jdd|fd|d d}
|
d }||| tdtjd|g|dtd|g}td}tjd||gdd}td}|| || | j| t|d kstt|| d kstd S )Nr	  r  c                    s   g | ]}j g d  dqS )r"   r  )r  r  )r  r  r  r   r   r     s      zBTestOperators.test_weighted_sample_blobs_queue.<locals>.<listcomp>r  c                    s   g | ]} j g d dddqS r  r  r  r  r   r   r     s   zproducer_exit_%dr   r
  r  r  Tr  c                    s     |  d S r   )r   )ZinsZouts)
status_lstr   r   r     s    z>TestOperators.test_weighted_sample_blobs_queue.<locals>.appendZweight_sample_dequeue_netr/  r"   )lowhighr   r  r   )r  table_idx_blobr  r  r  r  r  r  )printr
   r  r   r  r  r  r   r  r   r   randintZWeightedSampleDequeueBlobsuniformPythonr  r  r   r   r+   r   )rW   r	  r  r  r  Zqueuesr  Zproducer_exit_netsr   r[   r   r  Zexit_netr  r  r   Zconsumer_netr  r  r  r  r  r   )r  r  r  r  r
  r    test_weighted_sample_blobs_queue  s    	




  

  

  


z.TestOperators.test_weighted_sample_blobs_queuer  c                    sj   ddg t |jdkr  d tjddgdg d} fdd} fd	d
}| j|||g|d|d d S )Nr   r2   Z
ExpandDimsr  expandedr*   c                    s6   t t }|  | }|D ]}tj||d}q|fS )Nr   )r   r_  sortr   r   )r  r  kwZinc_dimsrdimr  r   r   expand_dims_ref  s    z?TestOperators.test_squeeze_expand_dims.<locals>.expand_dims_refc                    s:   t t }|jdd | }|D ]}tj||d}q |fS )NT)ro  r   )r   r_  r  r   Zsqueeze)r  r  r  Zdec_dimsr  r  r  r   r   squeeze_ref
  s    z;TestOperators.test_squeeze_expand_dims.<locals>.squeeze_ref)r   rZ   rO   rt  Zoutput_to_gradZgrad_reference)r+   r}   r   r
   rS   rV   )rW   r  rX   rY   rZ   r  r  r   r  r   test_squeeze_expand_dims  s&    
z&TestOperators.test_squeeze_expand_dimsc                 C   s@  d}t j| ddddg}ddddg}dddddg}tjddddgd	g|||d
}t jt jdt jdd}t 	dgd t j}	t
|||}
| jd| | jd|	 | jd|
 | j| | jjd	  }|dg}t 	ddddddddddddddddg}| jt j|| ddd d S ) Nr   r2   r"   rd   ZTTr   r   coresrL   )	inp_sizes	out_sizestt_ranks   r   r   g=]ﯾgލg<E6gå~÷>gtg)|
g&
>>g^jgU->gP>gFSḴgwrD{gcPrWq>g⩰KgLg~g|=delta)r   r   r   r
   rS   r   r   r   rA   rg   r   Zinit_tt_coresr   r   r   r   r   r   rG  assertAlmostEqualrH  rI  )rW   rX   rY   r   r  r  r  rZ   r   r   r  rL   goldenr   r   r   test_tt_layer  sT    	        zTestOperators.test_tt_layerc                 C   s  d}t j| dddg}dddg}tjdddddd	gd
dddg||ddddgdd}t ddddgt j}t ddddgt j}t ddddgt j}	t dddddgt j	}
t ddgt j
}| jd| | jd| | jd|	 | jd|
 | jd	| | j| | jjd
  }| t|jddg t ddddddddgddddddddgg}| jt j|| ddd d S )Nr   r3   r2   TTSparseLengthsSumcore0core1core2indexr|  rL   core0_outputcore1_outputrM  r"   r   r   Zfactor_iZfactor_jZranksZemb_sizer   r#   rd         r!  )r   r   r   r
   rS   onesr   rA   rg   r:   r@   r   r   r   r   r   r   r   r   r}   r#  rH  rI  )rW   rX   rY   r   
factor_vocfactor_widthrZ   c0c1c2r*  r|  rL   r$  r   r   r   test_tt_sls_layerG  s<    



	zTestOperators.test_tt_sls_layerc                 C   s  t ddddddddd	gd
ddg}tddddgtj}tddddgtj}tddddgtj}tddgtj}tdddgtj}tdddgtj}	tdddgdddgdddgdddgdddggtj}
tddgtj}| j	
d| | j	
d| | j	
d| | j	
d| | j	
d| | j	
d|	 | j	
d|
 | j	
d	| | j	| | j	jd
  }| j	jd  }| j	jd  }| t|jt|j | t|jt|j | t|jt|j d S )NTTSparseLengthsSumGradientr'  r(  r)  r|  	core0_out	core1_outrM  dYdCore0dCore1dCore2r3   r"   r#   r   rd   r2   r   r   @   r
   rS   r   r0  r   rA   rg   r@   r:   r   r   r   r   r   r   r   r   r}   rW   rX   rY   rZ   r3  r4  r5  r|  Zc0_outZc1_outrM  r:  r;  r<  r=  r   r   r   test_tt_sls_gradientopo  sP       z$TestOperators.test_tt_sls_gradientopc                 C   s  t ddddddddd	gd
ddg}tddddgtj}tddddgtj}tddddgtj}tddgtj}tdddgtj}tdddgtj}	tdddgdddgdddgdddgdddggtj}
tddgtj}| j	
d| | j	
d| | j	
d| | j	
d| | j	
d| | j	
d|	 | j	
d|
 | j	
d	| | j	| | j	jd
  }| j	jd  }| j	jd  }| t|jt|j | t|jt|j | t|jt|j d S )Nr7  r'  r(  r)  r|  r8  r9  rM  r:  r;  r<  r=  e   r"   r2   r   f      r#   rd   r   r   r?  r@  r   r   r   test_tt_sls_gradientop1  sP       z%TestOperators.test_tt_sls_gradientop1c              
   C   s   dddg}dddg}t jddddddgd	d
ddg||ddddgdd}tddddgtj}tddddgtj}tddddgtj}tdddddgtj}	tdddddddgtj}
| 	||||||	|
gddg d S )Nr3   r2   r&  r'  r(  r)  r*  r|  rL   r+  r,  rM  r"   r   r   r-  r   r#   rd   )
r
   rS   r   r0  r   rA   rg   r:   r@   rU   )rW   rX   rY   r1  r2  rZ   r3  r4  r5  r*  r|  r   r   r   test_tt_sls  s"    



	zTestOperators.test_tt_slsc              
   C   s  dddg}dddg}t jddddd	d
gddddg||ddddgdd}tddddgtj}tddddgtj}tddddgtj}tdddddgtj}	tdddddddgtj}
| j	
d| | j	
d| | j	
d| | j	
d	|	 | j	
d
|
 | j	| | j	jd  }| t|jddg tdgd dgd dgd dgd dgd dgd dgd g}| jtj|| ddd d S )N}         r#   r&  r'  r(  r)  r*  r|  rL   r+  r,  rM  r"   r   r>  r-  r   i= i N  i@B rd   r2   r   r.  r/  r!  )r
   rS   r   r0  r   rA   rg   r:   r@   r   r   r   r   r   r   r   r   r}   r#  rH  rI  )rW   rX   rY   r1  r2  rZ   r3  r4  r5  r*  r|  rL   r$  r   r   r   test_tt_sls_repro  s4    



	BzTestOperators.test_tt_sls_reproc                 C   s"  t ddddddddd	gd
ddg}tddddgtj}tddddgtj}tddddgtj}tdddddddgtj}tdddgtj}tdddgtj}	tdddgdddgdddgdddgdddggtj}
tddgtj}| j	
d| | j	
d| | j	
d| | j	
d| | j	
d| | j	
d|	 | j	
d|
 | j	
d	| | j	| | j	jd
  }| j	jd  }| j	jd  }| t|jt|j | t|jt|j | t|jt|j d S )Nr7  r'  r(  r)  r|  r8  r9  rM  r:  r;  r<  r=  rB  r"   r2   r   rC  rD  r#   r   rd   r   r   r?  r@  r   r   r   test_tt_sls_gradientop2   sP       z%TestOperators.test_tt_sls_gradientop2simpleZdagZ	async_dag)num_workersnet_typec                    s<  ddl m} ddlm} |dddd dd}tjd	 ttD ]}td| D ]}	d
	|d d|	 }
d
	|d d|	 d }d	|d d|	 }d	|d d|	 d }d
	||	}|j
|
|  dttj dfdttj dfd |j
||  dttj dfdttj dfd j||g| qZqJjddgd jdd dg j j j | _| _jj tt   fddfddt|D }|dd  D ]4}tj|d | jtt |ddd qd S )Nr   )ModelHelper)brewZ
test_modelr[   r3   r2   r     {}_{}r"   z{}_{}_mr  r  )dim_indim_outZweight_initZ	bias_initZ0_0labelZxentr   c                     s   dd l } | jd fddtd D }|D ]T}j|j| j | j	d jdj| j | j	d q2j
j fdd|D }|S )	Nr   rR  c                    s   g | ]}d   |qS )rS  r  )r   rj  )depthr   r   r   _  s     zCTestOperators.test_dag_net_forking.<locals>.run.<locals>.<listcomp>r2   r   rV  c                    s$   g | ]}j jt |   qS r   )r   r   r  r   )r   
input_blob)input_to_gradrW   r   r   r   h  s   )numpyr   r   r   r   r   r   r   r   rA   r   r  )r   Zinput_blobsrX  Z	gradients)drW  rX   rY  r  rH   rW   r   r   r   \  s"    z/TestOperators.test_dag_net_forking.<locals>.runc                    s   g | ]
}  qS r   r   r  r   r   r   r   m  s     z6TestOperators.test_dag_net_forking.<locals>.<listcomp>gg?RV@r   r!  )!Zcaffe2.python.model_helperrO  caffe2.pythonrP  r   r   r   reversedr   r  fcdictr   r  rz   ZSquaredL2Distancer   ZAddGradientOperatorsProtor   ZCopyFromparam_init_nettyperM  r   r   r  r  r   r   r#  r  r*  )rW   rN  rM  rX   rY   rO  rP  itersr   rj  Zbottom_1Zbottom_2Zmid_1Zmid_2topoutputsr  r   )r[  rW  rX   rY  r  rH   r   rW   r   test_dag_net_forking+  sd    
    

z"TestOperators.test_dag_net_forking)r/   r0   )r  	slice_dimar   is_emptyc                 C   s   |t |j }|r:tjjdgt|j  tj}|d7 }||j|  }||j|  d }tjt |jtjd}tj	t |jtjdd }	t
||||< t|||	|< tddddgd	g}
d
d }| ||
|||	g| | ||
|||	gddg d S )Nr   r"   rK   r/  ZSlicer  r  endr  c                 S   s0   t |jdkr| S dd t||D }| | fS )Nr   c                 S   s&   g | ]\}}t ||d krdn|qS )r/  N)slice)r   sieir   r   r   r     s     z?TestOperators.test_slice.<locals>.slice_ref.<locals>.<listcomp>)r+   r}   r  )r    seZslcr   r   r   	slice_ref  s    z+TestOperators.test_slice.<locals>.slice_ref)r+   r}   r   r   r   r   r   r@   r~   r0  minmaxr
   rS   rV   rU   )rW   r  rg  rh  r   ri  rX   rY   Z	start_vecZend_vecrZ   rp  r   r   r   
test_slices  s(     zTestOperators.test_slicec                 C   s,   t ddgdg}| |||gdd  d S )NShaper  r}   c                 S   s   | j fS r   r}   r   r   r   r   r=     rr   z*TestOperators.test_shape.<locals>.<lambda>rz  )rW   r  rX   rY   rZ   r   r   r   
test_shape  s    zTestOperators.test_shapec                 C   sP   dd }t jjt|jdd }tjddgdg|d}| ||||g| d S )	Nc                    s    fdd|D fS )Nc                    s   g | ]} j | qS r   ru  r]  r   r   r   r     s     zITestOperators.test_shape_with_axes.<locals>.shape_ref.<locals>.<listcomp>r   rh   r   r   r   	shape_ref  s    z5TestOperators.test_shape_with_axes.<locals>.shape_refr3   r   rt  r  r}   )axes)	r   r   r  r+   r}   r  r
   rS   rV   )rW   r  rX   rY   rw  ry  rZ   r   r   r   test_shape_with_axes  s    z"TestOperators.test_shape_with_axesrh   c                 C   sX   t dddgdg}| ||||gdd  t ddgdg}| |||gd	d  d S )
NZHasElementsr    ri   Zhas_elementsc                 S   s   t | dkpt |dkfS rm  r+   rh   r   r   r   r=     rr   z1TestOperators.test_has_elements.<locals>.<lambda>ZIsEmptyri  c                 S   s   t | dkfS rm  r{  r   r   r   r   r=     rr   rz  )rW   r    ri   rX   rY   rZ   r   r   r   test_has_elements  s    zTestOperators.test_has_elements)initial_iters	max_itersc                 C   s   t d}|dgdg | jdt|gtj	 | jdt|gtj	 t d}|
ddgdg | jdg t d}|t jd||gt dd | j| | jjd  }| |jtj	 | |d	 t|| d S )
Nr  iter	num_itersZcriteriastopr  r  r  r   )r
   r  Iterr   r   r   r   r!  r   r:   ru   r`  external_outputr  r  r  r  BlobReferencer   r   r   r   r-   rr  )rW   r}  r~  r  Zcriteria_netr  rc  r   r   r   /test_should_stop_as_criteria_net_execution_step  s*    


 z=TestOperators.test_should_stop_as_criteria_net_execution_stepc                 C   s   dd }|dd|dd|ddg}t jd|d	 t d
dt jd|d t ddt d|d g}dddg}t d}|t jd|dd | j| t|D ]2\}}| | jj	d
|d   d	 ||  qd S )Nc           	      S   s  d | }d | }td | }|jg |gdgdd |j|g|gdd td	 | }|jg dg|rpd
ndd}|j|g|gdd | j|g td | }|jg |gdgd
d td | }|jg dgd
d}|j|g|gdd | j|g ||||gS )Nzshould_stop_{}	output_{}zinit_{}r"   r  )r}   r  booltozcriterion_{}r   net_{}zender_{})r  r
   r  r  Castr`  r  r  )	r   disabledZshould_stopr  r  Z	criteriontmpr  enderr   r   r   
createNets  sD    


z>TestOperators.test_disabled_execution_step.<locals>.createNetsr"   Fr2   Trd   Zstep_1r   Zshould_stop_1r  Zstep_2Zshould_stop_2Zstep_3r   r  r  Z	all_stepsr
  r  )r
   r  r  r  r  r   r   r   r   r   r  r   )rW   r  netsZstepsexpectedr  r   r   r   r   r   test_disabled_execution_step  s4    -  	

z*TestOperators.test_disabled_execution_step)r}  r  c                 C   s   t d}|dgdg | jdt|gtj	 t 
d|g}|| t d}|| | j| | jjd  }| |jtj	 | |d ||  d S )Nr  r  r  r  r   )r
   r  r  r   r   r   r   r!  r   r:   ExecutionStepZSetIterr  r  r   r   r   r   r-   )rW   r}  r  r  r  r  rc  r   r   r   #test_iter_count_with_execution_step
  s    



z1TestOperators.test_iter_count_with_execution_step)r}  r  num_netsc                 C   s  t d}|g dg}| jdt|gtj	 t j
d|d}t|D ]>}t d|}||dgdg t 
d|g}	||	 qP|d t d	}
|
| t d
}|g dddg | j| | j|
 | j| | jjd  }| |jtj	 | |d |||   || dkr| jjd  }d}| ||k | jjd  }| || |t||  d S )Nr  
iter_mutexr  concurrent_stepsr
  r  r  Tr  	stats_net	stats_keyZ	stats_valZstats_tsr   s   atomic_iter/stats/iter/num_iter)r
   r  ZCreateMutexr   r   r   r   r!  r   r:   r  r   r  Z
AtomicIterZ
AddSubstepZSetConcurrentSubstepsr  r  ZStatRegistryExportr   r   r   r   r-   r  r   r*  )rW   r}  r  r  r  r  r  r   r  r  r  r  rc  r  Zatomic_iter_keyZstat_valr   r   r   &test_atomic_iter_with_concurrent_steps  s<    




z4TestOperators.test_atomic_iter_with_concurrent_steps)rh  srcdstuse_namec                    s   | |}tjtjg}||krL |krL tjk	rLt }t||j|j} fdd}	t	  }
|rvt
jj|
 }
tjddgdg|
d}| |||gdg | |||g|	\}|  |j d S )Nc                    s   |   gS r   )r   r  r  r   r   r\   V  s    z$TestOperators.test_cast.<locals>.refr  r   rL   r  r   )r   r   rA   r   r  iinfocliprq  rr  _NUMPY_TYPE_TO_ENUMr   ZTensorProtoDataTypeNamelowerr
   rS   rT   rV   r   r-   )rW   rh  r  r  r  rX   rY   Zftypesinfor\   r  rZ   rP   r   r  r   	test_castF  s    

zTestOperators.test_castg-C6?gGz?)r.   )rh  epsa_gradeps_gradc           
   	      s~    fdd}t jddgdg d}| |||gdg | |||g| t jddgdg|d}	| j||	|gddgdd	d
 d S )Nc                    s&   t |  d  } t | d|   fS )Nr   r"   )r   r  r  r  r  r   r   r\   h  s    z%TestOperators.test_logit.<locals>.refZLogitr   rL   r  r   g{Gz?gMb`?)r   r   )r
   rS   rT   rV   rU   )
rW   rh  r  r  r  rX   rY   r\   rZ   Zop_gradr   r  r   
test_logita  s     zTestOperators.test_logitT)rd  )rh  r  c                    sL    fdd}t jddgdg d}| |||gdg | |||g| d S )Nc                    s   t | } |t | < |fS r   )r   r   isnan)r  rP   r  r   r   r\     s    
z+TestOperators.test_replace_nan.<locals>.refZ
ReplaceNaNr   rL   r  r   rk   )rW   rh  r  rX   rY   r\   rZ   r   r  r   test_replace_nan{  s    zTestOperators.test_replace_nanrC   c                 C   s   t jd| t | dS Nr"   )r/   r-   r.   r)   tensorrJ   rK   r   r   r   r=     s     zTestOperators.<lambda>)r  	has_inputhas_extra_shapeextra_shapec                    s   |j j |j t tjkr tj |d|j|g}t  }|r|rntjddgdg||d}	t	|7 qtjddgdg|d}	n tjdg dg|t
d}	g }d
 fdd		}
| ||	|dg | ||	||
\}|  |j  d S )Nr   r  r   rL   )r-   r  r  )r-   r  )r-   r  r}   c                    s   t j d}|gS )Nr}   Z
fill_valuer-   )r   full)rO   re  r-   Zgt_shaper  r   r   r\     s    z-TestOperators.test_constant_fill.<locals>.ref)N)r-   rb  r   r  rk  r}   r  r
   rS   r  r   rT   rV   r   )rW   r  r  r  r  rX   rY   rO   	enum_typerZ   r\   rP   r   r  r   test_constant_fill  s:    
z TestOperators.test_constant_fillc                 C   s   t jd| t | dS r  r  rK   r   r   r   r=     s     c           	         s    j j j t tjkr tjtj dgd g}t }tjdddgdg|d} fdd}| 	|||dg | 
||||\}| |j  d S )	Nr   rK   r  r   VrL   c                    s   t j jd d}|gS )Nr   r  )r   r  r}   )r    r  re  r  r-   r  r   r   r\     s    z9TestOperators.test_constant_fill_from_tensor.<locals>.ref)r-   rb  r   r  rg   rk  r  r
   rS   rT   rV   r   )	rW   r  rX   rY   rO   r  rZ   r\   rP   r   r  r   test_constant_fill_from_tensor  s     z,TestOperators.test_constant_fill_from_tensor)r  rH   r[  c                    s  ddl m}m} tjd |jdd}| j	dddd	d
dg d| _
| j	ddg |j|d	d||dd |jddgdg |jdgdg |j jD ]}t| qtj| jddi\}}	dd t|	D }	td}
|
 jd d = |
 j	| |	d dks$tdddg}t| \}}}dddg}t| \}}}|
 j	dg |
 j	| j |
 j	| j ddd
ddg dg}tjd dd d!d"gd d gdd!gd#d$gd g fd%d&|D |||||| fd'd&|jD | |
 dgd(}tdtj|||tj  tdtjd#||tj  tdtjj!d||fd)tj" d*d+ }| j#t$j%|d,d& d-D |dd#dgd. d/D ]&}| &t$j%|d0d& d-D |dg qd S )1Nr   )model_helperrP  rR  ZElmanrQ  input_tseq_lengthsZtimestephidden_t_prev	gates_t_w	gates_t_brL  hidden_tZgates_tr2   )rT  rU  r   hidden_t_gradc                 S   s   i | ]\}}t |t |qS r   )r  )r   r  r  r   r   r   
<dictcomp>  s     z>TestOperators.test_elman_recurrent_network.<locals>.<dictcomp>ZElmanBackwardgates_t_grad)r  hiddenr   )r  r  r"   )r  r  r   )Zhidden_t_prev_gradhidden_gradr   )r  r  r"   )r  Z
input_gradr   r  hidden_inputZRecurrentNetworkr  r  Zhidden_outputZstep_workspacesr"   r/  c                    s   g | ]}  |qS r   r*  r]  rO   r   r   r   	  s    z>TestOperators.test_elman_recurrent_network.<locals>.<listcomp>c                    s   g | ]}  |qS r   r  )r   r   r  r   r   r   	  s     )Z	alias_srcZ	alias_dstZalias_offsetZrecurrent_statesZinitial_recurrent_state_idslink_internallink_externallink_offsetbackward_link_internalbackward_link_externalbackward_link_offsetr   step_netbackward_step_netZoutputs_with_gradsrx  c                 S   s  | j d }| j d }| j d }tj|d ||fd}|j d |d ksJt|j d |ks\t|j d |ksnt||dd d d d f< t|D ]b}	| |	 d||}
||	 d||}t||j}|d|||
d|| }t|||	d < q|dd  ||d d||fS )Nr   r"   r2   ru  r/  )	r}   r   r~   r   r   rG  dotr   r!   )r  r  Zgates_wZgates_br  r   r   r   r  r  r  r  Zgatesr   r   r   rt  &	  s    


z=TestOperators.test_elman_recurrent_network.<locals>.referencec                 S   s   g | ]}t |qS r   r   r  r   r[   r   r   r   r   ;	  s   )r  r  r  r  r  )Zoutputs_to_check)r   r2   rd   c                 S   s   g | ]}t |qS r   r  r  r   r   r   r   E	  s   )'r\  r  rP  r   r   r   rO  r`  Zexternal_inputr  rb  r  r^  r  rz   ZSigmoidra  rZ   r   ZRunOperatorOncer
   ZGradientRegistryZGetBackwardPassr   r  r   r  rS   paramsr  r   r   rA   r  r@   rV   r)   ru  rU   )rW   r  rH   r[  r  rP  r  rZ   Zbackward_opsZbackward_mappingr  linksr  r  r  Zbackward_linksr  r  r  Zrecurrent_inputsrt  r   r   r  r   test_elman_recurrent_network  s    
  
     




   	z*TestOperators.test_elman_recurrent_network)Zsuppress_health_checkr   )rH   chr9  pad
block_sizec	                 C   s   t |d|  | dk t |d|  | dk tj||||tj}	tjddgdg||d}
| ||
|	gdg | 	||
|	gddg d S )Nr2   r   ZSpaceToBatchr   rL   r  r  
r   r   r   r   r   rA   r
   rS   rT   rU   rW   rH   r  r  r9  r  r  rX   rY   r   rZ   r   r   r   test_space_to_batchK	  s    	 z!TestOperators.test_space_to_batchc	                 C   s   t |d|  | dk t |d|  | dk tj|| | ||d|  | |d|  | tj}	tjddgdg||d}
| ||
|	gdg | 	||
|	gddg d S )Nr2   r   ZBatchToSpacer   rL   r  r  r  r   r   r   test_batch_to_space\	  s     	
 z!TestOperators.test_batch_to_spaceg       )r   rc   scalec                 C   sL   t jddg|sdndg|d}| |||gdg | |||gddg d S )NZScaler   rL   )r  r   r{   )rW   r   rc   r  rX   rY   rZ   r   r   r   
test_scaleq	  s      zTestOperators.test_scale)rn  c                 C   sv   | dd}| jd| | jjd d}| jd| | || jjd   | || jjd   d S )Nasciiignorerh  r   )	encoder   r   r   r   	serializeZdeserializer   r   )rW   rn  Z
serializedr   r   r   test_string_serde}	  s    zTestOperators.test_string_serder   r  ZconstantZreflectZedge)r  r   input_channels
batch_sizer  modec	                    s   t |k tjddgdg |d}	|dkr\tj||||tjd }
 fdd}n,tj||||tjd }
 fd	d}| ||	|
g| | 	||	|
gd
g | 
||	|
gd
d
g d S )NPadImager   rL   )r  r  r  r  r  c                    s    t | dffdf fS N)r   r   r   r  r   r  r  r   r   numpy_pad_ref	  s
      z8TestOperators.test_same_pad_image.<locals>.numpy_pad_refc                    s    t | ddfff fS r  r  r   r  r   r   r  	  s
      r   )r   r
   rS   r   r   r   r   rA   rV   rT   rU   )rW   r  r   r  r  r  r  rX   rY   rZ   r   r  r   r  r   test_same_pad_image	  sF    
      z!TestOperators.test_same_pad_image)	pad_tpad_lpad_bpad_rr   r  r  r  r  c                    s   t |tk tjddgdg |d	}|dkrrtj||||tjd } fdd}n2tj||||tjd } fd	d}| 	|
||g| | 
|||gd
g | |
||gd
d
g d S )Nr  r   rL   )r  r  r  r  r  r  r  r  c                    s    t | dffdf fS r  r  r   r  r  r  r  r  r   r   r  	  s
     z3TestOperators.test_pad_image.<locals>.numpy_pad_refc                    s    t | ddfff fS r  r  r   r  r   r   r  	  s
     r   )r   rr  r
   rS   r   r   r   r   rA   rV   rT   rU   )rW   r  r  r  r  r   r  r  r  r  rX   rY   rZ   r   r  r   r  r   test_pad_image	  sL          zTestOperators.test_pad_imager   )r   r  r  r  r%  c                    sV  t jddddgdg|d}tjd tjtjd }	tjtjd }
tj tjd }|d	kr|d
ddd} fddfdd}| 	||||	|
g|dkr̈n| t
j }d|fd|	fd|
fg}|D ]\}}||| qtdD ]}|| q|D ]"\}}tj|j|  | q.d S )NZInstanceNormr   r  biasrL   )r  r%  rR  r  r  r"   r2   rd   c                    s   |     } | | dd d tjf  }|t| d d d tjf  }|  }|| ddd }|| ddd }|fS r   )rG  Zmeanr   Znewaxisr+  rA  r    r  r  ri   )r  r%  r  r   r   r   ref_nchw	  s    &z2TestOperators.test_instance_norm.<locals>.ref_nchwc                    s:   |  dd dd}  | ||d }| dd ddfS )Nr2   rd   r"   r   )swapaxesr  )r  r   r   ref_nhwc	  s    z2TestOperators.test_instance_norm.<locals>.ref_nhwcr   re   )r
   rS   r   r   r   r   r   rA   r  rV   r   r   r   r   r   r   r   r   r   r   r   )rW   r   r  r  r  r%  rX   rY   rZ   r  r  r   r  r   ZfeedsZblobr  r   r   )r  r%  r  r  r   r   test_instance_norm	  sL    	   	  
z TestOperators.test_instance_norminpc                 C   s   t t| | dS )N)r.   r-   )r?   r)   rJ   )dtr   r   r   r=   
  s    c                 C   sz  |\}}}|j t dkr@|jtjtjhkr@td|j  d S |jtjtjhkr^|tj	}|j t dt dfkrt
t|  tdjk   t
t| tj tdjk  t|f|jdd   |j }tdddd	gd
g}tdddgd
g}	dd }
dd }| |||||g|
 | ||	||g| |j tjkrv| |||||gddg d S )NrA   z
Bailout {}r:   r@   r"   ZSparseToDenseIr   r   rL   c                 S   s>   t j|j|jd}t| D ]\}}||  || 7  < q|gS r~  )r   r~   r}   r-   r   )r  r   r   Or   r   r   r   r   sparse_to_dense-
  s    z;TestOperators.test_sparse_to_dense.<locals>.sparse_to_densec                 S   sV   t jt | d f|jdd   |jd}t| D ]\}}||  || 7  < q2|gS r  )r   r~   rr  r}   r-   r   )r  r   r  r   r   r   r   r   sparse_to_dense_noshapeinfer3
  s    *zHTestOperators.test_sparse_to_dense.<locals>.sparse_to_dense_noshapeinferr   )r-   r   r   r   ZCUDAZHIPr  r  r   r@   r   r   Zravelrr  r  allr:   r  r~   r}   r
   rS   rV   rA   rU   )rW   r  rX   rY   Z	first_dimr   r  r   rZ   Zop_noshapeinferr  r  r   r   r   test_sparse_to_dense
  s$    
"&*"z"TestOperators.test_sparse_to_densec                 C   s   |\}}t dddgd}dd }| ||||g| | ||||gdg | ||||gddg | ||||gddg d S )	NZ
DotProductr   rL   rP   c                 S   s   t j| | ddfS Nr"   r   )r   r  r   r   r   r   
dotproductD
  s    z2TestOperators.test_dot_product.<locals>.dotproductr   r"   )r
   rS   rV   rT   rU   )rW   rO   rX   rY   r   rL   rZ   r  r   r   r   test_dot_product>
  s    zTestOperators.test_dot_product)r   Mr   	pad_valuec                    s   t jt jd }t j t jd }tjdddgdd}	 fdd}
| ||	||g|
 | ||	||gd	g | 	||	||gd	d	g | 	||	||gd
d	g d S )Nr  DotProductWithPaddingr   rL   rP   )r  c                    s|   t t ft j } k rP| |d d d f< t j|| ddfS ||d d d  f< t j||  ddfS d S r  )r   r0  rr  r   rA   r  )r   rL   Zr   r
  r   r  r   r   r  X
  s     z?TestOperators.test_dot_product_with_padding.<locals>.dotproductr   r"   
r   r   r   r   rA   r
   rS   rV   rT   rU   )rW   r   r
  r   r  rX   rY   r   rL   rZ   r  r   r  r   test_dot_product_with_paddingL
  s    	z+TestOperators.test_dot_product_with_padding)r   r
  r  c           
         s   d  t j|t jd }t j| t jd }tjdddgdd|d} fd	d
}	| ||||g|	 | ||||gdg | 	||||gddg | 	||||gddg d S )Nr2   r  r  r   rL   rP   T)Z	replicater  c                    sd   dd l m}  k r:|| d  }tj|| ddfS ||d  }tj||  ddfS d S )Nr   r"   r   )Znumpy.matlibZmatlibZrepmatr   r  )r   rL   Znpmr  r   r
  r   r   r  s
  s    zCTestOperators.test_dot_product_with_rep_padding.<locals>.dotproductr   r"   r  )
rW   r   r
  r  rX   rY   r   rL   rZ   r  r   r  r   !test_dot_product_with_rep_paddingf
  s    	z/TestOperators.test_dot_product_with_rep_padding)r   r
  c                 C   s   t j||t jd }tddgd}| |||gdd  | |||gdg t j||t jd }tddgd}| |||gdd  | |||gdg d S )	Nr  ZEnsureDenser   c                 S   s   | gS r   r   r   r   r   r   r=   
  rr   z1TestOperators.test_ensure_dense.<locals>.<lambda>r   rP   c                 S   s   | gS r   r   r   r   r   r   r=   
  rr   )	r   r   r   r   rA   r
   rS   rV   rT   )rW   r   r
  rX   rY   r   rZ   r   r   r   test_ensure_dense
  s    zTestOperators.test_ensure_dense)r   r
  num_bucketsc           	         sz   t j t j}d\tjddgddgd} fdd}| |||gd	d
g | |||g| d S )N)r  r  ZAccumulateHistogramr   cur_histacc_hist)lower_boundupper_boundr  c                    s   t jd ft jd}  }t j ft jd}d|| k < d || k< | | k| k @   | d t j|| k| k @ < t|jd D ]0}t|jd D ]}||| |   d7  < qq|| }}||gS )Nr2   rK   r   r"   )r   r~   r@   r   r   r}   )r   histsegmentrL   r   rj  r  r  r
  r   r  r  r  r   r   	histogram
  s$    
z=TestOperators.test_accumulate_histogram_op.<locals>.histogramr   r"   )	r   r   r   r   rA   r
   rS   rT   rV   )	rW   r   r
  r  rX   rY   r   rZ   r  r   r  r   test_accumulate_histogram_op
  s    
z*TestOperators.test_accumulate_histogram_oprR   )queue_capacity
time_sleepnum_blobs_to_equeuenum_blobs_to_dequeuec              	      s    fdd}t d}d| _|jg d||d}t }||  t d}	d|	 _|		|g|}
||	}t
j||gd}|  | t |  |  W 5 Q R X d	S )
a8  
        Tests SafeDequeueBlobsOp being cancellable.

        Create a queue with the number of BlobsQueue less than the number
        SafeDequeueBlobs to cause the hanging behavior when running the Net.

        Then call cancel from the previous sleeping thread to ensure exception
        is raised.
        c                    s   t   |   d S r   )r  sleepcancel)net_instancer  r   r   _net_instance_cancel
  s    
z^TestOperators.test_safe_dequeue_blob__raises_exception_when_hang.<locals>._net_instance_cancelr  Zasync_schedulingZ
queue_namer  r  r  N)r
   r  r`  rb  r  r   r   Z
create_netr   r  r  r  r  r   r   r  )rW   r  r  r   r!  r&  r  r  r   r  r   r$  r  r   r%  r   2test_safe_dequeue_blob__raises_exception_when_hang
  s(    




z@TestOperators.test_safe_dequeue_blob__raises_exception_when_hang)}__name__
__module____qualname__rx   r   r)   rI   r'   rm   r_   r   r|   rv   r   r   r   r   r   r   r  r   r   r   r   r(   rB   Zexpanded_device_optionsr   r;   r   unittestZskipIfr   has_gpu_supportZfloatsr   r   r  r,   r  r   rA   r   r@   r  r  r#  r.  staticmethodr?  rF  rJ  rK  rT  rU  rW  rY  Z
gcs_no_hiprc  rv  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r%  r6  rA  rE  rF  rJ  rK  rf  rs  rv  rz  r|  r  r  r  r  r   r   r  r  r  r  rE   r:   r>   r  r  r  r	   Zfilter_too_muchr  r  r  textr  r  r  r  r  r	  r  r  r  r  r'  r   r   r   r   ro   ~   s  	 
 


	A





&
!

"
 
  	
!+
 
 
 
 
 



P




:



V"
+

'

+

*


"

*B

E



$


  
& 

r




	










 





	

%

0%



ro   __main__)r"   r#   )N)BrZ  r   r   r  	functoolsr   r   Zfuture.utilsr   r   Z
hypothesisr   r   r   r	   Zhypothesis.strategiesZ
strategiesr'   r+  r  r\  r
   r   r   r   Z"caffe2.python.hypothesis_test_utilpythonZhypothesis_test_utilr)   Zcaffe2.protor   ZInitOpsLibraryr,  r   r!   Z	compositer1   rA   r?   r  FLOATr@   ZINT32r  ZBOOLZuint8ZUINT8Zint8ZINT8Zuint16ZUINT16Zint16ZINT16r:   ZINT64r   ZDOUBLEr  rE   r_   ra   rn   ZHypothesisTestCasero   r(  mainr   r   r   r   <module>   s   



         
  
  
                     
