U
    ‰dã#  ã                   @   sl   d dl Z d dlZd dlmZmZmZmZmZ d dl	m
Z
mZ d dlmZ G dd„ deƒZG dd„ deƒZdS )	é    N)ÚbrewÚcoreÚ	workspaceÚcnnÚ	optimizer)ÚInitializerÚPseudoFP16Initializer)ÚModelHelperc                   @   sP   e Zd ZdZejjfdd„Zdd„ Ze	 
ej d¡ejjfdd„ƒZd	d
„ ZdS )ÚOptimizerTestBaseú³
    This is an abstract base class.
    Don't inherit from unittest.TestCase, and don't name it 'Test*'.
    Do, however, do these things in classes which inherit from this.
    c                 C   s"  t  dddddg¡ t j¡}t j d¡ |tjjkr:t jnt j	}|tjjkrPt
nt}t jjdd|jfd |¡}t  ||¡d d …t jf }td	d
did}tj|dd|jddi fdi fd||d
}|tjjkrÞ| ||d ¡}| |dg¡}	| |	d¡}
| |
g¡}|  |d tj¡ ||||fS )Né   é   é   r   é   é{   é   ©ÚsizeÚtestÚorderÚNCHW)ÚnameZ	arg_scopeÚdataÚfcÚConstantFill)ÚaxisZWeightInitializerZBiasInitializerZ_fp32ÚlabelÚavg_lossÚfc_w)ÚnpÚarrayÚastypeÚfloat32ÚrandomÚseedr   ÚDataTypeÚFLOATZfloat16r   r   Úrandintr   ÚdotÚnewaxisr	   r   r   ÚFLOAT16ÚHalfToFloatÚSquaredL2DistanceÚAveragedLossÚAddGradientOperatorsÚassertIsInstanceZBlobReference)ÚselfÚdtypeÚperfect_modelZnumpy_dtypeZinitializerr   r   ÚmodelÚoutÚsqÚlossÚgrad_map© r8   úE/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/optimizer_test_util.pyÚ_createDense   sB    ÿþþ      üzOptimizerTestBase._createDensec                 C   sÚ   |   ¡ \}}}}|  |¡}t d|d ¡ t d|d ¡ t |j¡ t |jd¡ tdƒD ]H}t	j
 |jd ¡}t d|| ¡ t d|| ¡ t |j ¡ j¡ q\t	jj|t	jd d …f t d¡dd |  |¡ d S )	Nr   r   r   TéÐ  r   ç{®Gáz„?©Zatol)r:   Úbuild_optimizerr   ÚFeedBlobÚ
RunNetOnceÚparam_init_netÚ	CreateNetÚnetÚranger   r#   r'   ÚshapeÚRunNetÚProtor   ÚtestingÚassert_allcloser)   Ú	FetchBlobÚcheck_optimizer)r0   r3   r2   r   r   r   Ú_Úidxr8   r8   r9   Ú	testDense2   s"    
ýzOptimizerTestBase.testDensezNo gpu supportc              	   C   sâ   t  tjd¡}t  |¡j |  |¡\}}}}|t jjkrR| dd¡}| 	|d¡ n| 	dd¡ t 
d|d ¡ t 
d|d ¡ W 5 Q R X tj|dddd	dd
 |  |¡ | jr²d S t |j¡ t |jd¡ t |j ¡ j¡ d S )Nr   r   Úfc_fp32_for_hostZfc_cpur   r   Zfc2r   é
   )Zdim_inZdim_outr   T)r   ZDeviceOptionr   ZGpuDeviceTypeZDeviceScoper:   r%   r*   r+   ZCopyGPUToCPUr?   r   r   r>   Z	_skip_gpur@   rA   rB   rC   rF   rG   r   )r0   r1   Z
device_optr3   Z_perfect_modelr   r   rO   r8   r8   r9   ÚtestGPUDenseF   s     
zOptimizerTestBase.testGPUDensec              	   C   sú  d}t  dddddg¡ t j¡}t j d¡ t jjdd|j| fd t j¡}t  |t  	||¡¡}t
jd	d
d}|jjg d|jgdd}|j |¡ |j |dgd¡}| |d¡}| |dg¡}	| |	d¡}
| |
g¡}|  |d tj¡ |  |¡}t d¡ t d¡ t jt jfD ]Ö}t |j¡ t |jd¡ t dƒD ]”}t j |j!d ¡}t  	t  "|j¡|¡|| dk }|jdkr€q:t #d| $|jf¡ |¡¡ t #dt  || ¡ t j¡¡ t %|j &¡ j'¡ q:t j(j)|t *d¡dd q|  +|¡ d S )Nr   r   r   r   r   r   r   r   r   r   )r   Úwç        )rE   ÚvalueÚindicesZgatherÚsumr   r   Tr;   r<   r=   ),r   r    r!   r"   r#   r$   r'   r   r(   Úrepeatr   ZCNNModelHelperrA   r   ÚparamsÚappendrC   ZGatherZReduceFrontSumr,   r-   r.   r/   r   ÚGradientSlicer>   r   Z
CreateBlobZint32Zint64r@   rB   rD   rE   Zaranger?   ZreshaperF   rG   r   rH   rI   rJ   rK   )r0   ZDUPLICATIONr2   r   r   r3   rR   Zpickedr4   r5   r6   r7   r   Zindices_typerL   rM   rU   r8   r8   r9   Ú
testSparsea   sj    þþ   ÿ


ÿ
ÿþÿý
zOptimizerTestBase.testSparseN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r%   r&   r:   rN   ÚunittestZskipIfr   Zhas_gpu_supportrQ   r[   r8   r8   r8   r9   r
      s   r
   c                   @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚLRModificationTestBaser   c                 C   s†   ddl m} d}|D ]N}t|j| |jƒs4|j| n
|j| j}t |¡}|t 	tj
 |¡d¡7 }qt |¡}	|}
|
t |
|	¡ }|S )Nr   )r   rS   g       @)Úcaffe2.pythonr   Ú
isinstanceÚparam_to_gradrZ   Úvaluesr   rJ   r   ÚpowerZlinalgZnormÚsqrtÚmaximum)r0   r3   rX   Úmax_gradient_normr   Zsum_squared_normsÚparamZgradÚvalZglobal_normZ	clip_normÚ
norm_ratior8   r8   r9   Ú_gradient_ratio_referencež   s     þÿ
û

z0LRModificationTestBase._gradient_ratio_referencec                 C   s"  d}|   ¡ \}}}}| j||d}g }|jddD ]*}||jkr2t|j| tjƒs2| |¡ q2t 	d|d ¡ t 	d|d ¡ t 
|j¡ t |jd¡ |  |j¡ tj |jd ¡}	t 	d||	 ¡ t 	d||	 ¡ t |j ¡ j¡ |  |||¡}
t d¡}tj ||
¡ |  |
dk d	¡ d S )
Ng      ð?)ri   T)Z	top_scoper   r   r   z#norm_clipped_grad_update/norm_ratioz$Bad test, gradient not being scaled.)r:   r>   Z	GetParamsrd   rc   r   rZ   rY   r   r?   r@   rA   rB   rC   ÚassertIsNotNoneÚ_lr_multiplierr   r#   r'   rE   rF   rG   r   rm   rJ   rH   Zassert_almost_equalÚ
assertTrue)r0   ri   r3   r2   r   r   ÚoptrX   rj   rM   Ú	referencerl   r8   r8   r9   Ú(test_global_norm_based_gradient_clipping°   sB    
þýÿ ÿz?LRModificationTestBase.test_global_norm_based_gradient_clippingc                 C   s¸   |   ¡ \}}}}| j|ddd}t d|d ¡ t d|d ¡ t |j¡ t |jd¡ |  |j	¡ |  
t ¡ d¡ t d¡ |  
t ¡ d¡ t |j ¡ j¡ |  
t d¡d¡ d S )Nr   T)ri   Zallow_lr_injectionr   r   r   Zlr_multiplier)r:   r>   r   r?   r@   rA   rB   rC   rn   ro   ÚassertEqualr   Zget_lr_injectionZset_lr_injectionrF   rG   r   rJ   )r0   r3   r2   r   r   rq   r8   r8   r9   Útest_lr_injectionÖ   s       ÿ
z(LRModificationTestBase.test_lr_injectionN)r\   r]   r^   r_   rm   rs   ru   r8   r8   r8   r9   ra   —   s   &ra   )r`   Znumpyr   rb   r   r   r   r   r   Z#caffe2.python.modeling.initializersr   r   Zcaffe2.python.model_helperr	   Úobjectr
   ra   r8   r8   r8   r9   Ú<module>   s    