U
    ‰dñw  ã                   @   sL  d dl mZ d dlm  mZ d dlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlZd dlm Z m!Z! d dl"Z"d dl#Z#G d	d
„ d
eeƒZ$G dd„ deeƒZ%G dd„ deeeƒZ&G dd„ deeeƒZ'G dd„ deeƒZ(G dd„ deeƒZ)G dd„ deeeƒZ*G dd„ deeƒZ+G dd„ deeƒZ,G dd„ deeeƒZ-G dd„ deeeƒZ.G dd „ d eeeƒZ/G d!d"„ d"eeeƒZ0G d#d$„ d$eeeƒZ1G d%d&„ d&eeeƒZ2G d'd(„ d(eeeƒZ3G d)d*„ d*eeƒZ4G d+d,„ d,eeeƒZ5G d-d.„ d.eƒZ6G d/d0„ d0eƒZ7G d1d2„ d2eƒZ8dS )3é    )Ú
caffe2_pb2N)Ú	build_sgdÚbuild_multi_precision_sgdÚ
build_ftrlÚbuild_gftrlÚbuild_wngradÚbuild_adagradÚbuild_adadeltaÚ
build_adamÚbuild_yellowfinÚbuild_rms_propÚbuild_stormÚbuild_decay_adagradÚadd_weight_decayÚSgdOptimizer)ÚUseOptimizer)ÚOptimizerTestBaseÚLRModificationTestBase)ÚcoreÚ	workspace)ÚTestCase)Úassert_allcloseÚassert_equalc                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚTestLarsc                 C   s   t  d¡‚d S ©Nzno sparse support©ÚunittestZSkipTest©Úself© r   ú@/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/optimizer_test.pyÚ
testSparse   s    zTestLars.testSparsec                 K   s   d| _ t|fdddœ|—ŽS )NFçš™™™™™¹?ç      à?©Úbase_learning_rateÚlars©Ú	_skip_gpur   ©r   ÚmodelÚkwargsr   r   r    Úbuild_optimizer   s    zTestLars.build_optimizerc                 C   sX   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ](}t |¡}tjj	t 
dg¡|dd q*d S ©Nç      ð?çñhãˆµøä>©Zatol©Ú
assertTrueÚget_auxiliary_parametersÚsharedÚassertFalseÚlocalr   Ú	FetchBlobÚnpÚtestingr   Úarray©r   Ú	optimizerÚparamZtensorr   r   r    Úcheck_optimizer   s
    
zTestLars.check_optimizerN©Ú__name__Ú
__module__Ú__qualname__r!   r,   r>   r   r   r   r    r      s   r   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestMomentumSgdc                 K   s   d| _ t|fdddœ|—ŽS )NFr"   )r%   Zmomentumr'   r)   r   r   r    r,   '   s    zTestMomentumSgd.build_optimizerc                 C   sX   |   | ¡ j¡ |   | ¡ j¡ | ¡ jD ](}t |¡}tjjt 	dg¡|dd q*d S r-   )
r2   r3   r4   r6   r   r7   r8   r9   r   r:   r;   r   r   r    r>   +   s
    
zTestMomentumSgd.check_optimizerN©r@   rA   rB   r,   r>   r   r   r   r    rC   &   s   rC   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestSgdc                 K   s   d| _ t|fddi|—ŽS ©NFr%   r"   r'   r)   r   r   r    r,   4   s    zTestSgd.build_optimizerc                 C   sX   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ](}t |¡}tjj	t 
dg¡|dd q*d S r-   r1   r;   r   r   r    r>   8   s
    
zTestSgd.check_optimizerNrD   r   r   r   r    rE   3   s   rE   c                       s<   e Zd Zdd„ Zdd„ Ze ej d¡‡ fdd„ƒZ	‡  Z
S )ÚTestMultiPrecisionSgdc                 K   s   d| _ t|fddi|—ŽS rF   )r(   r   r)   r   r   r    r,   C   s    ÿÿÿz%TestMultiPrecisionSgd.build_optimizerc                 C   sX   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ](}t |¡}tjj	t 
dg¡|dd q*d S r-   r1   r;   r   r   r    r>   I   s
    
z%TestMultiPrecisionSgd.check_optimizerzNo GPU supportc                    s   t t| ƒ tjj¡ d S ©N)ÚsuperrG   ÚtestGPUDenser   ÚDataTypeZFLOAT16r   ©Ú	__class__r   r    rJ   P   s    z"TestMultiPrecisionSgd.testGPUDense)r@   rA   rB   r,   r>   r   ÚskipIfr   Úhas_gpu_supportrJ   Ú__classcell__r   r   rL   r    rG   @   s   rG   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestFtrlc                 K   s"   d| _ t|fd dddddœ|—ŽS ©NTr.   r"   ç        )ZengineÚalphaÚbetaZlambda1Zlambda2)r(   r   r)   r   r   r    r,   V   s    ÿúùzTestFtrl.build_optimizerc                 C   s>   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ]}t |¡ q*d S rH   ©r5   r3   r4   r2   r6   r   r7   ©r   r<   r=   r   r   r    r>   b   s    zTestFtrl.check_optimizerNrD   r   r   r   r    rQ   U   s   rQ   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )Ú	TestGFtrlc                 C   s   t  d¡‚d S r   r   r   r   r   r    r!   j   s    zTestGFtrl.testSparsec                 K   s"   d| _ t|fd dddddœ|—ŽS rR   )r(   r   r)   r   r   r    r,   m   s    ÿúùzTestGFtrl.build_optimizerc                 C   s>   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ]}t |¡ q*d S rH   rV   rW   r   r   r    r>   y   s    zTestGFtrl.check_optimizerNr?   r   r   r   r    rX   i   s   rX   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestAdagradc                 K   s   d| _ t|fdddœ|—ŽS )NFr.   r#   r$   ©r(   r   r)   r   r   r    r,      s    zTestAdagrad.build_optimizerc                 C   s>   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ]}t |¡ q*d S rH   rV   rW   r   r   r    r>   …   s    zTestAdagrad.check_optimizerNrD   r   r   r   r    rY   €   s   rY   c                   @   s,   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	S )
ÚTestRowWiseAdagradc                 K   s   d| _ t|fddddœ|—ŽS )NTr.   r#   )r%   r&   ÚrowWiserZ   r)   r   r   r    r,      s    ÿ  ÿÿz"TestRowWiseAdagrad.build_optimizerc                 C   s>   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ]}t |¡ q*d S rH   rV   rW   r   r   r    r>   “   s    z"TestRowWiseAdagrad.check_optimizerc                 C   s   t  d¡‚d S ©Nzno dense supportr   r   r   r   r    Ú	testDense™   s    zTestRowWiseAdagrad.testDensec                 C   s   t  d¡‚d S r]   r   r   r   r   r    rJ   œ   s    zTestRowWiseAdagrad.testGPUDenseN©r@   rA   rB   r,   r>   r^   rJ   r   r   r   r    r[   Œ   s   r[   c                   @   s,   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	S )
ÚTestRowWiseAdagradWithCounterc                 K   s    d| _ t|fdddddœ|—ŽS )NTr.   r#   é   )r%   r&   r\   Zcounter_halfliferZ   r)   r   r   r    r,       s    ÿûúz-TestRowWiseAdagradWithCounter.build_optimizerc                 C   sŒ   |   | ¡ j¡ |   | ¡ j¡ |   t d¡¡ t d¡}tjj	t 
dg¡|dd | ¡ jD ]}t |¡ q^| ¡ jD ]}t |¡ qxd S ©NÚoptimizer_iterationiÐ  r/   r0   ©r2   r3   r4   r6   r   ÚHasBlobr7   r8   r9   r   r:   ©r   r<   Ziteration_tensorr=   r   r   r    r>   «   s    
þz-TestRowWiseAdagradWithCounter.check_optimizerc                 C   s   t  d¡‚d S r]   r   r   r   r   r    r^   ¸   s    z'TestRowWiseAdagradWithCounter.testDensec                 C   s   t  d¡‚d S r]   r   r   r   r   r    rJ   »   s    z*TestRowWiseAdagradWithCounter.testGPUDenseNr_   r   r   r   r    r`   Ÿ   s   r`   c                   @   s   e Zd Zdd„ Zdd„ ZdS )Ú
TestWngradc                 K   s   d| _ t|fddi|—ŽS )NTr%   g      9@)r(   r   r)   r   r   r    r,   ¿   s    zTestWngrad.build_optimizerc                 C   s>   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ]}t |¡ q*d S rH   rV   rW   r   r   r    r>   Ã   s    zTestWngrad.check_optimizerNrD   r   r   r   r    rg   ¾   s   rg   c                   @   s   e Zd Zdd„ Zdd„ ZdS )Ú	TestStormc                 K   s   d| _ t|fddi|—ŽS )NTr%   ç       @)r(   r   r)   r   r   r    r,   Ë   s    zTestStorm.build_optimizerc                 C   s>   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ]}t |¡ q*d S rH   rV   rW   r   r   r    r>   Ï   s    zTestStorm.check_optimizerNrD   r   r   r   r    rh   Ê   s   rh   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestAdadeltac                 K   s   d| _ t|fdddœ|—ŽS )NFr.   g×£p=
×ï?)r%   Zdecay)r(   r	   r)   r   r   r    r,   ×   s    zTestAdadelta.build_optimizerc                 C   s>   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ]}t |¡ q*d S rH   rV   rW   r   r   r    r>   Û   s    zTestAdadelta.check_optimizerNrD   r   r   r   r    rj   Ö   s   rj   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestAdamc                 K   s   d| _ t|fddi|—ŽS rF   ©r(   r
   r)   r   r   r    r,   ã   s    zTestAdam.build_optimizerc                 C   sŒ   |   | ¡ j¡ |   | ¡ j¡ |   t d¡¡ t d¡}tjj	t 
dg¡|dd | ¡ jD ]}t |¡ q^| ¡ jD ]}t |¡ qxd S rb   rd   rf   r   r   r    r>   ç   s    
þzTestAdam.check_optimizerNrD   r   r   r   r    rk   â   s   rk   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestSmartDecayAdamc                 K   s$   d| _ d|d< t|fdddœ|—ŽS )NFrS   Zbeta1r"   T)r%   Zuse_smart_decayrl   r)   r   r   r    r,   õ   s    z"TestSmartDecayAdam.build_optimizerc                 C   sˆ   |   | ¡ j¡ |   | ¡ j¡ |   t d¡¡ t ¡ }|   tdd„ |D ƒƒ¡ | ¡ jD ]}t |¡ qZ| ¡ jD ]}t |¡ qtd S )Nrc   c                 s   s   | ]}|  d ¡V  qdS )Z
_last_seenN)Úendswith)Ú.0Zbnr   r   r    Ú	<genexpr>ÿ   s     z5TestSmartDecayAdam.check_optimizer.<locals>.<genexpr>)	r2   r3   r4   r6   r   re   ZBlobsÚanyr7   )r   r<   Z
blob_namesr=   r   r   r    r>   ú   s    z"TestSmartDecayAdam.check_optimizerNrD   r   r   r   r    rm   ô   s   rm   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚTestDecayAdagradc                 K   s   d| _ t|fddi|—ŽS )NTr%   r.   )r(   r   r)   r   r   r    r,     s    z TestDecayAdagrad.build_optimizerc                 C   sŒ   |   | ¡ j¡ |   | ¡ j¡ |   t d¡¡ t d¡}tjj	t 
dg¡|dd | ¡ jD ]}t |¡ q^| ¡ jD ]}t |¡ qxd S rb   rd   rf   r   r   r    r>   
  s    
þz TestDecayAdagrad.check_optimizerc                 C   s   t  d¡‚d S r   r   r   r   r   r    r!     s    zTestDecayAdagrad.testSparseN©r@   rA   rB   r,   r>   r!   r   r   r   r    rr     s   rr   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestSparseRAdamc                 K   s   d| _ t|fdddœ|—ŽS )NTr"   )r%   ZenableRAdamrl   r)   r   r   r    r,     s    zTestSparseRAdam.build_optimizerc                 C   sŒ   |   | ¡ j¡ |   | ¡ j¡ |   t d¡¡ t d¡}tjj	t 
dg¡|dd | ¡ jD ]}t |¡ q^| ¡ jD ]}t |¡ qxd S rb   rd   rf   r   r   r    r>     s    
þzTestSparseRAdam.check_optimizerNrD   r   r   r   r    rt     s   rt   c                   @   s€   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
e d¡dd„ ƒZe d¡e ej d¡dd„ ƒƒZdS )ÚTestYellowFinc                 C   s   d| _ t|ddS )NFr"   ©r%   )r(   r   )r   r*   r   r   r    r,   0  s    zTestYellowFin.build_optimizerc                 C   sŒ   |   | ¡ j¡ |   | ¡ j¡ |   t d¡¡ t d¡}tjj	t 
dg¡|dd | ¡ jD ]}t |¡ q^| ¡ jD ]}t |¡ qxd S rb   rd   rf   r   r   r    r>   4  s    
þzTestYellowFin.check_optimizerc                 C   s   t  d¡‚d S r   r   r   r   r   r    r!   A  s    zTestYellowFin.testSparsec                 C   s   |r|d||   S |S d S )Nr.   r   )r   ÚvalrU   ÚiÚzero_debiasr   r   r    ÚdebD  s    zTestYellowFin.debc                 C   s"  |dkrN|| }t  |¡d t  |¡d  d }dt  |¡ d | }||fS |d |d  d | }t || d| | |  ¡ | d }	|	dkršdndt t |	¡d	¡ }
|
|d
 |
  }|d }t|dƒ}|| }tt  |¡d t  |¡d  d |d ƒ}dt  |¡ d | }||fS )Nr   é   é   gh/¡½„öÂ?ri   rS   r.   g      ð¿gUUUUUUÕ?g      @gé!çýÿï?)r8   ÚsqrtÚmathÚpowÚfabsÚminÚmax)r   ZdistanceZgrad_varÚh_minÚh_maxZdrÚmuZlr_minÚpZw3ÚwÚyÚrootr   r   r    Ú	get_lr_muJ  s     ("
*zTestYellowFin.get_lr_muc               
   C   s6  i }d}d}d}	d}
d}t  d¡}t  d¡}t ¡  t  t  tj¡¡@ |jg dd	gd
t j	j
d}| g dg¡}| ||g|g¡ W 5 Q R X |jg d|g|d}|r²| |gd¡}| |gd¡}|j||gddd}|jg d|gdd}dd„ }||_||_tj|||	|
||d |||¡ t |¡ tj|dd t|ƒD ]ô}t |¡ t d¡}|d	 }|d }|d }|d }t d¡}t d¡d
 }t d¡d
 }|  ||	|d	 |¡}t|  ||	|d	 |¡| |¡ |ƒ}|d
kr<t |  ||	|d	 |¡¡t |  ||	|d	 |¡¡||  ||	|d	 |¡||d œ||< q<|S )!Nr.   rS   ç+‡ÙÎ÷ï?é   gíµ ÷Æ°>ÚnetÚparam_init_netÚ	iterationr{   r   )ÚshapeÚvalueZdtypeZiteration_mutexÚpre_grad)r   r‘   Ziteration_cpuÚiteration_floatÚgradT)Ú	broadcastr‡   c                   S   s   d S rH   r   r   r   r   r    Ú<lambda>ƒ  ó    z0TestYellowFin.caffe2_yellowfin.<locals>.<lambda>)rT   r…   rU   Úcurv_win_widthÚepsilonry   )Ú	overwriteZw_scalars_memoryr|   é   é   Zw_g_avgZw_lr_avgZw_mu_avg©r„   rƒ   ÚvarÚdistÚlrr…   )r   ZNetr   ZResetWorkspaceÚDeviceScopeÚDeviceOptionr   ÚCPUZConstantFillrK   ZINT64ZCreateMutexZ
AtomicIterZCopyCPUToGPUZCastZMulZblobr”   r<   ZYellowFinOptimizerZ_runZ
RunNetOnceZ	CreateNetÚrangeZRunNetr7   rz   r‚   Údotr8   Úexp) r   ry   Ú	grad_coefÚn_dimÚn_iterÚgpuZ
caffe2_resrT   r…   rU   r˜   r™   r   rŽ   r   Z
iter_mutexr’   r“   r”   r‡   Z
param_inforx   Zscalars_memory_blobZg_norm2_avgZg_norm2_min_avgZg_norm2_max_avgZdistance_avgZ
g_avg_blobZres_lrZres_muZg_debZvariancer   r   r    Úcaffe2_yellowfin]  s¨    

ûüþúö



ÿý

ý
ýôzTestYellowFin.caffe2_yellowfinc                 C   s€  i }d}d}d}	d}
d}d}d}d}t |ƒD ]L}|d | }d|	 dt |t |g¡ d ¡  }	d|
 dtj |t |g¡ ¡  }
d| d|  }d| dt |d | ¡  }d| dt td|d d ƒ| d | ¡  }|r"|	dd|d    |d | dd|d   d   }n|	|d |  }d| d|
 |	  }|dkr,|r|  |dd|d    |t 	|dd|d    ¡t 	|dd|d    ¡¡\}}d| d|  }d| d|  }t 	|dd|d    ¡t 	|dd|d    ¡||dd|d    ||d	œ||< q,|  ||t 	|¡t 	|¡¡\}}d| d|  }d| d|  }t 	|¡t 	|¡||||d	œ||< q,|S )
NrS   r.   r{   r‹   gü©ñÒMbP?r|   rŒ   r   r   )
r¤   r8   ÚsumZonesZlinalgZnormÚlogr‚   rŠ   r¦   )r   ry   r§   r¨   r©   rª   Z	numpy_resZtarget_h_maxZtarget_h_minZtarget_g_norm_squared_avgZtarget_g_norm_avgZtarget_g_avgZtarget_dist_avgZ	target_lrZ	target_murx   Zgrad_valZ
target_varr    r…   r   r   r    Únumpy_yellowfin·  sŠ    ÿÿÿ&ÿÿþ
ÿ
ÿüú	üúzTestYellowFin.numpy_yellowfinc                 C   sÎ   ||||||ƒ}||||||ƒ}	t t|ƒt|	ƒƒ tdt|ƒƒD ]ˆ}
t ||
  ¡ |	|
  ¡ ƒ ||
  ¡ D ]\}dt|
ƒ d d | d d t|ƒ d d t|ƒ }t||
 | |	|
 | d|d qjq@d S )	Nr{   zi=z,
zfeat=z
grad_coef=zzero_debias=ç{®Gáz„?)ZrtolÚerr_msg)r   Úlenr¤   ÚkeysÚstrr   )r   Zmodel0Zmodel1ry   r§   r¨   r©   rª   Z
model0_resZ
model1_resrx   Zfeatr°   r   r   r    Úcompare_yellowfin_modelsü  s:    ÿÿÿþþþýýÿ
ýz&TestYellowFin.compare_yellowfin_modelsz5Results might vary too much. Only for individual use.c                 C   sX   d}d}t  tj¡}t  |¡0 dD ]$\}}| j| j| j||||dd q$W 5 Q R X d S )Né@B é2   ))Fr.   )Fr"   )Fr¯   )Tr.   F©rª   )r   r¢   r   r£   r¡   r´   r«   r®   )r   r¨   r©   Zcpu_device_optry   r§   r   r   r    Útest_caffe2_cpu_vs_numpy  s    ùz&TestYellowFin.test_caffe2_cpu_vs_numpyzNo gpu supportc                 C   s`   d}d}t  tjd¡}t  |¡6 dD ]*}dD ] }| j| j| j||||dd q.q&W 5 Q R X d S )Nrµ   r¶   r   )FT)r.   r"   r¯   Tr·   )r   r¢   r   ZGpuDeviceTyper¡   r´   r«   r®   )r   r¨   r©   Zgpu_device_optry   r§   r   r   r    Útest_caffe2_gpu_vs_numpy*  s    ùz&TestYellowFin.test_caffe2_gpu_vs_numpyN)r@   rA   rB   r,   r>   r!   rz   rŠ   r«   r®   r´   r   Úskipr¸   rN   r   rO   r¹   r   r   r   r    ru   -  s   ZE
ru   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚTestRmsPropc                 K   s   d| _ t|fdddœ|—ŽS )NFr"   )r%   r™   )r(   r   r)   r   r   r    r,   ?  s    ÿ ÿÿzTestRmsProp.build_optimizerc                 C   s>   |   | ¡ j¡ |  | ¡ j¡ | ¡ jD ]}t |¡ q*d S rH   rV   rW   r   r   r    r>   E  s    zTestRmsProp.check_optimizerc                 C   s   t  d¡‚d S r   r   r   r   r   r    r!   K  s    zTestRmsProp.testSparseNrs   r   r   r   r    r»   >  s   r»   c                   @   s   e Zd Zdd„ ZdS )ÚTestMultiOptimizersc              	      s<  ddl m}m}m‰  ddlm} |dd}| |dddd	¡}| ||d
d	d¡}| ||ddd¡}| |dgddg¡\}}	| |	g¡ ˆ  	|¡‰‡ ‡fdd„}
ˆ j
dd}ˆ j
dd}ˆ  ¡ }| |
dƒ¡ ||j|jddƒ W 5 Q R X | |
dƒ¡ ||j|jddƒ W 5 Q R X g }|j ¡ jD ]B}|jdkrD|jd dksT|jd dkr$| |jd ¡ q$|  |d |d ¡ | |
dƒ¡ ||j|jddƒ W 5 Q R X | |
dƒ¡ ||j|jdd ƒ W 5 Q R X g }|j ¡ jD ]V}|jdkr|jd dks|jd dkrä|  |jd |k¡ | |jd ¡ qä|  |d |d ¡ | |
d!ƒ¡ ||j|jd!d"ƒ W 5 Q R X | |
d#ƒ¡ ||j|jd#d$ƒ W 5 Q R X g }|j ¡ jD ]j}|jd%krØ|jd d!ksè|jd d#kr¸|  |jd |k¡ |  |jd |k¡ | |jd ¡ q¸|  |d |d ¡ d S )&Nr   )Úbrewr   r<   ©ÚModelHelperÚtest)ÚnameÚdataÚfc1éd   r¶   Úfc2é   Zfc3é
   ÚlabelÚsoftmaxÚlossc                    s   ˆ   | d | ¡ˆ¡S )Nz{}_grad)Zget_param_deviceÚformat)Z	blob_name©r<   Zparam_to_devicer   r    Úinfer_blob_device`  s
      ÿzGTestMultiOptimizers.test_multiple_optimizers.<locals>.infer_blob_devicer"   rv   çš™™™™™É?Zfc1_wZ
fc1_w_gradZfc1_bZ
fc1_b_gradÚWeightedSumr›   r{   Zfc2_wZ
fc2_w_gradZfc2_bZ
fc2_b_gradZfc3_wZ
fc3_w_gradZfc3_bZ
fc3_b_gradZAdagrad)Úcaffe2.pythonr½   r   r<   Úcaffe2.python.model_helperr¿   ÚfcÚSoftmaxWithLossÚAddGradientOperatorsZ_get_param_to_devicer   ZAdagradOptimizerr¡   r   rŽ   ÚProtoÚopÚtypeÚinputÚappendÚassertEqualr2   )r   r½   r   r¿   r*   rÃ   rÅ   ÚpredrÉ   rÊ   rÍ   Zsgd_1Zsgd_2ZadagradZfc1_lr_blobsrÖ   Zfc2_lr_blobsZfc3_lr_blobsr   rÌ   r    Útest_multiple_optimizersP  sh    
þ
ÿÿÿz,TestMultiOptimizers.test_multiple_optimizersN)r@   rA   rB   rÜ   r   r   r   r    r¼   O  s   r¼   c                   @   s   e Zd Zdd„ ZdS )ÚTestWeightDecayc                 C   s*  ddl m} ddlm} |dddid}| |dd	d
d
d¡}| ||ddd¡}| ||ddd¡}| |dgddg¡\}}| |g¡ t|dd t	|dƒ dddh}	|j
 ¡ jD ]^}
|
jdkr°d|
jkr°|
jd |	krêtd |
jd ¡ƒ |  |
jd |	k¡ |	 |
jd ¡ q°|  |	tƒ d |	¡¡ d S )Nr   )r½   r¾   rÀ   ÚorderÚNCHW©rÁ   Z	arg_scoperÂ   Úcnvé    rœ   ÚarÄ   éÈ   Úbra   rÈ   rÉ   rÊ   ç-Cëâ6?©Zweight_decayç)\Âõ(¼?Úb_w_gradÚa_w_gradÚ
cnv_w_gradrÏ   Úwd_0_0ú%Unexpected param for weight_decay: {}ú Not all weights were decayed: {})rÐ   r½   rÑ   r¿   ÚconvrÒ   rÓ   rÔ   r   r   r   rÕ   rÖ   r×   rØ   ÚoutputÚprintrË   r2   ÚremoverÚ   Úset)r   r½   r¿   r*   rá   rã   rÛ   rÉ   rÊ   Úexpected_weight_gradrÖ   r   r   r    Útest_weight_decay“  s:    þ

ÿÿýz!TestWeightDecay.test_weight_decayN)r@   rA   rB   rõ   r   r   r   r    rÝ   ‘  s   rÝ   c                   @   s   e Zd Zdd„ ZdS )ÚTestOptimizerContextc              
   C   s  ddl m}m} ddlm} |dddid}|jd }td	ƒ}td
ƒ}tdƒ}t|ƒ | |ddddd¡}	W 5 Q R X t||dœƒ | 	||	ddd¡}
W 5 Q R X | 	||
ddd¡}| 
|dgddg¡\}}| |g¡ t|dd t|dƒ dddh}d |¡d d |d! ¡d"d |d# ¡d$d |d% ¡d&i}|j ¡ jD ]¬}|jd'kr d(|jkr |jd |kr|td) |jd ¡ƒ |  |jd |k¡ | |jd ¡ |jd*kr<d}|jD ]}|jd+kr¶|j}q¶|  |||jd  ¡ q<|  |tƒ d, |¡¡ d S )-Nr   )r½   r<   r¾   rÀ   rÞ   rß   rà   r   g333333Ã?rÎ   r"   rÂ   rá   râ   rœ   )ZWEIGHTZBIASrã   rÄ   rä   rå   ra   rÈ   rÉ   rÊ   ræ   rç   rè   ré   rê   rë   zSgdOptimizer_{}_lr_cpug333333Ã¿r{   gš™™™™™É¿r|   gš™™™™™¹¿r›   g)\Âõ(¼¿rÏ   rì   rí   ZLearningRateZbase_lrrî   )rÐ   r½   r<   rÑ   r¿   Z_optimizer_instance_countr   r   rï   rÒ   rÓ   rÔ   r   r   rË   r   rÕ   rÖ   r×   rØ   rð   rñ   r2   rò   ÚargrÁ   ÚfZassertAlmostEqualrÚ   ró   )r   r½   r<   r¿   r*   ÚcountZ	cnv_optimZweight_optimZ
bias_optimrá   rã   rÛ   rÉ   rÊ   rô   Zexpected_learning_raterÖ   rw   r÷   r   r   r    Útest_optimizer_context»  sj    

þ

    üÿÿ

þýz+TestOptimizerContext.test_optimizer_contextN)r@   rA   rB   rú   r   r   r   r    rö   ¹  s   rö   )9Zcaffe2.protor   Zcaffe2.python.optimizerÚpythonr<   r   r   r   r   r   r   r	   r
   r   r   r   r   r   r   Zcaffe2.python.optimizer_contextr   Z!caffe2.python.optimizer_test_utilr   r   rÐ   r   r   Zcaffe2.python.test_utilr   Znumpyr8   Znumpy.testingr   r   r~   r   r   rC   rE   rG   rQ   rX   rY   r[   r`   rg   rh   rj   rk   rm   rr   rt   ru   r»   r¼   rÝ   rö   r   r   r   r    Ú<module>   sJ   @
  ÿ  B(