U
    ‰d(  ã                   @   s¶   d dl m  mZ d dlmZ d dlZd dl	m
Z d dlmZmZmZmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ G dd	„ d	eƒZG d
d„ deƒZdS )é    N)ÚcoreÚlayer_model_instantiatorÚregularizerÚschemaÚ	workspace)ÚLayersTestCase)ÚSgdOptimizer)ÚL1NormÚRegularizationBy)ÚRegularizerContextÚUseRegularizer)Úgivenc                   @   s*   e Zd Zeejddgdddd„ ƒZdS )ÚTestRegularizerContexté   é   )Údims©ÚXc              	      sR  t dƒ}t dƒ}||dœ}d}|  t tjdf¡¡}t ||g¡ t|ƒþ t 	¡  
d¡}t 	¡  
d¡}tdƒ}	||ks€td	ƒ‚||kstd
ƒ‚| jj|||	|	||d}
t d|
f¡| j_|  t tj|ff¡|
¡ t | j¡\}}| ¡ j‰ ‡ fdd„ttˆ ƒƒD ƒ}| d¡dkst‚| d¡dks0t‚| d¡dksDt‚W 5 Q R X d S )Ngš™™™™™É?r   )ÚWEIGHTÚBIASr   )r   r   r   g333333Ã?z+fail to get correct weight reg from contextz)fail to get correct bias reg from context)Zweight_optimZ
bias_optimÚ
weight_regÚbias_regÚ	fc_outputc                    s   g | ]}ˆ | j ‘qS © )Útype)Ú.0Úi©Úopsr   úB/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/regularizer_test.pyÚ
<listcomp>2   s     zCTestRegularizerContext.test_regularizer_context.<locals>.<listcomp>ZLpNormZScaleé   ZLpNormGradient)r	   Z
new_recordr   ZScalarÚnpÚfloat32Z
FeedRecordr   r   ÚcurrentZget_regularizerr   ÚAssertionErrorÚmodelZFCÚStructZoutput_schemaÚassertEqualr   Zgenerate_training_netsZProtoÚopÚrangeÚlenÚcount)Úselfr   Zweight_reg_outZbias_reg_outZregularizersZoutput_dimsZinput_recordr   r   Zoptimr   Ú_Ú	train_netZops_type_listr   r   r   Útest_regularizer_context   s>    

ÿþú	
z/TestRegularizerContext.test_regularizer_contextN)Ú__name__Ú
__module__Ú__qualname__r   ÚhuÚarraysr0   r   r   r   r   r      s   r   c                   @   sZ  e Zd Zeejddgejddddddd	„ ƒZef ejddgejdddde 	¡ e 	¡ ejd
ddejdddejddddœej
—Ždd„ ƒZee dd¡e dd¡e dd¡ddd„ ƒZee dd¡e dd¡e dd¡ddd„ ƒZee dd¡e dd¡e dd¡e dd¡ddd„ ƒZee dd¡ejddde ¡ dd d!„ ƒZd"S )#ÚTestRegularizerr   r   g      ð¿ç      ð?)Z	min_valueZ	max_value)r   Úelementsr   c           
      C   s¤   t  d¡}t ||¡ |  ¡ \}}t d¡}||||tjd}||||d tj	d t 
|¡ t 
|¡ dd„ }tt ||g¡||ƒƒD ]\}}	tj||	dd q†d S )	Nr   r7   ©Úby©Úgradr:   c                 S   s<   t  t  t  t  | dd ¡¡ ¡d ¡ t j¡t  | dd ¡fS )Ng•Ö&è.>g      à?)r"   ÚarrayÚsumÚlogÚclipÚastyper#   r   r   r   r   ÚrefJ   s
    &ÿüz-TestRegularizer.test_log_barrier.<locals>.refgü©ñÒMbP?)Zrtol)r   ÚBlobReferencer   ÚFeedBlobÚget_training_netsr   Z
LogBarrierr
   ÚON_LOSSÚAFTER_OPTIMIZERÚ
RunNetOnceÚzipZ
FetchBlobsÚnptÚassert_allclose)
r-   r   ÚparamÚtrain_init_netr/   ÚregÚoutputrB   ÚxÚyr   r   r   Útest_log_barrier9   s"    

û

 z TestRegularizer.test_log_barriergíµ ÷Æ°>g-Cëâ6?)r   Ú	left_openÚ
right_openÚepsÚubÚlbc	                    sÒ   ˆˆr
ˆ nd ˆˆrˆ nd k r$d S t  d¡}	t |	|¡ |  ¡ \}
}tjˆˆˆˆˆ d}|||
|	tjd}|||
|	d tj	d t 
|
¡ t 
|¡ ‡ ‡‡‡‡fdd„}|d ks´t‚tjtj|	 ||ƒdd	 d S )
Nç        r   )rW   rV   rS   rT   Úepsilonr9   r;   c                    s&   t  | ˆˆrˆ nd ˆˆrˆ nd ¡S )NrX   )r"   r@   r   ©rU   rW   rS   rT   rV   r   r   rB   r   s
      ÿz3TestRegularizer.test_bounded_grad_proj.<locals>.refgH¯¼šò×z>)Zatol)r   rC   r   rD   rE   r   ZBoundedGradientProjectionr
   rF   rG   rH   r%   rJ   rK   Úblobs)r-   r   rS   rT   rU   rV   rW   ÚgcÚdcrL   rM   r/   rN   rO   rB   r   rZ   r   Útest_bounded_grad_projU   s2    
 
    ÿû

z&TestRegularizer.test_bounded_grad_projé   é
   é   é   r   )Ú
output_dimÚ	input_numÚ
reg_weightc              
   C   sì   dd„ }t j ||¡ t j¡}t jjd|d d}dg}t  |t  t jjt	d|d ƒ|dd¡¡}t  ||g¡}t  
|¡}t d¡}	t |	|¡ |  ¡ \}
}t |d	 | ¡ ¡}|||
|	tjd
}t |
¡ t |¡ ||||d	 |ƒ dS )zý
        1. create a weight blob
        2. create random group splits
        3. run group_l1_nrom with the weight blob
        4. run equivalent np operations to calculate group l1 norm
        5. compare if the results from 3 and 4 are equal
        c                 S   sp   t  | |dd… ¡}t  dd„ |D ƒ¡}t  |t  dd„ |D ƒ¡¡}t  t  |¡|¡}tj|tj	| dd d S )Nr_   éÿÿÿÿc                 S   s   g | ]}t  t  |¡¡‘qS r   )r"   r>   Úsquare©r   Úgr   r   r   r    ‰   s     zQTestRegularizer.test_group_l1_norm.<locals>.compare_reference.<locals>.<listcomp>c                 S   s   g | ]}t  |jd  ¡‘qS )r_   )r"   ÚsqrtÚshaperh   r   r   r   r    ‹   s     r   ©Údecimal)
r"   Zhsplitrj   Úmultiplyr=   r>   rJ   Úassert_almost_equalr   r[   )ÚweightÚgroup_boundariesZ
reg_lambdarO   Zgroup_splitsZl2_regZl2_normalizedÚresultr   r   r   Úcompare_reference‡   s    ÿz=TestRegularizer.test_group_l1_norm.<locals>.compare_referencer_   )ÚlowÚhighr   F)ÚreplaceÚweight_blobçš™™™™™¹?r9   N)r"   ÚrandomÚrandrA   r#   ÚrandintÚappendÚsortÚchoicer*   Zdiffr   rC   r   rD   rE   r   ZGroupL1NormÚtolistr
   rF   rH   )r-   rc   rd   re   rs   rp   Zfeature_numrq   Z
split_inforw   rM   r/   rN   rO   r   r   r   Útest_group_l1_normz   s2    ÿþ

   ÿ

z"TestRegularizer.test_group_l1_normé	   )Ú	param_dimÚkre   c                 C   s°   t j |¡ t j¡}t d¡}t ||¡ |  	¡ \}}t
 |d |¡}||||tjd}	t |¡ t |¡ t  t  t  |¡¡d || … ¡| d }
tj|
tj|	 dd d S ©Nrw   rx   r9   r   rl   )r"   ry   rz   rA   r#   r   rC   r   rD   rE   r   ZL1NormTrimmedr
   rF   rH   r>   r}   ÚabsoluterJ   ro   r[   )r-   r‚   rƒ   re   rp   rw   rM   r/   rN   rO   rr   r   r   r   Útest_l1_norm_trimmed©   s    
   ÿ

*z$TestRegularizer.test_l1_norm_trimmed)r‚   rƒ   Úl1Úl2c                 C   sÖ   t j |¡ t j¡}t d¡}t ||¡ |  	¡ \}}t
 |d |d |¡}	|	|||tjd}
t |¡ t |¡ t  t  t  |¡¡d || … ¡}t  t  |¡¡}|| d || d  }tj|tj|
 dd d S r„   )r"   ry   rz   rA   r#   r   rC   r   rD   rE   r   ZElasticNetL1NormTrimmedr
   rF   rH   r>   r}   r…   rg   rJ   ro   r[   )r-   r‚   rƒ   r‡   rˆ   rp   rw   rM   r/   rN   rO   Zl1_normZl2_normrr   r   r   r   Útest_elastic_l1_norm_trimmed¾   s"    
   ÿ

"z,TestRegularizer.test_elastic_l1_norm_trimmedg      @)Úrow_dimÚnormÚdata_strategyc                 C   sb  t j |d¡ t j¡}t j |d¡ t j¡}| tjt jddt	 
t  |jd ¡¡d¡}t  |¡}| ¡ }d}t  t j||f d ddd¡}	t  |	d|¡}
||f  |
||	  9  < t d	¡}t ||¡ t d
¡}t ||¡ t d¡}t ||¡ tj||d}|  ¡ \}}tj|dd}|||||tjd t |¡ t |¡ tj|t d	¡dd d S )Nr   r_   r   )ÚdtypeZmin_dimZmax_dimr8   gê-™—q=r   T)ZaxisZkeepdimsrw   Ú	grad_blobÚindices)r   ÚvaluesZfp16)r   r9   rl   ) r"   ry   rz   rA   Zfloat16Zdrawr4   ZtensorZint64ÚstZsampled_fromZarangerk   ÚuniqueÚcopyrj   r>   r@   r   rC   r   rD   ZGradientSlicerE   r   ZMaxNormr
   rG   rH   rJ   ro   Z	FetchBlob)r-   rŠ   r‹   rŒ   rp   r<   r   rr   rU   ZnormsZdesiredrw   rŽ   Zindices_blobZgrad_blob_slicerM   r/   rN   r   r   r   Útest_fp16_max_normÖ   sF    üÿ
 


    ÿ

z"TestRegularizer.test_fp16_max_normN)r1   r2   r3   r   r4   r5   ZfloatsrR   r‘   ZbooleansZgcs_cpu_onlyr^   Zintegersr€   r†   r‰   Údatar”   r   r   r   r   r6   8   sL    
úù	



ý
*


ý




ü

ýr6   ) Z"caffe2.python.hypothesis_test_utilÚpythonZhypothesis_test_utilr4   Zhypothesis.strategiesZ
strategiesr‘   Znumpyr"   Znumpy.testingZtestingrJ   Zcaffe2.pythonr   r   r   r   r   Zcaffe2.python.layer_test_utilr   Zcaffe2.python.optimizerr   Zcaffe2.python.regularizerr	   r
   Z!caffe2.python.regularizer_contextr   r   Z
hypothesisr   r   r6   r   r   r   r   Ú<module>   s   )