U
    ‰d0F  ã                   @   sÞ   d dl mZ d dlmZ d dlmZ d dlmZ ddlmZ ddlmZ ddlm	Z	 d d	l
Z
d d	lZd d	lZd d	lZe ¡  e d
¡Ze ej¡ dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZG dd„ deƒZd	S )é    )Úabstractmethod)Ú	workspace)Útimeout_guard)Údata_parallel_modelé   )Ú
checkpoint)ÚModuleRegister©Ú
module_mapNÚAnyExpc                 C   sÜ   t  dddg¡ | d d dks8| d d dks8tdƒ‚| d d dk}|rX| d d n
| d d }|rt| d d	 n
| d d
 }|rˆdnd| d d< |o¢| d d | d d< || d d< || d d< i | d< i | d d< | S )NZcaffe2z--caffe2_log_level=2z--caffe2_gpu_memory_tracking=0ÚdistributedZnum_gpusr   Znum_cpuszCNeed to specify num_gpus or num_cpus to decide which device to use.Zfirst_cpu_idZfirst_gpu_idÚcpuZgpuÚdeviceÚmodel_paramÚcombine_spatial_bnÚnum_xpusÚfirst_xpu_idÚtemp_varÚmetrics_output)r   Z
GlobalInitÚAssertionError)ÚoptsZtrainWithCPUr   Ú	first_xpu© r   úD/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/contrib/playground/AnyExp.pyÚinitOpts   s2    ÿÿþÿ
ÿÿ
ÿÿ
r   c                   C   s   t tƒ d S ©N)ÚregisterModuleMapr
   r   r   r   r   ÚinitDefaultModuleMap6   s    r   c                 C   s   t  | ¡ d S r   )r   r   r	   r   r   r   r   :   s    r   c                 C   s   t  | d d ¡}| | ¡S )NÚinputZinput_name_py)r   Z	getModuleÚget_input_dataset)r   ZmyAquireDataModuler   r   r   ÚaquireDatasets>   s    r    c                 C   s   t  t| ¡S r   )r   ZconstructTrainerClassÚAnyExpTrainer)r   r   r   r   ÚcreateTrainerClassC   s    r"   c                 C   s   t  | |¡S r   )r   ÚoverrideAdditionalMethods)ZmyTrainerClassr   r   r   r   r#   G   s    r#   c                  O   s   t j| |ŽS r   )r   Úinitialize_params_from_file)ÚargsÚkwargsr   r   r   r$   K   s    r$   c                   @   sp  e Zd Zdd„ Zedd„ ƒZedd„ ƒZedd„ ƒZd	d
„ Zdd„ Z	dd„ Z
dd„ Zedd„ ƒZdd„ Zedd„ ƒZdd„ Zdd„ Zedd„ ƒZedd„ ƒZdd „ Zd!d"„ Zed#d$„ ƒZed%d&„ ƒZed'd(„ ƒZed)d*„ ƒZd+d,„ Zed-d.„ ƒZed/d0„ ƒZed1d2„ ƒZed3d4„ ƒZed5d6„ ƒZed7d8„ ƒZed9d:„ ƒZ ed;d<„ ƒZ!d=d>„ Z"d?d@„ Z#dAdB„ Z$dCdD„ Z%dEdF„ Z&dGS )Hr!   c                 C   s€  dd l }| ¡  | d¡}| |j¡ || _|| _d | _d | _d | _	d | _
i | _g | _g | _g | _g | _|d d | _|d d }|d d }t||| ƒ| _| jd d	 | jd d  | jd d
  | _| jd d | j | _t|d d ƒdkr0|d d d | _	t|d d ƒdkr0|d d d | _
|d d | _|d d | _|d d | _|d d | _| d t|ƒ¡¡ d S )Nr   r   r   r   r   r   r   Ú
epoch_iterZbatch_per_deviceZ
num_shardsZnum_train_sample_per_epochr   Zdatasetsé   r   Úshard_idÚstart_epochÚepochÚnum_epochs_per_flow_schedulezopts: {})ÚloggingÚbasicConfigÚ	getLoggerÚsetLevelÚDEBUGÚlogr   Útrain_datasetÚtest_datasetZtrain_dfZtest_dfÚmetricsÚplotsIngredientsÚrecord_epochsÚsamples_per_secÚsecs_per_trainr   ÚrangeÚxpusÚtotal_batch_sizeÚepoch_iterationsÚlenr)   r*   r+   Zepochs_to_runÚinfoÚformatÚstr)Úselfr   r-   r2   r   r   r   r   r   Ú__init__Q   sL    
ÿþÿÿÿzAnyExpTrainer.__init__c                 C   s   d S r   r   ©rB   r   r   r   r   r   ‚   s    zAnyExpTrainer.get_input_datasetc                 C   s   d S r   r   ©rB   r   r   r   Úget_model_input_fun†   s    z!AnyExpTrainer.get_model_input_func                 C   s   d S r   r   rE   r   r   r   Ú
init_modelŠ   s    zAnyExpTrainer.init_modelc                 C   sX   | j d d }|D ]@}|  |d ¡}|f d| j i|d —Ž}|  |d ||d ¡ qd S )NÚoutputr5   Zmeter_pyr   Zmeter_kargsÚnameÚis_train)r   ÚgetMeterClassÚ
add_metric)rB   r5   ÚmetricZ
meterClassZmeterInstancer   r   r   Úinit_metricsŽ   s
    zAnyExpTrainer.init_metricsc                 C   s   t  ||¡S r   )r   ZgetClassFromModule)rB   Z	meterNamer   r   r   rK   —   s    zAnyExpTrainer.getMeterClassc                 C   s6   | j }i ||< ||| d< ||| d< g || d< d S )NÚ
calculatorrJ   rH   )r5   )rB   rI   rO   rJ   r5   r   r   r   rL   š   s
    zAnyExpTrainer.add_metricc                 C   s¤   | j }|sJ| j|d< | j|d< | j|d< | j ¡ D ]\}}|d ||< q2nV|d  | j¡ |d  | j¡ |d  | j¡ | j ¡ D ]\}}||  |d ¡ q„d S )NZepochsr8   r9   rH   )r   r7   r8   r9   r5   ÚitemsÚextend)rB   r   rM   Úvaluer   r   r   ÚextendMetricsOutput¡   s    


z!AnyExpTrainer.extendMetricsOutputc                 C   s   d S r   r   rE   r   r   r   Ú
init_plots°   s    zAnyExpTrainer.init_plotsc                 C   s8   | j }i }||d< ||d< ||d< ||d< | |¡ d S )NÚxÚx_titleÚysÚy_title)r6   Úappend)rB   rU   rV   rW   rX   r6   ZaPlotIngredientsr   r   r   Úadd_plot´   s    zAnyExpTrainer.add_plotc                 C   s   d S r   r   rE   r   r   r   Ú	init_logs½   s    zAnyExpTrainer.init_logsc                 C   s0   t | jd d | j| jd d  ƒ}t| j|ƒS )Nr'   Ú
num_epochsr,   )Úminr   r+   r:   )rB   Ziter_end_pointr   r   r   Úlist_of_epochsÁ   s    ÿÿzAnyExpTrainer.list_of_epochsc                 C   s   t d| jƒS )Nr   )r:   r=   rE   r   r   r   Úlist_of_epoch_itersÇ   s    z!AnyExpTrainer.list_of_epoch_itersc                 C   s   d S r   r   ©rB   r+   r   r   r   Úfun_per_epoch_b4RunNetÊ   s    z$AnyExpTrainer.fun_per_epoch_b4RunNetc                 C   s   d S r   r   r`   r   r   r   Úfun_per_epoch_aftRunNetÎ   s    z%AnyExpTrainer.fun_per_epoch_aftRunNetc              	   C   s2   t  d| j|  d|d ¡|d | jtdƒ¡| _d S )NTr   z-inf)r   Zsave_model_paramsÚtrain_modelÚgen_checkpoint_pathr   ÚfloatÚ
model_pathr`   r   r   r   r   Ò   s        þzAnyExpTrainer.checkpointc                 C   s&   |rd  |¡}nd}| jd d | S )Nzmodel_checkpoint_epoch{}.pklzmodel_final.pklrH   Zcheckpoint_folder)r@   r   )rB   Zis_checkpointr+   Úfilenamer   r   r   rd   ×   s    z!AnyExpTrainer.gen_checkpoint_pathc                 C   s   d S r   r   ©rB   r+   r'   r   r   r   Úfun_per_iter_b4RunNetâ   s    z#AnyExpTrainer.fun_per_iter_b4RunNetc                 C   s   d S r   r   rh   r   r   r   Úfun_per_iter_aftRunNetB4Testæ   s    z*AnyExpTrainer.fun_per_iter_aftRunNetB4Testc                 C   s   d S r   r   rh   r   r   r   Úfun_per_iter_aftRunNetAftTestê   s    z+AnyExpTrainer.fun_per_iter_aftRunNetAftTestc                 C   s   d S r   r   rD   r   r   r   Úfun_conclude_operatorî   s    z#AnyExpTrainer.fun_conclude_operatorc                 C   s   |   ¡  | j| _d S r   )rS   rf   Zmodel_outputrE   r   r   r   ÚcreateMetricsPlotsModelsOutputsò   s    z-AnyExpTrainer.createMetricsPlotsModelsOutputsc                 C   s   d S r   r   rE   r   r   r   ÚassembleAllOutputsö   s    z AnyExpTrainer.assembleAllOutputsc                 C   s   d S r   r   ©rB   ÚmodelÚdatasetrJ   r   r   r   Úgen_input_builder_funú   s    z#AnyExpTrainer.gen_input_builder_func                 C   s   d S r   r   ro   r   r   r   Úgen_forward_pass_builder_funþ   s    z*AnyExpTrainer.gen_forward_pass_builder_func                 C   s   d S r   r   ro   r   r   r   Úgen_param_update_builder_fun  s    z*AnyExpTrainer.gen_param_update_builder_func                 C   s   d S r   r   ro   r   r   r   Úgen_optimizer_fun  s    zAnyExpTrainer.gen_optimizer_func                 C   s   d S r   r   ro   r   r   r   Úgen_rendezvous_ctx
  s    z AnyExpTrainer.gen_rendezvous_ctxc                 C   s   d S r   r   rE   r   r   r   Úrun_training_net  s    zAnyExpTrainer.run_training_netc              	   C   s@   | j d krd S d}t |¡ t | j j ¡ j¡ W 5 Q R X d S )Ng     @Ÿ@)Ú
test_modelr   ZCompleteInTimeOrDier   ZRunNetÚnetÚProtorI   )rB   Útimeoutr   r   r   Úrun_testing_net  s
    
zAnyExpTrainer.run_testing_netc                 C   s   |   ¡  |  ¡  |  ¡  d S r   )rN   rT   r[   rE   r   r   r   Úplanning_output  s    zAnyExpTrainer.planning_outputc                 C   s(   |   | j| jd¡ |   | j| jd¡ d S )NTF)Úprep_a_data_parallel_modelrc   r3   rx   r4   rE   r   r   r   Úprep_data_parallel_models   s     ÿ ÿz'AnyExpTrainer.prep_data_parallel_modelsc                 C   s0  |d krd S t  d¡ | jd k	r.|  |||¡nd }t  d¡ | jd k	rT|  |||¡nd }t  d¡ | jd d }tj||  |||¡|  |||¡||| j	|  
|||¡d| jd d | jd d || jd	 d
 dk| jd d d t  d¡ t |j¡ t  d¡ t  d |j ¡ ¡¡ t |j¡ t  d¡ d S )Nzin prep_a_data_parallel_modelz0in prep_a_data_parallel_model param_update done z-in prep_a_data_parallel_model optimizer done r   Úmax_concurrent_distributed_opsFZmemongerZ	cuda_ncclr   r   r   r   )Zinput_builder_funZforward_pass_builder_funZparam_update_builder_funZoptimizer_builder_funZdevicesZ
rendezvousZbroadcast_computed_paramsZoptimize_gradient_memoryZuse_ncclr€   Z
cpu_devicer   z/in prep_a_data_parallel_model Parallelize done z.in prep_a_data_parallel_model RunNetOnce done zmodel.net.Proto() {}z-in prep_a_data_parallel_model CreateNet done )r2   r?   rt   ru   r   r   ZParallelizerr   rs   r;   rv   r   Z
RunNetOnceZparam_init_netr@   ry   rz   Z	CreateNet)rB   rp   rq   rJ   Zparam_updateZ	optimizerZmax_opsr   r   r   r~   &  sN    
ÿþ
ÿþ
  ÿï

z(AnyExpTrainer.prep_a_data_parallel_modelc           	      C   sÈ   | j }|d d }|d d }|d d }|d k	rntj |¡r¸t d |¡¡ tj| j	|||ddd	\}}}nJ|d k	r¸tj |¡r¸t d
 |¡¡ tj| j	|||d|d d d	\}}}t
 | j	¡ d S )Nr   Zcheckpoint_modelÚpretrained_modelr   r   zLoad previous checkpoint:{}TF)rp   Zweights_filer   r   Zbroadcast_computed_paramÚreset_epochzLoad pretrained model: {}r   r‚   )r   ÚosÚpathÚexistsr2   r?   r@   r   r$   rc   r   ZFinalizeAfterCheckpoint)	rB   r   Zprevious_checkpointr   r   r*   Zprev_checkpointed_lrZ_best_metricZbest_metricr   r   r   ÚloadCheckpoint^  s<    ÿúÿ
	
úÿ
zAnyExpTrainer.loadCheckpointc              	   C   s¸  t  d t|ƒ¡¡ t  d t| ƒ¡¡ t  d t| ƒ¡¡ t  d | j¡¡ t  d t | j	¡¡¡ t  d t | j
¡¡¡ t  d t | j¡¡¡ | jd k	rºt  d t | j¡¡¡ nt  d	 t | j¡¡¡ t  d
 t | j¡¡¡ t  d t | j¡¡¡ |  ¡  |  ¡  |  ¡  |  ¡  |  ¡  |  ¡ D ]j}t  d |¡¡ |  |¡ |  ¡ D ]*}t ¡ | _|  ||¡ | jd k	rŠ|  ¡  |  ||¡ t ¡ | _||d d  dkrx| j| j }| j |¡ | j }|| }| j! |¡ |t"|ƒ| j#  | _$| j% | j$¡ | j&D ]@}| j&| }|d s,q|d  '¡  |d  |d  (¡ ¡ qt ¡ | _)t*d|d d ƒD ]@}	|  +¡  | j&D ]*}| j&| }|d rœq€|d  '¡  q€qnt ¡ | _,| j,| j) | _-| j& .¡ D ]*}|d räqÒ|d  |d  (¡ ¡ qÒd | j$| j/d d || j#|¡}
|
d ||d d | j-¡7 }
| j& 0¡ D ]"\}}|
d ||d d ¡7 }
qDt  d |
¡¡ |  1||¡ qZ|  2|¡ |  3|¡ q2|  4¡  |  5¡  |  ¡ S )Nz(in buildModelAndTrain, trainer_input: {}zcheck type self: {}zcheck self dir: {}zcheck self source: {}z(check self get_input_dataset methods: {}z+check self gen_input_builder_fun method: {}z2check self gen_forward_pass_builder_fun method: {}z2check self gen_param_update_builder_fun method: {}z'check self gen_optimizer_fun method: {}z(check self assembleAllOutputs method: {}z/check self prep_data_parallel_models method: {}zstart training epoch {}r'   Znum_train_iteration_per_testr   rJ   rO   rH   Znum_test_iterz)epoch:{}/{} iter:{}/{} secs_per_train:{} r\   z-samples_per_sec:{} loop {} tests takes {} secz {}:{} éÿÿÿÿzIter Stats: {})6r2   r?   r@   rA   ÚtypeÚdirÚ__dict__ÚinspectÚ	getsourcer   rr   rs   rt   ru   rn   r   rF   rG   r}   r†   r^   ra   r_   ÚtimeZiter_start_timeri   rc   rw   rj   Ziter_end_timer9   rY   r<   r8   re   r=   Zfract_epochr7   r5   ZAddZComputeZtest_loop_start_timer:   r|   Ztest_loop_end_timeZsec_per_test_loopÚvaluesr   rP   rk   r   rb   rl   rm   )rB   r   r+   r'   r9   Zsample_trainedr8   ÚkeyrM   Z
_test_iterZlogStrrR   r   r   r   ÚbuildModelAndTrain  sÂ    
ÿ
ÿ
ÿ

ÿ
ÿ
ÿ
ÿ



ÿÿÿ








ÿ
   þ 
þ
z AnyExpTrainer.buildModelAndTrainN)'Ú__name__Ú
__module__Ú__qualname__rC   r   r   rF   rG   rN   rK   rL   rS   rT   rZ   r[   r^   r_   ra   rb   r   rd   ri   rj   rk   rl   rm   rn   rr   rs   rt   ru   rv   rw   r|   r}   r   r~   r†   r   r   r   r   r   r!   O   sl   1


	
	














8!r!   )Úabcr   Zcaffe2.pythonr   r   r   Ú r   r   r
   rƒ   r‹   r   r-   r.   r/   r2   r0   r1   r   r   r   r    r"   r#   r$   Úobjectr!   r   r   r   r   Ú<module>   s*   
