U
    d1                     @   s(  d dl mZmZ d dlmZ d dlmZmZmZ d dl	m
Z
mZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ G dd deZG dd deZeedd Zeedd Zeedd Z ee
dd Z!eedd Z"eedd Z#dd Z$G dd deZ%G dd  d ee%Z&d!d" Z'd#d$ Z(d%d& Z)d'd( Z*d)d* Z+dGd-d.Z,d/d0 Z-e&ed1d2 Z.e&ed3d4 Z/e&ed5d6 Z0d7d8 Z1e&ed9d: Z2d;d< Z3e&e
d=d> Z4e&edHd?d@Z5e&edAdB Z6dCdD Z7dEdF Z8d+S )I    )OperatorDefNetDef)Job)NetExecutionStepPlan)Task	TaskGroupWorkspaceType
TaskOutputdefaultdict)contextmanager)copy)viewkeys)chain)binary_type	text_typec                   @   s    e Zd Zedd Zdd ZdS )Visitorc                    s<   t dsi _n jks*td  fdd}|S )Nvisitorsz{} already registered!c                    s   | j  < | S N)r   )funcTypecls =/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/net_printer.py	_register   s    
z#Visitor.register.<locals>._register)hasattrr   AssertionErrorformat)r   r   r   r   r   r   register   s    
zVisitor.registerc                 O   sR   |d krd S t |}|| jjkr4td| jj|f | jj| }|| |f||S )Nz%s: unsupported object type: %s)type	__class__r   	TypeError__name__)selfobjargskwargsr   r   r   r   r   __call__#   s     zVisitor.__call__N)r%   
__module____qualname__classmethodr!   r*   r   r   r   r   r      s   
r   c                   @   sD   e Zd ZdhZdd Zedd Zeddd	Zd
d Z	dd Z
dS )AnalyzerZdistributed_ctx_initc                 C   s   t dd | _g | _d S )Nc                   S   s   t dd S )Nc                   S   s   dS Nr   r   r   r   r   r   <lambda>4       z5Analyzer.__init__.<locals>.<lambda>.<locals>.<lambda>r   r   r   r   r   r0   4   r1   z#Analyzer.__init__.<locals>.<lambda>)r   
workspacesworkspace_ctxr&   r   r   r   __init__3   s    zAnalyzer.__init__c                 C   s
   | j d S N)r3   r4   r   r   r   	workspace7   s    zAnalyzer.workspaceNFc                 c   sV   |d k	r|}n|d k	r&| j t| }n| j}|r8t|}| j| |V  | jd= d S r6   )r2   strr8   r   r3   append)r&   nodewsdo_copyr   r   r   set_workspace;   s    zAnalyzer.set_workspacec                 C   s   | j |  d7  < d S )N   )r8   r&   blobr   r   r   define_blobI   s    zAnalyzer.define_blobc                    s6   t  fddtjD rd S  | jks2td  d S )Nc                 3   s   | ]}  |V  qd S r   )
startswith).0prA   r   r   	<genexpr>M   s     z%Analyzer.need_blob.<locals>.<genexpr>zBlob undefined: %s)anyr.   PREFIXES_TO_IGNOREr8   r   r@   r   rF   r   	need_blobL   s    zAnalyzer.need_blob)NNF)r%   r+   r,   rI   r5   propertyr8   r   r>   rB   rJ   r   r   r   r   r.   0   s   
r.   c                 C   s0   |j D ]}| | q|jD ]}| | qd S r   )inputrJ   outputrB   )analyzeropxr   r   r   
analyze_opR   s    

rQ   c                 C   s   |  jD ]}| | q
d S r   )ProtorO   )rN   netrP   r   r   r   analyze_netZ   s    rT   c           	         s     }| j|jd |jrF| jdd |  |j W 5 Q R X t }   fdd|jD  }|D ]}| j|jd }| | |j	r| 
|j	 W 5 Q R X |jrltt|tt| j }t||@ dkstdd||@  ||O }qlW 5 Q R X |D ]}| | qd S )Nr=   Tc                    s   g | ]}  |qS r   get_netrD   nstepr   r   
<listcomp>h   s     z analyze_step.<locals>.<listcomp>r   z3Error: Blobs created by multiple parallel steps: %s, )rR   r>   create_workspace
report_netrW   setSubstepsnetworkconcurrent_substepsshould_stop_blobrJ   r   r8   lenr   joinrB   )	rN   r[   protoZall_new_blobssubstepssubstepZws_inZ	new_blobsrP   r   rZ   r   analyze_step`   s2    rj   c              	   C   sr   |  }t|j}|| t|  }|dk s@td| | t	j
k}| j|d | | W 5 Q R X d S )Ni   zeDue to a protobuf limitation, serialized tasks must be smaller than 64Mb, but this task has {} bytes.rU   )get_stepr   r;   ZAddStepre   rR   ZSerializeToStringr   Zworkspace_typer
   GLOBALr>   )rN   taskr[   ZplanZ	proto_len
is_privater   r   r   analyze_tasky   s    


ro   c              
   C   s8   |   D ]&}| j|jd | | W 5 Q R X qd S )N)r;   )tasks_by_nodetasksr>   r;   )rN   tgrm   r   r   r   analyze_task_group   s    rs   c                 C   s   | |j  | |j d S r   )
init_groupepoch_group)rN   jobr   r   r   analyze_job   s    
rw   c                 C   s   t  |  dS )z
    Given a Job, visits all the execution steps making sure that:
      - no undefined blobs will be found during execution
      - no blob with same name is defined in concurrent steps
    N)r.   )r'   r   r   r   analyze   s    rx   c                   @   s0   e Zd Zdd Zedd Zdd Zdd Zd	S )
Textc                 C   s   d| _ dg| _g | _d S r/   )_indent_lines_in_contextlinesr4   r   r   r   r5      s    zText.__init__c                 c   sp   |d k	r0|  d|  |  jd7  _| jd d V  |d k	rl| jd dkrV|  d |  jd8  _| jd= d S )Nzwith %s:   r   r7   pass)addrz   r{   r:   r&   textr   r   r   context   s    
zText.contextc                 C   s,   | j d  d7  < | jd| j |  d S )Nr7   r?    )r{   r|   r:   rz   r   r   r   r   r      s    zText.addc                 C   s   d | jS )N
)rf   r|   r4   r   r   r   __str__   s    zText.__str__N)r%   r+   r,   r5   r   r   r   r   r   r   r   r   ry      s
   
ry   c                       s   e Zd Zd fdd	Z  ZS )PrinterFTc                    s2   t t|   t t|   || _|| _d | _d S r   )superr   r5   ry   factor_prefixes	c2_syntaxc2_net_name)r&   r   r   r#   r   r   r5      s
    zPrinter.__init__)FT)r%   r+   r,   r5   __classcell__r   r   r   r   r      s   r   c                 C   sj   t | tr| }n"t | tr*| jddd}nt| }t|dk rFd| S d|d d  dt|d   S d S )Nasciiignore)errors@   '%s'z...<+len=%d>)
isinstancer   r   decoder9   re   )sZ	sanitizedr   r   r   _sanitize_str   s    

r   c                 C   s   |  drt| jS |  dr(t| jS |  dr<t| jS | jrPtt| jS | jrdtt| jS | j	r~tdd | j	D S dS )Nfir   c                 S   s   g | ]}t |qS r   )r   )rD   r   r   r   r   r\      s     z_arg_val.<locals>.<listcomp>z[])
HasFieldr9   r   r   r   r   ZfloatslistZintsstrings)argr   r   r   _arg_val   s    





r   c                 C   sJ   | sdS t | }t| }t|D ]$\}}||| kr |d|   S q |S )z:Given a list of strings, returns the longest common prefix N)minmax	enumerate)ms1s2r   cr   r   r   commonprefix   s    r   c                 C   s.   t | tr"dddd | D  S t| S d S )Nz[%s]r]   c                 s   s   | ]}d t | V  qdS )r   Nr9   rD   vr   r   r   rG      s     zformat_value.<locals>.<genexpr>)r   r   rf   r9   )valr   r   r   format_value   s    
r   c                    sV   dd | D } t | dkr&|r&t| nd d fdd| D } rRd |f S |S )	Nc                 S   s   g | ]}t |qS r   )r   r   r   r   r   r\      s     z!factor_prefix.<locals>.<listcomp>r?   r   r]   c                 3   s   | ]}|t  d  V  qd S r   )re   r   prefixr   r   rG      s     z factor_prefix.<locals>.<genexpr>z%s[%s])re   r   rf   )valsZdo_itZjoinedr   r   r   factor_prefix   s    r   NFc                 C   sx   |s
d}nHdd |D }dd |D }d dd tt||gdd |D D }d	| |f }|sf|S d
t|||f S )Nr   c                 S   s   g | ]}t |ts|qS r   r   tuplerD   ar   r   r   r\      s     
 zcall.<locals>.<listcomp>c                 S   s   g | ]}t |tr|qS r   r   r   r   r   r   r\      s     
 r]   c                 s   s   | ]}|r|V  qd S r   r   rD   rP   r   r   r   rG      s   zcall.<locals>.<genexpr>c                 s   s   | ]}d | V  qdS )z%s=%sNr   )rD   kvr   r   r   rG     s     z%s(%s)z%s = %s)rf   r   r   )rO   inputsoutputsr   Zinputs_vZ	inputs_kvcallr   r   r   r      s    


 r   c                 C   s4   | r| j s| js| jsd S td| j | jd| j gS )NZDeviceOptionr   )Zdevice_typeZ	device_idZ	node_namer   )Zdev_optr   r   r   format_device_option  s    r   c              
   C   s   dd |j D }t|j}|r,|d|f | jrd| t| jd |j t|j	t|j
g|  n&| t|jt|j	| |j
| jd |j D ]4}|dr| d|j  | |j W 5 Q R X qd S )Nc                 S   s   g | ]}|j t|fqS r   )namer   r   r   r   r   r\     s     zprint_op.<locals>.<listcomp>device_option.)r   rY   zarg: %s)r   r   r   r:   r   r   r   r"   r   rL   rM   r   r   r   r   rY   )r   rO   r(   Zdev_opt_txtr   r   r   r   print_op  s&    


r   c                 C   sb   | j r.| tdd|j g|jg |j| _n| d|j  |jD ]}| | qD| j r^d | _d S )Nzcore.Netr   z	# net: %s)r   r   r   r   r   rO   )r   Znet_defrO   r   r   r   print_net_def-  s    


r   c                 C   s   | |   d S r   )rR   )r   rS   r   r   r   	print_net:  s    r   c                 C   s   |   }|jrtddfS |jr<|jdkr<td|jgdfS |jdkrhtdd|jfgt|  dkfS |jo|t|  dk}|rtddfS |jrtddfS dS )	NZloopFr?   parallelZnum_instancesTZrun_once)NF)	rR   rd   r   Znum_iterZnum_concurrent_instancesre   ra   rc   r_   )r[   rg   Z
concurrentr   r   r   _get_step_context?  s"    

r   c           
         s4     }t \}}| |
 |jrX| td|jg |  |j W 5 Q R X    fdd|jD  }|D ]}t	|t
r|  nd }|d k	r|jrtdt|d|jfg}n0|r|d k	r|jrdnd}	t|	t|g}nd }| |* | | |jr| td|jg W 5 Q R X qxW 5 Q R X d S )	Nr_   c                    s   g | ]}  |qS r   rV   rX   rZ   r   r   r\   Z  s     zprint_step.<locals>.<listcomp>ZreporterZinterval_msr8   r[   zyield stop_if)rR   r   r   r_   r   Zreport_intervalrW   ra   rb   r   r   Zrun_every_msr9   r^   rd   r   )
r   r[   rg   Zstep_ctxZ
do_substeprh   ri   Z	sub_protoZsubstep_ctxtitler   rZ   r   
print_stepR  s8    r   c                 C   s,   t | tstdddd | jD  d S )NzOutput[r]   c                 s   s   | ]}t |V  qd S r   r   r   r   r   r   rG   r  s     z%_print_task_output.<locals>.<genexpr>])r   r   r   rf   names)rP   r   r   r   _print_task_outputp  s    r   c              	   C   s^   d dd | D }d|jfd|jfd|fg}| td| | |  W 5 Q R X d S )Nr]   c                 s   s   | ]}t |V  qd S r   )r   )rD   or   r   r   rG   w  s     zprint_task.<locals>.<genexpr>r;   r   r   r   )rf   r   r;   r   r   r   rk   )r   rm   Zoutsr   r   r   r   
print_tasku  s    r   c              	   C   s<   |  |ptd  |  D ]}| | q W 5 Q R X d S )Nr	   )r   r   rp   rq   )r   rr   headerrm   r   r   r   print_task_group}  s    r   c              	   C   sd   | |j d | |jd | d  |jD ]}| t| q*W 5 Q R X | |jd | |jd d S )NzJob.current().init_groupzJob.current().epoch_groupzJob.current().stop_conditionszJob.current().download_groupzJob.current().exit_group)rt   ru   r   Zstop_conditionsr   r   Zdownload_groupZ
exit_group)r   rv   outr   r   r   	print_job  s    
r   c                 K   s   t f |}||  t|S )z
    Given a Net, ExecutionStep, Task, TaskGroup or Job, produces a string
    with detailed description of the execution steps.
    )r   r9   )r'   r)   printerr   r   r   	to_string  s    
r   c                 C   sl   t | tsttt| }t | ts(t|  jD ]4}t }t|| |t| | j	|g q2|S )z
    Given a Net, produce another net that logs info about the operator call
    before each operator execution. Use for debugging purposes.
    )
r   r   r   r9   rR   rO   ry   r   ZLogInfoextend)rS   	debug_netrO   r   r   r   r   r     s    
r   )NNF)N)9Zcaffe2.proto.caffe2_pb2r   r   Zcaffe2.python.checkpointr   Zcaffe2.python.corer   r   r   Zcaffe2.python.taskr   r	   r
   r   collectionsr   
contextlibr   r   Zfuture.utilsr   	itertoolsr   sixr   r   objectr   r.   r!   rQ   rT   rj   ro   rs   rw   rx   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>   sf   "





		
	







