U
    dh                 	   @   s  d dl mZmZmZ d dlmZ d dlmZmZm	Z	 d dl
mZ d dlmZmZmZ d dlmZ d dlmZmZmZ d dlmZ d d	lmZmZmZmZ d d
lm  mZ  d d
l!Z!d d
l"Z"d d
l#Z$d d
l%Z%d d
l&Z&d d
l'Z'e%j(dkrde ) kre*d ej+Z+ej,Z,G dd dZ-dd Z.e.  dd Z/e/ a0dddZ1g Z2dd Z3dd Z4dd Z5dd Z6dd  Z7dd!d"Z8dd#d$Z9d%d& Z:d'd( Z;d)d* Z<d+d, Z=ed-d.d/gZ>G d0d1 d1e?Z@d2d3 ZAd4d5 ZBdd6d7ZCdd9d:ZDdd<d=ZEdd>d?ZFd@dA ZGedBdCdDdEgZHedFdGdHdIdJgZIedKdLdMdNdOdIdJgZJG dPdQ dQe?ZKG dRdS dSe?ZLeLMdTe eLMdUe eLMdVe ddWdXZNdYdZ ZOd[d\ ZPd]d^ ZQd_d` ZRdadb ZSeReReSeSeSdcZTddde ZUddfdgZVdhdi ZWddjdkZXG dldm dme?ZYdndo ZZdpdq Z[drds Z\dtdu Z]G dvdw dwZ^ddxdyZ_ddzd{Z`dd|d}Zad~d Zbdd Zcdd ZdG dd de?Zedd ZfG dd de?ZgdddZhdddZidd Zjdd Zke jlZme jnZoe jpZqe jrZsd
S )    )
namedtupleOrderedDictdefaultdict)
basestring)	viewitemsviewkeys
viewvalues)chain)binary_typestring_types	text_type)
caffe2_pb2)scopeutils	workspace)TriggerLazyImport)gen_do_gradientgen_if_gradientgen_while_gradientdisambiguate_grad_if_op_outputNdarwinZleveldbzIf you are using homebrew leveldb on a Mac OS, you might see an error warning you that malloc_zone_unregister() failed. This is not a caffe2 issue but is due to the homebrew leveldb having an incompatible memory allocator. It does not affect usage.c                   @   sH   e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdS )DataTyper                           	   
               N)__name__
__module____qualname__	UNDEFINEDFLOATZINT32ZBYTESTRINGBOOLZUINT8ZINT8ZUINT16ZINT16ZINT64ZFLOAT16ZDOUBLEZZERO_COLLISION_HASHZREBATCHING_BUFFER r-   r-   6/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/core.pyr   *   s   r   c                  C   sH   t jj D ]6\} }tt| d }||krtd|  d| d| qd S )Nz	DataType z3 does not match the value defined in caffe2.proto: z vs )r   ZTensorProtor   itemsgetattrAssertionError)namevalueZpy_valuer-   r-   r.   _CheckDataType<   s    r4   c                   C   s   t t S N)setr   ZRegisteredOperatorsr-   r-   r-   r.   _GetRegisteredOperatorsK   s    r7   Tc                 C   s   | r
t   t ad S r5   )r   r7   _REGISTERED_OPERATORS)Ztrigger_lazyr-   r-   r.   RefreshRegisteredOperatorsR   s    r9   c                 C   s&   t   t| dd   t|  d S Nr   )r   _GLOBAL_INIT_ARGSextendCZglobal_init)argsr-   r-   r.   
GlobalInit\   s    r?   c                   C   s   t d d  S r5   )r;   r-   r-   r-   r.   GetGlobalInitArgsb   s    r@   c                 C   s   t | ddS )NDEFAULTengine)IsOperatorWithEngine)op_typer-   r-   r.   
IsOperatorf   s    rF   c                 C   s   t   t| |tkS r5   )r   r=   Zop_registry_keyr8   )rE   rC   r-   r-   r.   rD   j   s    rD   c                 C   s   | t jt jhkS r5   )r   ZCUDAZHIP)device_typer-   r-   r.   IsGPUDeviceTypeo   s    rH   c                 C   sd   t  }| |_||_|d k	r"||_|d k	r0||_|d k	rL| t jksFt||_|d k	r`|j	
| |S r5   )r   DeviceOptionrG   	device_id	node_namerandom_seedCPUr1   numa_node_id
extra_infor<   )rG   rJ   rL   rK   rN   rO   optionr-   r-   r.   rI   s   s    rI   c                 C   s`   | r|s| |kS |s$| j |j kr$dS |s8| j|jkr8dS | jrD|jsT| j oR|j S | j|jkS NF)rK   rL   rG   rJ   )opt1opt2Zignore_node_nameZignore_random_seedr-   r-   r.   device_option_equal   s    rT   c                 C   sF   i }|   jD ]2}|j}|dkr,ttj}|jD ]}|||< q2q|S )z|
    Compute mapping from parameters to devices by looking at the
    device option of the op that creates the blob has
    N)Protoopdevice_optionr   rI   rM   output)netmappingrV   Z	op_devicebr-   r-   r.   InferBlobDevices   s    
r\   c                    sP   t \  fddttjD }fddttjD }||fS )Nc                    s   i | ]}j |  | qS r-   )input.0i)input_dev_listrV   r-   r.   
<dictcomp>   s    z,InferOpBlobDevicesAsDict.<locals>.<dictcomp>c                    s   i | ]} j | | qS r-   )rX   r^   )rV   output_dev_listr-   r.   rb      s    )InferOpBlobDevicesrangelenr]   rX   )rV   Z
input_dictZoutput_dictr-   )ra   rV   rc   r.   InferOpBlobDevicesAsDict   s    rg   c                 C   sr   t |  }g }g }|d D ] }t }|| || q|d D ] }t }|| || qH||fS )Nr   r   )r=   Zinfer_op_input_output_deviceSerializeToStringr   rI   ParseFromStringappend)rV   Zdevice_infoZ
input_infoZoutput_infoZdev_strrW   r-   r-   r.   rd      s    

rd   c                 C   s<   | j r| j nt }|gt| j }|gt| j }||fS r5   )rW   r   rI   rf   r]   rX   )rV   Zop_dev	input_dev
output_devr-   r-   r.   InferOpDeviceAsBlobDevices   s    rm   GradientSliceindicesvaluesc                   @   s   e Zd ZdZdddZdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zd ddZdd Zdd ZdS )!BlobReferencezA wrapper around a blob in a net.

    BlobReference gives us a way to refer to the network that the blob is
    generated from. Note that blobs are, essentially, just strings in the
    current workspace.
    Nc                 C   sD   t |tr|| _n"t |tr*|d| _n
t|| _|| _i | _dS )zInitializes a blob reference.

        Note that this does not prepends the namescope. If needed, use
        ScopedBlobReference() to prepend the existing namespace.
        utf-8N)
isinstancer   _namer
   decodestr	_from_netmeta)selfr2   rY   r-   r-   r.   __init__   s    


zBlobReference.__init__c                 C   s
   t | jS r5   )hashrt   ry   r-   r-   r.   __hash__   s    zBlobReference.__hash__c                 C   sL   t |tr| j|kS t |tr.| j|dkS t |trD| j|jkS dS d S )Nrr   F)rs   r   rt   r
   ru   rq   ry   otherr-   r-   r.   __eq__   s    



zBlobReference.__eq__c                 C   s
   | |k S r5   r-   r~   r-   r-   r.   __ne__   s    zBlobReference.__ne__c                 C   s   | j S r5   )rt   r|   r-   r-   r.   __str__   s    zBlobReference.__str__c                 C   s   d | jS )NzBlobReference("{}"))formatrt   r|   r-   r-   r.   __repr__   s    zBlobReference.__repr__c                 C   s$   t |tstdt| j| | jS )Nz)Cannot add BlobReference to a non-string.rs   r   RuntimeErrorrq   rt   rw   r~   r-   r-   r.   __add__   s    
zBlobReference.__add__c                 C   s$   t |tstdt|| j | jS )Nz)Cannot add a non-string to BlobReference.r   r~   r-   r-   r.   __radd__  s    
zBlobReference.__radd__c                 C   s   | j S r5   )rw   r|   r-   r-   r.   Net  s    zBlobReference.Netc                 C   s   | j d | j tjd  S r:   rt   rfindr   Z_NAMESCOPE_SEPARATORr|   r-   r-   r.   GetNameScope	  s    zBlobReference.GetNameScopec                 C   s   | j | j tjd d  S r:   r   r|   r-   r-   r.   GetUnscopedName  s    zBlobReference.GetUnscopedNamec                 O   sN   |dkrg n|}t |ts$t |tr*|g}|d|  | j||f||S )zmInternal function that routes the operator generation to the
        network's __getattr__ function.
        Nr   )rs   rq   r   insertrw   __getattr__)ry   rE   inputsr>   kwargsr-   r-   r.   _CreateAndAddToNet  s
    z BlobReference._CreateAndAddToNetc                    sj     drtd jdkr*tdt s\td  d d dtj  d	  fd
dS )zA wrapper allowing one to initiate operators from a blob reference.

        Example: for a blob reference b that comes from network n, doing
            b.Relu(...)
        is equivalent to doing
            net.Relu([b], ...)
        __Attribute {} not found.NzYou cannot use a blob reference that does not have a net source to create operators. Create the operator from an explicit net object.Method  is not a registered operator. Did you mean: [,]c                     s   j  f| |S r5   )r   r>   r   rE   ry   r-   r.   <lambda>/  s   z+BlobReference.__getattr__.<locals>.<lambda>)	
startswithAttributeErrorr   rw   rF   joinr   r=   nearby_opnamesry   rE   r-   r   r.   r     s"    


zBlobReference.__getattr__c                 C   s6   t   dd tD }ttttt| t| j|S )Nc                 S   s    g | ]}d |ksd|kr|qS )_ENGINE_Z_ENGINE_CUDNNr-   r_   rV   r-   r-   r.   
<listcomp>4  s    z)BlobReference.__dir__.<locals>.<listcomp>	r   r8   sortedr6   r	   dirtyper   __dict__ry   Zadditional_methodsr-   r-   r.   __dir__2  s    
zBlobReference.__dir__)N)N)r&   r'   r(   __doc__rz   r}   r   r   r   r   r   r   r   r   r   r   r   r   r-   r-   r-   r.   rq      s   


rq   c                 C   s    t | tr| d} t |  S )z'prefix the name with the current scope.ascii)rs   r
   ru   r   ZCurrentNameScoper2   r-   r-   r.   
ScopedName?  s    

r   c                 O   s   t t| f||S )z-Returns a blob reference with scope prefixed.)rq   r   )r2   r>   r   r-   r-   r.   ScopedBlobReferenceF  s    r   c                 C   s   t | tst | tr"t| |dgS t| tkr4| gS t| ttfkrg }| D ]b}t |tsdt |trx|t||d qLt|tkr|| qLt	d
t|t|t|qL|S t	dt| t| f dS )zZA helper function to rectify the input or output of the CreateOperator
    interface.
    rY   z/I/O blob #{} of unsupported type: {} of type {}z)Unknown input/output type: %s of type %s.N)rs   r   r
   r   r   rq   listtuplerj   	TypeErrorr   rf   rv   )ZblobsrY   Z	rectifiedblobr-   r-   r.   _RectifyInputOutputK  s0      r    c	                 K   sh  t  }
tjdr0t }d|dd |
_| |
_	||
_
t|}t|}|
jdd |D  |
jdd |D  |rt|}|
jdd |D  |dk	r|
j| nt dk	r|
jt  |dk	r||
_|dk	r||
_d	|	kr|	d	 |
j_|	d	= |dk	r|
j| t|	D ],\}}|dk	r"|
j t|| q"t rdt|
 |
S )
zA function wrapper that allows one to create operators based on the
    operator type. The type should be a string corresponding to an operator
    registered with Caffe2.
    ZCAFFE2_DEBUGr   Nc                 S   s   g | ]}t |qS r-   r   r^   r-   r-   r.   r     s     z"CreateOperator.<locals>.<listcomp>c                 S   s   g | ]}t |qS r-   r   r_   or-   r-   r.   r     s     c                 S   s   g | ]}t |qS r-   r   r^   r-   r-   r.   r     s     rL   )r   OperatorDefosenvironget	tracebackformat_stackr   
debug_infor   r2   r   r]   r<   rX   control_inputrW   CopyFromr   ZCurrentDeviceScoperC   rL   argr   addr   MakeArgumentr   IsImmediateRunOperatorImmediate)Zoperator_typer   outputsr2   r   rW   r   rC   r   r   operatorstackkeyr3   r-   r-   r.   CreateOperatork  s@    




r   Fc                 C   s|   |r|| }|j } |j}n@t| tr:| d | d | d } t|trZ|d |d |d }t| |d}|rxt|| |S )Nr   r   r   r   )ZforwardZbackwardrs   r   r=   Zregister_python_opZregister_python_gradient_op)fgrad_fpython_func_typepass_workspacefunctokenr-   r-   r.   _RegisterPythonImpl  s    

r   c                 O   s(   t | |||d|d< td||f||S )aC  
    `f` should have a signature (inputs, outputs)

    If `pass_workspace` is True, the signature is changed to
    (inputs, outputs, workspace) where `workspace` is the workspace the op
    is going to run on. This is potentially dangerous (as the op can manipulate
    the workspace directly), use on your own risk.
    r   r   Python)r   r   )r   r   r   r   r   r   r>   r   r-   r-   r.   CreatePythonOperator  s       
r   c                 C   sN   t | D ]@\}}||kr |  S t|tkr|j|ks@|j|kr|  S qdS )zRA helper function to get the index from a gradient list, None if not
    matching.N)	enumerater   rn   ro   rp   )Zg_listr2   r`   gr-   r-   r.   GetIndexFromGradientList  s    
r   OpSSArV   in_versionsout_versionsGradGenMetagrad_opidxgradientrW   SparseGradGenMetaZgrad_op_indicesZidx_indicesZgrad_op_valuesZ
idx_valuesc                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dZdZdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$Zd%d& Zd'd( Zd)d* Zd+S ),IRznA simple IR class to keep track of all intermediate representations used
    in the gradient computation.
    c                 C   sh   g | _ tdd | _tt| _i | _tdd | _tt| _tt| _	|D ]}| 
| qJ| | d S )Nc                   S   s   t tS r5   r   r   r-   r-   r-   r.   r         zIR.__init__.<locals>.<lambda>c                   S   s   t tS r5   r   r-   r-   r-   r.   r     r   )ssar   input_usagesintfrontiergradient_frontiergradient_generatorsr   out_version_historyin_version_historyPlaySanityCheckry   	operatorsrV   r-   r-   r.   rz     s    


zIR.__init__c                 C   sB   |D ]8}|j dkr|jd | jkrtd|jd t|qd S )NZStopGradientr   zsStopGradient's output '{}' is orphan.
You typically want to specify same input and output for
StopGradient. Op:

{})r   rX   r   
ValueErrorr   rv   r   r-   r-   r.   r     s    
 zIR.SanityCheckc                 C   s   i }|j D ]L}| j| ||< | j| | j|  t| j | j| || j| f q
i }|jD ]H}|| jkr| j|  d7  < | j| ||< | j| || j| f qb| jt	||| dS )z"Adds an op to the current IR, and update the internal states to
        reflect the blobs and versions after the execution of the op.
        r   N)
r]   r   r   rj   rf   r   r   rX   r   r   )ry   rV   r   sr   r-   r-   r.   r     s    
 

zIR.Playc                    s    j | \}}}t||} fdd}	 fdd}
|dk	r~|j| }||  j| kr|td||| | j| f |	| n||kr j| || krtd|||  j| f |	| n^||kr j| || krtd|||  j| f |
| n||krtd	|t|f dS )
z>Checks if the gradient operators can be correctly carried out.c                    sJ   d}|d7 }|d | 7 } j|  D ] \}}|d ||7 }|d7 }q$|S )NDEBUG HELP:
z8Maybe you use same output blob twice for different ops?
 == Version history of blob [{}]
zVersion (out) {} <-- {}
)r   r   r2   r   rV   versr|   r-   r.   versionMismatchInfoOut4  s    
z=IR.CheckGradientOperatorInput.<locals>.versionMismatchInfoOutc                    sJ   d}|d7 }|d | 7 } j|  D ] \}}|d ||7 }|d7 }q$|S )Nr   z.Maybe the blob was overwritten by another op?
r   zversion (in) {} <-- {}r   )r   r   r   r|   r-   r.   versionMismatchInfoIn=  s    
z<IR.CheckGradientOperatorInput.<locals>.versionMismatchInfoInNzgGradient name "%s" is expected to correspond to version %d of "%s", but currently we have version %d.

zVGradient operator needs output "%s" at version %d, but currently we have version %d.

zUGradient operator needs input "%s" at version %d, but currently we have version %d.

zlBlob name "%s" not in the scope of operator: %s
and is not generated by any of the local gradient operators.)r   r   rX   r   r   r   rv   )ry   Zgrad_op_inputg_output
fwd_op_idxlocally_generated_blobs
forward_opr   r   Zoriginal_indexr   r   original_namer-   r|   r.   CheckGradientOperatorInput-  sd    
	
 
 	 


zIR.CheckGradientOperatorInputc              	   C   s$  t |D ]\}}t |D ] \}}t|dkr:|d }nt|dksJt|d \}}}	}
}}|d \}}}}}}||ks~t||kstdtt|d ks|d kst|	d ks|d kst|dks|dkst|
dks|dkstt|p||| |	p||
| ||}| j| | | qqd S )Nr   r   r   z0Unequal devices for sparse generators: {} and {})	r   rf   r1   r   dev1dev2r   r   rj   )ry   sparse_generatorsr2   Zinput_generatorsversion
generators	generatorZop1_iZidx1_iZop1_vZidx1_vZg1Zdev_1Zop2_iZidx2_iZop2_vZidx2_vZg2Zdev_2r-   r-   r.   AppendSparseGeneratorst  s6    

    zIR.AppendSparseGeneratorsc              
   C   s  | j | \}}}g }tdd }	|D ]}
|
jD ]}| |||| q.|dd |
jD  t|
jD ]\}}t||}|dk	rd|j| }|| }|| }t|t	kr|j
|krt|
|dd||
j}n"|j|ksttdd|
|||
j}|	| | | qd| j| | t|
|||
j qdq$| |	 t|D ]\}}|j| }|| }|sTq0t|t	krt|j
|krt|j|kr| j| | tdddd||j n.t||kr0| j| | tdd||j q0t|D ]0\}}|dk	r|j| }|| }|| j|< qdS )z1Updates gradient_generators and gradient_frontierc                   S   s   t tS r5   r   r-   r-   r-   r.   r     r   z,IR.BuildGradientGenerators.<locals>.<lambda>c                 S   s   g | ]}t |qS r-   rv   )r_   r   r-   r-   r.   r     s     z.IR.BuildGradientGenerators.<locals>.<listcomp>Nr   )r   r   r]   r  r<   rX   r   r   r   rn   ro   r   rW   rp   r1   rj   r   r   r  rv   r   )ry   r   gradient_opsr   g_inputr   r   r   r   r  r   r   r`   rX   Zinput_index
input_nameinput_versionr   mr-   r-   r.   BuildGradientGenerators  s    
   


             



zIR.BuildGradientGeneratorsc                 C   s   dd }|D ]}t |tkr<|\}}}}|r|j|   S qt |tksLt|\}}	}
}}}|rt||j|	 d  S |
r||
j| d  S q|d S )Nc                 S   s    |  |r| d t|  S | S r5   )endswithrf   )r   suffixr-   r-   r.   remove_suffix  s    
z-IR._GetSumOpOutputName.<locals>.remove_suffixZ_indicesZ_valuesZ_grad)r   r   rX   r   r1   )ry   r  r  r  r   r   r   _op_iidx_iop_vidx_vr-   r-   r.   _GetSumOpOutputName  s    zIR._GetSumOpOutputNameZis_auto_gen_sum_ops!only_keep_is_auto_gen_sum_ops_tagc                 C   s   d}|D ]*}|j jD ]}|dtjkrd} qqq|r\|D ]}|j jdtjg q<n>|D ]8}|D ]*}|j |j  |j jdtjg qh qq`d S )NFz{}:1T)rW   rO   r   r   !ONLY_KEEP_IS_AUTO_GEN_SUM_OPS_TAGr<   IS_AUTO_GEN_SUM_OPS_TAGr   )ry   sum_opsr  r  r  rO   rV   r-   r-   r.   _SetSumOpsDeviceOption  s$    


zIR._SetSumOpsDeviceOptionc                 C   sL   d|j |  d| }|jdkr0t||| n
||j |< |j | |d fS )Nr  z_autosplit_{}Ifr   )rX   r   r   r   )ry   r   r   cntZnew_grad_outputr-   r-   r.   _DisambiguateGradOpOutput  s    

zIR._DisambiguateGradOpOutputc                 C   s   t |t |krtdd S )NznThe gradient output of empty gradient op can not be the same as the normal name of the current input gradient.)rv   r   )ry   out_base_namer   r-   r-   r.   _CheckSumOpsConflict  s    zIR._CheckSumOpsConflictc                 C   s   g }d}t |dkstd}|D ]p}|\}}}	}
t|	tk	s@t|rv|rXd}|j| }n| |||\}}|| q | ||	 |t|	 q ||kr|	|}|| |d  |d< ||< t
ddd |D t|g}||fS )Nr   r   TFZSumc                 S   s   g | ]}t |qS r-   rq   r_   xr-   r-   r.   r   ?  s     z'IR._MakeDenseSumOps.<locals>.<listcomp>)rf   r1   r   rn   rX   r   rj   r"  rv   indexr   rq   )ry   r  r!  Zsum_op_inputr  Zfirst_grad_opr  r   r   r   r  outr  r-   r-   r.   _MakeDenseSumOps!  s4    
 zIR._MakeDenseSumOpsc                 C   s6  g }g }d}d}|D ]}t |tks(t|\}}	}
}}}|rZ| ||	|\}}|| n| ||j ||j |
r| |
||\}}|| q| ||j ||j q|d }|d }|d }|d }tddd |D d	d ||fD dd
tddd |D dd ||fD dd
g}t	||d}||fS )Nr   Z_indices_concatZ_indices_concat_splitZ_values_concatZ_values_concat_splitZConcatc                 S   s   g | ]}t |qS r-   r#  r$  r-   r-   r.   r   d  s     z(IR._MakeSparseSumOps.<locals>.<listcomp>c                 S   s   g | ]}t |qS r-   r#  r$  r-   r-   r.   r   e  s     )Zaxisc                 S   s   g | ]}t |qS r-   r#  r$  r-   r-   r.   r   k  s     c                 S   s   g | ]}t |qS r-   r#  r$  r-   r-   r.   r   l  s     ro   rp   )
r   r   r1   r   rj   r"  ro   rp   r   rn   )ry   r  r!  Zindices_concat_inputZvalues_concat_inputZcnt_iZcnt_vr  r  r  r  r  r   r  r'  Zindices_concat_outputZindices_concat_splitZvalues_concat_outputZvalues_concat_splitr  Zsum_op_outputr-   r-   r.   _MakeSparseSumOpsC  sV    zIR._MakeSparseSumOpsc                 C   s   | j | | }| ||}ttdd |D }t|dks@t|d tkr^| ||\}}n |d tksnt| 	||\}}| 
|| ||fS )Nc                 s   s   | ]}t |V  qd S r5   r   r$  r-   r-   r.   	<genexpr>z  s     z!IR._MakeSumOps.<locals>.<genexpr>r   r   )r   r  r   r6   rf   r1   r   r(  r   r*  r  )ry   r  r  r  r!  typesr  r   r-   r-   r.   _MakeSumOpsw  s    zIR._MakeSumOpsc                    s   t dd |D dkrtdt |dk r.dS g }g  |D ]\}|jrP |j t|tkrp|jr||j q:t|tkst	|jj
r:||jj
 q:t  dkrt fdd dd  D std	d
S )Nc                 S   s   h | ]}t |qS r-   r+  r_   r   r-   r-   r.   	<setcomp>  s     z/IR._VerifyGradientGenerators.<locals>.<setcomp>r   zQAutomatic aggregation of a mix of sparse and dense gradients is not supported yetr   Fc                 3   s   | ]}t | d  V  qdS r   N)rT   )r_   dZall_device_optionsr-   r.   r,    s   z/IR._VerifyGradientGenerators.<locals>.<genexpr>zBUnexpected behavior: not all grad ops have the same device option.T)rf   r   rW   rj   r   r   r   r   r   r1   rp   all)ry   r  Zall_gradient_namesr   r-   r3  r.   _VerifyGradientGenerators  s,    
zIR._VerifyGradientGeneratorsc                 C   s   | j | \}}}g }i }tt|jD ]\}}|| }	| j| |	 }
t|
dks&||
d kr^q&| j| |	 }z| |s|W q&W n2 tk
r } ztd	||W 5 d}~X Y nX | 
||	\}}|| |||< q&||fS )a  For each input name in the forward op, check if we will need to
        add gradient accumulation. If so, do gradient accumulation and return
        the list of gradient operators.

        The criteria for doing gradient accumulation is:
        (1) the specific input version has been used by multiple operators.
        (2) the current fwd_op_idx is the first to use that input, i.e. in the
            backward pass, is the last to optionally generate the gradient for
            the op.
        (3) For the operators that used the input, their gradient operators
            have generated more than 1 gradient.

        When accumulating operators, our current solution is to rename all the
        created gradients with an internal intermediate name, and then add a
        Sum() operator that adds up all the gradients. This may use more memory
        due to intermediate storage, but is usually the fastest approach as one
        can do one single sum for multiple intermediate gradients.
        r   r   z/Gradients for param ''{}'' failed to verify: {}N)r   r   r6   r]   r   rf   r   r5  r   r   r.  r<   )ry   r   r   r   r   additional_sum_opsgrad_mapZ_ir  r  Zinput_usager  errr  r   r-   r-   r.   DoGradientAccumulation  s.    


zIR.DoGradientAccumulationc                 C   sB   t ||rdnd t||j}| jt| | jt|  | d S Nr   )r   rv   rW   r   r   rj   )ry   ygradautograd_opr  r-   r-   r.   _AppendAutoGradGenerator  s     
 zIR._AppendAutoGradGeneratorZ_autogen_gradc                 C   s   i }g }t |D ]\}}d }|d krVtd|gt|tj gdd}|| |jd }t|trztt|d t|d nt||t|< |d k	r| 	||| q||fS )NConstantFillg      ?)r3   r   r   )
r   r   rv   r   AUTOGEN_GRAD_SUFFIXrj   rX   rs   rn   r>  )ry   ysinput_to_gradr
  r;  r   r=  r-   r-   r.   _GetInitGradients  s(      


zIR._GetInitGradientsc                    s   i }g }| j | \}}}t fdd|jD }tdd |D rL|jdkrt||\}}	| ||||	 t|j	|	D ].\}
}|d k	s|
 ks|
t|jkrx|||
< qx||fS )Nc                 3   s   | ]}  |d V  qd S r5   r   r_   r2   rB  r-   r.   r,    s    z4IR._GenerateGradientsForForwardOp.<locals>.<genexpr>c                 s   s   | ]}|d kV  qd S r5   r-   r/  r-   r-   r.   r,    s     ZZeroGradient)
r   r   rX   r4  r   GradientRegistryGetGradientForOpr  zipr]   )ry   forward_op_idxrB  Znew_input_to_gradr
  r   r   r   r   r  r2   r<  r-   rF  r.   _GenerateGradientsForForwardOp  s6        
z!IR._GenerateGradientsForForwardOpc                 C   sL  t |trtdd |D }nt |ts0tdt|D ]<}| j| | j|< | jt| | jt|  	t
| j q8| |\}}ttt
| jD ]F}| ||\}}|| ||7 }| |\}}	||	 ||7 }qi }
t|D ]X\}}|dk	rt |tst |trt|}ntt|d t|d }||
t|< q||
fS )a  Gets the backward pass that computes the derivatives of given blobs.

        Inputs:
          ys: a list or a dictionary specifying what blobs we want to compute
              derivatives of. If the input is a list, we will automatically
              generate their gradients with all-one values; if the input is a
              dictionary, for any dictionary entries that are not None, we will
              take the corresponding blobs as their gradients; for all those
              that are None, we will auto-fill them with 1.
        c                 s   s   | ]}|d fV  qd S r5   r-   )r_   r;  r-   r-   r.   r,  #  s     z%IR.GetBackwardPass.<locals>.<genexpr>z%ys should either be a list or a dict.Nr   r   )rs   r   dictr   r   r   r   r   rv   rj   rf   r   rC  reversedre   rK  updater9  r   r   r
   rq   rn   )ry   rA  r;  Zall_input_to_gradZall_gradient_opsrJ  rB  r
  r6  r7  Zall_input_to_grad_outr   valZgrad_outr-   r-   r.   GetBackwardPass  sD    

 




zIR.GetBackwardPassN)r&   r'   r(   r   rz   r   r   r  r  r  r  r  r  r  r   r"  r(  r*  r.  r5  r9  r>  r@  rC  rK  rP  r-   r-   r-   r.   r     s.   
GK	"4#.r   c                   @   sF   e Zd ZdZi Zedd Zedd Zedd Zedd	d
Z	dS )rG  zEGradientRegistry holds the mapping from operators to their gradients.c                    s    fdd}|S )z.A decorator for registering gradient mappings.c                    s   |  j < | S r5   )gradient_registry_)r   clsrE   r-   r.   Wrapper]  s    
z2GradientRegistry.RegisterGradient.<locals>.Wrapperr-   )rS  rE   rT  r-   rR  r.   RegisterGradientY  s    z!GradientRegistry.RegisterGradientc                    sz   dd   fdd|D }t | |\}}dd fdd|D }g }|D ] }t }|| || qP||fS )Nc                 S   s   | d kr t  }| st|S z.| \}}t  }||_||_| sJt|W S  tk
r   t  }| |_|	 sxt| Y S X d S r5   )
r=   ZGradientWrapperis_emptyr1   ro   rp   	is_sparser   denseis_dense)r<  wro   rp   r-   r-   r.   from_untypedf  s     z:GradientRegistry._GetGradientForOpCC.<locals>.from_untypedc                    s   g | ]} |qS r-   r-   )r_   r<  )r[  r-   r.   r   x  s     z8GradientRegistry._GetGradientForOpCC.<locals>.<listcomp>c                 S   s4   |   rd S |  r"t| j| jS |  s.t| jS r5   )rV  rW  rn   ro   rp   rY  r1   rX  )grad_wrapperr-   r-   r.   
to_untyped|  s    z8GradientRegistry._GetGradientForOpCC.<locals>.to_untypedc                    s   g | ]} |qS r-   r-   )r_   r\  )r]  r-   r.   r     s     )r=   Zget_gradient_defsrh   r   r   ri   rj   )rS  Zop_defr   Zgrad_defs_strr  Z	grad_defsZgrad_def_strZgrad_defr-   )r[  r]  r.   _GetGradientForOpCCc  s     
z$GradientRegistry._GetGradientForOpCCc              
   C   s   z|  ||\}}W n^ tk
rr } z@|j| jkrJ| j|j ||\}}ntd|j|t|W 5 d }~X Y nX |d krg |fS t|tk	r|g}||fS )Nz5Exception when creating gradient for [{}]:{}.
Op: 
{})r^  	Exceptionr   rQ  r   rv   r   )rS  rV   r   r
  r  er-   r-   r.   rH    s(    
 
  z!GradientRegistry.GetGradientForOpFc                 C   s   t |}||S )a  Gets the backward pass for the list of operators.

        Args:
            operators: a list of operators constituting the forward pass.
            ys: a list or a dictionary specifying what blobs we want to compute
                derivatives of. If the input is a list, we will automatically
                generate their gradients with all-one values; if the input is a
                dictionary, for any dictionary entries that are not None, we'll
                take the corresponding blobs as their gradients; for all those
                that are None, we will auto-fill them with 1.
        Returns:
            gradient_ops: a list of gradient operators to run.
            all_input_to_grads: a map from input to their corresponding
                gradients.
        )r   rP  )rS  r   rA  Zys_generate_gradientZirr-   r-   r.   rP    s    z GradientRegistry.GetBackwardPassN)F)
r&   r'   r(   r   rQ  classmethodrU  r^  rH  rP  r-   r-   r-   r.   rG  U  s   
	
(
rG  ZDor  Whilec           	         s
  t | tr|  n| }t |tjs&t dkr2i  t | trR fdd| D  fS |jD ]}| krXd t|< qXg }|j	D ]}|js|j
D ]}| krd |< q fdd|j
D }|jD ] } t|dd  t|< q fdd|jD }|||f q|| fS )a  
    Given a net, return a structure containing the version of each input and
    output blob used by each operator.

    Args:
        net:            either a Net or a NetDef
        blob_versions:  (optional) map with current version number for given
                        blob names. If not provided or blob not found, start
                        from version 0.
    Returns:
        Tuple (ssa, blob_versions)
        ssa:            list of tuples (versioned_inputs, versioned_outputs)
                        for each op in the net. A versioned input is a tuple
                        (blob_name, version).
        blob_versions:  updated map with latest version of each blob found in
                        the net.
    Nc                    s   g | ]}t | qS r-   )get_ssar_   nblob_versionsr-   r.   r     s     zget_ssa.<locals>.<listcomp>r   c                    s$   g | ]}t | t |d fqS r   rv   r   r^   rf  r-   r.   r     s     r   c                    s    g | ]}t | t | fqS r-   r	  r   rf  r-   r.   r     s     )rs   r   rU   r   NetDefr1   r   external_inputrv   rV   r]   rX   r   rj   )	rY   rg  protor`   r   rV   r   r   r   r-   rf  r.   rc    s*    





rc  c                 C   s.   t  }| D ]\}}|t dd |D O }q
|S )z
    Given a ssa in the format produced by get_ssa(), return a set of blobs that
    are used before they are defined, which corresponds to inputs at version 0.
    c                 s   s   | ]\}}|d kr|V  qdS r1  r-   )r_   r2   verr-   r-   r.   r,    s      z&get_undefined_blobs.<locals>.<genexpr>)r6   )r   undef_blobsr   Z_outputsr-   r-   r.   get_undefined_blobs  s    ro  c                 C   s0   i }t | D ]\}\}}|D ]}|||< qq|S )z
    Given a ssa in the format produced by get_ssa(), returns a map from
    versioned blob into the operator index that produces that version of
    the blob. A versioned blob is a tuple (blob_name, version).
    )r   )r   	producersr`   Z_inputsr   r   r-   r-   r.   get_output_producers  s
    rq  c                    s   t  fdd|D }t| } fdd|D }t  }t|dkr| }||kr6||kr6|| }	|	|kr6||	hO }| |	 \}}
|| q6t|S )a  
    Given a ssa and blob_versions as produced by get_ssa(), returns the list
    of op indices that are necessary in order to generate the blobs in
    `outputs`, given blobs in `inputs`.
    Consider that the `inputs` are given in their latest version.
    c                 3   s"   | ]}t | t | fV  qd S r5   r	  r^   rf  r-   r.   r,    s     z%get_op_ids_in_path.<locals>.<genexpr>c                    s    g | ]}t | t | fqS r-   r	  r   rf  r-   r.   r     s     z&get_op_ids_in_path.<locals>.<listcomp>r   )r6   rq  rf   popr<   r   )r   rg  r   r   Z
inputs_setrp  queueused_op_idsr   op_idr  r-   rf  r.   get_op_ids_in_path  s    
rv  c                    s    fdd}| j D ]j}t|jdkrNtt|jD ]}||j| |j|< q2q|jdkrf||j|_q|jdrt|  qdS )a/  
    Parameters
    ----------
    op : Caffe2 operator (RecurrentNetworkOp or RecurrentNetworkGradientOp).
    prefix: this argument is not used in this function, just for legacy support.
    blob_remap : Dictionary that represents the map from old blob name to new.

    Updates blob names in arguments of RecurrentNetworkOp and
    RecurrentNetworkGradientOp to conform to cloned input and output of both
    operators and also makes sure names of locally generated blobs in arguments
    have the same prefix as the input and output of the operators.
    c                    s&   t | tr| d}  | | dS )Nrr   )rs   r
   ru   r   encode)blob_str
blob_remapr-   r.   get_remapped_str!  s    

z4recurrent_network_op_remap.<locals>.get_remapped_strr   Ztimestepstep_netN)r   rf   stringsre   r2   r   r  remap_proto)rV   prefixrz  r{  argumentr`   r-   ry  r.   recurrent_network_op_remap  s    

r  c                 C   s   g }| j dks| j dkr"ddg}nddg}| jD ]l}|j|kr0|js`td| j  d |j d	 t|j}|j|jjr~|jjnd
d |d}|j|	  q0d S )Nr  AsyncIfZthen_netZelse_netZloop_netZcond_netzExpected non empty net in z's z	 argumentr   Z	_remapped)r2   rz  )
r   r   r2   re  r1   r   Clone_netr   rU   )rV   r  rz  Znet_arg_namesr  subnetZremapped_subnetr-   r-   r.   control_op_remap1  s    



r  )RecurrentNetworkZRecurrentNetworkGradientr  rb  r  c                 C   s*   t | j}|d|}| j|  d S )Ncloned_sub_net)r   re  r  r   rU   )r  rz  r  r  r-   r-   r.   r~  K  s    
r~  c                 C   sp  ddl m} t| tst|dkr&i }|dk	rt||js>t|  }|dk	sRtt| t|  }t	|dkstdj
||  | | dtt| | }	t| | D ]$\}
}|
|	krt||t|	|
 < q|  }t|\}}t|}t|D ]6}||krq
n ||kr2|||< n|| ||< q
| j|||d}|sh|rh|| ||fS )a  
    Clone the given Net, binding its input schema to the given `inputs` record.
    Blob names defined by the net are prepended with the given `prefix`.

    Args:
        net:        the net to clone
        name:       the name of the new net
        prefix:     the prefix to append to local blobs
        blob_remap: (optional) dict with additional blob name remapping.
        inputs:     (optional) input record that will provide actual input
                    values for the cloned net. Must be compatible with the
                    net's input schema or be a strict superset of it
        keep_schema: by default (True), the original schema will be kept and
                     remapped accordingly. otherwise, the schema will be set as
                     inputs or left empty if inputs is not given.
    Returns:
        Tuple (cloned_net, blob_remap)
        clone_net:  the cloned Net
        blob_remap: a map from original blob names into remapped blob names
    r   schemaNzhSchemas don't match, extra fields {diff} found in the net {name}. original: {original}; inputs: {inputs})diffr2   originalr   keep_schema)caffe2.pythonr  rs   r   r1   ZFieldinput_recordr6   field_namesrf   r   NamerL  rI  field_blobsrv   rU   rc  ro  r   r  set_input_record)rY   r2   r  rz  r   r  r  r  r  Zoriginal_mappingfnZfbrl  r   rg  rn  r   Z
cloned_netr-   r-   r.   clone_and_bind_netV  sH      




r  c                 C   s   t ttr| S t| S r5   )rs   r]   rq   )Zblob_name_or_refr-   r-   r.   _get_blob_ref  s    r  c                    sB   ddl m}  fdd| D }|s&dS |j| fdd|D dS )z
    Tries to recover record by taking a subset of blob names with
    a given prefix name and interpreting them as schema column names
    r   r  c                    s&   g | ]}|  r|t d  qS r5   )r   rf   rE  r  r-   r.   r     s    
z-_recover_record_by_prefix.<locals>.<listcomp>Nc                    s   g | ]}t  | qS r-   r  rE  r  r-   r.   r     s     )Z	col_blobs)r  r  Zfrom_column_list)namesr  r  Zcolumn_namesr-   r  r.   _recover_record_by_prefix  s    r  c                   @   s"  e Zd Ze Zi Zedd Zedd Zd~ddZ	dd	d
Z
dd Zdd Zdd ZdddZdd Zdd ZdddZdd Zdd Zdd  Zd!d" Zdd#d$Zdd%d&Zd'd( Zd)d* Zdd+d,Zd-d. Zdd0d1Zdd2d3Zdd4d5Zd6d7 Zd8d9 Z d:d; Z!d<d= Z"dd?d@Z#dAdB Z$dCdD Z%dEdF Z&dGdH Z'dIdJ Z(dKdL Z)dMdN Z*dOdP Z+e,dQdR Z-e,dSdT Z.dUdV Z/dWdX Z0dYdZ Z1d[d\ Z2d]d^ Z3d_d` Z4dadb Z5dcdd Z6dedf Z7ddhdiZ8eddjdkZ9ddldmZ:dndo Z;dpdq Z<ddrdsZ=dtdu Z>dvdw Z?ddxdyZ@dzd{ ZAd|d} ZBdS )r   c                  C   s&   ddl m}  | jdd}|r"|jS dS )Nr   
NetBuilderF)requiredr   )caffe2.python.net_builderr  currentr2   )r  Zbuilderr-   r-   r.   current_prefix  s    zNet.current_prefixc                 C   s`   d dd t | fD  }} d}|tjkrH| d t| }|d7 }q$t jt|gO  _|S )N/c                 s   s   | ]}|r|V  qd S r5   r-   r$  r-   r-   r.   r,    s     z)Net._get_next_net_name.<locals>.<genexpr>r   r  )r   r   r  _net_names_usedrv   r6   basenamer2   Znext_idxr-   r-   r.   _get_next_net_name  s    



zNet._get_next_net_nameFc              	   C   s  d| _ d| _t | _d| _t | _t | _tt| _	t
|tjkrf|}|rT|| _nt | _| j| dd | jjD }| jt| jj t }| jjD ]}|t|j q|D ]}|| q|D ]}| j| qt| jjd }	g }
|D ]D}|| jjd rz|
t||	  W q tk
r8   Y qX qt|
rVt|
d | _nd| _| jj}n|}t | _d| _t|| j_i | _dS )a  
        Create a Net.
        Args:
            name_or_proto:  If a NetDef is provided, clone it (or take ownership,
                            depending on the value of `inplace`). Otherwise,
                            create an empty net with the given name.
            inplace: If a NetDef is provided, take ownership when `inplace` is True;
                     otherwise, clone it.
        NFc                 S   s   g | ]}t |jqS r-   )r   rX   r   r-   r-   r.   r     s     z Net.__init__.<locals>.<listcomp>_blob_r   r   )_input_record_output_recordr6   _registered_blob_names_recreate_lookup_tables_op_outputs_external_input_mapr   r   
_attr_dictr   r   rj  r  r   rV   rN  rk  r]   rf   r2   r   rj   r   r   max_next_name_indexr   r  _next_blob_name_ids)ry   Zname_or_protoZinplacerl  existing_outputsZexisting_namesrV   rX   Zouts
prefix_lenZautogen_indicesr   r2   r-   r-   r.   rz     sN    





zNet.__init__Nc                    s   t |tst| jD ],}|  jkr| jkr  j| q  j fdd| jD  | j	}|d k	rdd |D }|D ]}|j
| q|D ]@}|jdkr|jD ]*}|jdr|jj	D ]}|j
| qqq |  S )Nc                    s   g | ]}|   jkr|qS r-   )rU   external_outputr   r|   r-   r.   r     s   z!Net.AppendNet.<locals>.<listcomp>c                 S   s   g | ]}t |qS r-   )copydeepcopyr   r-   r-   r.   r     s     r  r|  )rs   r   r1   rU   rk  r  rj   r  r<   rV   rW   r   r   r   r2   r  re  
_ExtendOps)ry   rY   rW   r`   opsrV   r   Zstep_opr-   r|   r.   	AppendNet  s2    





zNet.AppendNetc                 G   sD   |D ]:}t |ts.| jg | dg |gd}n|}| |g  qd S )Nlogshaperp   )rs   rq   GivenTensorStringFillNextNameZPrint)ry   Zmsg_or_blobsZmsg_or_blobr   r-   r-   r.   LogInfo"  s    
  zNet.LogInfoc                 C   s   | j | | dS )z
        Add `obj` to the list of attributes in this net under the given `name`.
        Attributes are user-defined objects and have no pre-defined semantics.
        N)r  rj   )ry   r2   objr-   r-   r.   add_attribute,  s    zNet.add_attributec                 C   s   | j |g S )z
        Returns the list of attributes in this net for a given `name`.
        Attributes are user-defined objects added with `add_attribute'.
        )r  r   ry   r2   r-   r-   r.   get_attributes3  s    zNet.get_attributesd   Tc                 C   sn   |r|rt dt|  jD ]J\}}|r4|| }n,|r\tt|t| ttjj	 }n|}||j
_qdS )a  
        Adds a random seed to each op in the net.
        If sequence_seed is set, the i-th op has rand_seed=`seed + i`
        If seed_on_op_def is set, the op rand_seed=hash(str(op))
        sequence_seed and seed_on_op_def cannot be both set to True.
        z<sequence_seed and seed_on_op_def cannot be both set to True.N)r1   r   rU   rV   r{   rv   npZiinfoZuint32r  rW   rL   )ry   seedZsequence_seedZseed_on_op_defr`   rV   Z	curr_seedr-   r-   r.   set_rand_seed:  s    

$zNet.set_rand_seedc                 C   s   | j jS r5   )r  r2   r|   r-   r-   r.   r  L  s    zNet.Namec                 C   s   |   S r5   )r  r|   r-   r-   r.   r   O  s    zNet.__str__c                    s   t  tr"| jg pdtj dS |d kr6t  ntj |d  fdd} jtjkrh|| j	S  jtj
kr~|| jS  jtjkr|| jS  jtjkr|| jS || jS d S )Nr   )dtyper3   )r  c                    s   | g p
d j    dS )Nr   r  )r  flattentolist)r   arrayblob_outr-   r.   do_set_  s    
zNet.Const.<locals>.do_set)rs   boolr?  r   r,   r  r  r  Zint32ZGivenTensorIntFillZint64ZGivenTensorInt64Fillrv   r  ZGivenTensorBoolFillZGivenTensorFill)ry   r  r  r  r  r-   r  r.   ConstR  s(    




z	Net.Constc                 C   s*   | j r|   t|}|| jkp(|| jkS )z
        Returns true if the given BlobReference is produced as output of
        an operator in this net, or if it is provided as an external input.
        )r  _RecreateLookupTablesrv   r  r  ry   r   r2   r-   r-   r.   BlobIsDefinedq  s    zNet.BlobIsDefinedc                 C   s<   t |}| jjD ] }|jD ]}||kr  dS qq|| jkS )z
        Returns true iff the given BlobReference is used by any operator
        or this net, or if it is one of the external inputs of the net.
        T)rv   r  rV   r]   r  )ry   r   	blob_namerV   r]   r-   r-   r.   UsesBlob{  s    
zNet.UsesBlobc                 C   sd   t  }| jjD ] }|t |jO }|t |jO }q| jjrH|t | jjO }| jjr`|t | jjO }|S )z=
        Returns a set of blob names used in the net
        )r6   r  rV   r]   rX   rk  r  )ry   Z
blob_namesrV   r-   r-   r.   UsedBlobNames  s    zNet.UsedBlobNamesc                 C   s(   t |}| |std| t|| S )z
        Given the name of a blob produced by this net, return a BlobReference
        to it. If the blob is not produced by any op in this net,
        raises KeyError.
        zNet does not define blob %s)rv   r  KeyErrorrq   )ry   r  r-   r-   r.   
GetBlobRef  s    
zNet.GetBlobRefc                    s  dkri n}t  | | jt }| |_ dkrLi  |dkrhtt	dt
j} fddfdd fdd|jdd= |jfd	d
|D  |j |j t||rDddlm}	 | jr|	| jfdd
| j D _| jrD|	| jfdd
| j D _j| j |rt }
t } jdd=  jdd=  jD ]T}|jD ].}||
kr j|g n
|| q|jD ]}|
| qΐq|
D ]"}||kr j|g qS )aA  
        Clone this net.
        Args:
            name:        name of the cloned net
            blob_remap:  optional map with list of blob names to replace
            op_id_mask:  optional list of operator indices to include in
                         the cloned net. If not provided, all ops are included.
        Nr   c                    s   t | }t  ||S r5   ri  )r   rx  ry  r-   r.   r{    s    z#Net.Clone.<locals>.get_remapped_strc                    s*    fdd| D }| d d = |  | d S )Nc                    s   g | ]} |qS r-   r-   r_   r[   r{  r-   r.   r     s     z1Net.Clone.<locals>.remap_list.<locals>.<listcomp>)r<   )Z
proto_listnew_listr  r-   r.   
remap_list  s    
zNet.Clone.<locals>.remap_listc                    sR   t  }||  |j |j |jkrN|j |rFd nd  |S )Nr  r   )r   r   r   r]   rX   r   )rV   new_op)rz  r2   remap_funcsr  r-   r.   remap_op  s    



zNet.Clone.<locals>.remap_opc                    s   g | ]} j | qS r-   )rV   )r_   ru  )rl  r  r-   r.   r     s     zNet.Clone.<locals>.<listcomp>r  c                    s   g | ]}t  |d qS r   r#  r_   r   r{  new_netr-   r.   r     s   c                    s   g | ]}t  |d qS r  r#  r  r  r-   r.   r     s   )DEFAULT_REMAP_FUNCSr  rN  r  r   rj  r   r2   r   re   rf   rV   r<   rk  r  r   r  r  r  Zfrom_blob_listr  r  r  r6   rU   r]   r   rX   )ry   r2   rz  Z
op_id_maskr  r  Zupdate_external_listZorig_remap_funcsZ	new_protor  r  Zused_outputsrV   ibobr-   )rz  r{  r2   r  rl  r  r  r  r.   r    sh    







z	Net.Clonec                    s  t |totdd |D }t |ttfr.|n|r:t|ntt||}|D ]}| |sLtd|qLdd t	|D dd |D }| j
}dd |D }	t||	\}
}	t|
|	||t|
|	g |}ttt|@ d	kstd
fddt|
D }t|tt |r |d ndfddfddt|	D  | | | fddtD t } fdd|D } jdd=  j| tt|_ jdd=  j| fdd|D fS )a  
        Clone this net, including only ops that are necessary in order to
        compute `outputs` given `inputs`. Return references to the cloned
        outputs. Internal blobs (blobs that are produced and consumed inside
        the net but not used as outputs) will be remapped to avoid name
        conflict.

        Args:
            name:    the name of the cloned net
            inputs:  map where the keys correspond to BlobReferences in the
                     original net, and the values correspond to external inputs
                     in the partially cloned net. If `inputs` is a list, don't
                     remap input names.
            outputs: outputs to be produced by the cloned net.

        Returns:
            Tuple (new_net, new_outputs)
                new_net:       a new Net object.
                new_outputs:   list of BlobReferences corresponding to the
                               outputs produced by new_net.
        c                 s   s$   | ]}t |tot|d kV  qdS )r   N)rs   r   rf   r^   r-   r-   r.   r,    s    z#Net.ClonePartial.<locals>.<genexpr>{} is not definedc                 S   s   i | ]\}}t |t |qS r-   r	  )r_   kvr-   r-   r.   rb   #  s      z$Net.ClonePartial.<locals>.<dictcomp>c                 S   s   g | ]}t |qS r-   r	  r   r-   r-   r.   r   $  s     z$Net.ClonePartial.<locals>.<listcomp>c                 S   s   i | ]}t |d qS rh  r	  r^   r-   r-   r.   rb   &  s      r   zTCannot partially clone net: some of the ops required would generate the given input.c                    s   g | ]\}}| kr|qS r-   r-   )r_   r`   rV   )rt  r-   r.   r   .  s      r  r   c                    s(   |  kr |  S | kr| S |  S d S r5   r-   )r  )input_namesr  rn  r-   r.   remap2  s
    zNet.ClonePartial.<locals>.remapc                    s   i | ]}| |qS r-   r-   r  )r  r-   r.   rb   :  s      c                    s   g | ]} | qS r-   r-   r^   blob_mappingr-   r.   r   <  s    c                    s   g | ]} | qS r-   r-   r   r  r-   r.   r   >  s     Nc                    s   g | ]}  |qS r-   )r  r   )r  r-   r.   r   D  s     )rs   r   r4  rL  r   rI  r  r1   r   r   r  rc  rv  rf   r6   r   ro  r   r  rU   rk  r<   r  r  )ry   r2   r   r   r  Zinput_is_pair_listrX   Zoutput_namesrl  rg  r   Zdisallowed_op_idsZsub_ssaZnew_inZnew_outr-   )r  r  r  r  r  rn  rt  r.   ClonePartial  sJ    
zNet.ClonePartialc                 C   s   |    | jS r5   )_InvalidateLookupTablesr  r|   r-   r-   r.   rU   F  s    z	Net.Protoc                 C   sp   |dkst |  j|d }|  j|d= |  j|g |  j| | j|j | j|j dS )zF inserting operator at index. Will update external blob list.
        r   N)r1   rU   rV   r<   external_outputsrX   external_inputsr]   )ry   rV   op_idxZtemp_opsr-   r-   r.   insert_op_at_idxJ  s    zNet.insert_op_at_idxc                    s   fdd t  fddjD }|d  jd jkrfddjD } jd	d	=  j| d}|r jD ]"}||krt	|i |d }q|S )
a   reroute tensor to new_producer. And feed new tensor to consumers
        and interseciton with can_modify if provided.
        Inputs:
            tensor: str or blob_reference the tensor to reroute
            new_producer: an op takes in tensor gives new_tesnor
            can_modify: a list/set of operators that consumes tensor and can be
            modified

        Returns:
            reroute_cnt: how many consumer op has been changed

        Note: assume no inplace blob in net
        c                    sx   | j krd}nd|  jks,td j| d}t jD ]4\}}d}|jD ]}|| krPd}|} qjqP|r> qtq>|S )Nr   z#new producer {} is not taking in {}FT)	r  r]   r1   r   r   r   rU   rV   rX   )tensorr  r&  rV   Zif_foundr   )new_producerry   r-   r.   _find_tensor_input_opc  s&    
 
z1Net.reroute_tensor.<locals>._find_tensor_input_opc                 3   s   | ]} |V  qd S r5   r-   )r_   t)r  r-   r.   r,  y  s     z%Net.reroute_tensor.<locals>.<genexpr>r   r   c                    s   g | ]}|kr n|qS r-   r-   r  )
new_tensorr  r-   r.   r   ~  s     z&Net.reroute_tensor.<locals>.<listcomp>N)
r  r]   r  rX   r  rU   r  r<   rV   remap_input)ry   r  r  Z
can_modifyr  r  Zreroute_cntrV   r-   )r  r  r  ry   r  r.   reroute_tensorU  s    


zNet.reroute_tensorc                 C   sX   t j|  d }|d k	rTt|  jD ]*\}}||kr(dtt	|| d |_
q(d S )N:r   )r   operator_tracebacksr   r  r   rU   rV   r   maprv   r2   )ry   Znet_tbr   rV   r-   r-   r.   PopulateProtoWithFileName  s
    zNet.PopulateProtoWithFileNameunnamedc                 C   s   t |}| |S )a   Return the blob that has not been defined or registered in the
        current net. It returns `ScopedBlobReference(prefix)`, if it's valid,
        otherwise `ScopedBlobReference(prefix) + '_auto_' + ?`. Different calls
        is guaranteed to return blob with different names.
        )r   NextBlob)ry   r  output_blob_baser-   r-   r.   NextScopedBlob  s    zNet.NextScopedBlobc                 C   sV   t |}|}d}t|| jks(| |rB|d t| }|d7 }q| jt| |S )a  Return the blob that has not been defined or registered in the
        current net. It returns `BlobReference(prefix)`, if it's valid,
        otherwise `BlobReference(prefix) + '_auto_' + ?`. Different calls
        is guaranteed to return blob with different names.r   Z_auto_r   )rq   rv   r  r  r   )ry   r  r  Zoutput_blobr&  r-   r-   r.   r    s    
zNet.NextBlobc                 C   s   |r| j jd | }|}|dk	r0|dt| 7 }|}| j|d}| tt|r|d t| }|dk	r||dt| 7 }|d7 }|| j|< qBn$| j jd t| j }|  jd7  _t|S )zReturns the next name to be used, if you do not want to explicitly
        name your blob. [Deprecated, use NextBlob, NextScopedBlob instead]r  Nr  r   r  r   r  )r  r2   rv   r  r   r  r   r  )ry   r  	output_idZoutput_name_baseZoutput_namer   r&  r-   r-   r.   r    s     zNet.NextNamec                 C   s4   | j j| |D ]}| jdd |jD  qd S )Nc                 S   s   g | ]}t |qS r-   r   r   r-   r-   r.   r     s     z"Net._ExtendOps.<locals>.<listcomp>)r  rV   r<   r  rN  rX   )ry   new_opsrV   r-   r-   r.   r    s    zNet._ExtendOpsc                 C   sx   t  }| jjD ]}|jD ]}|| qqt  }| jjD ]}|| q8|| jt  ks^t|| j	t  ksttdS )zt
        Called from unit tests to validate the internal lookup tables
        match the protobuf contents.
        N)
r6   r  rV   rX   r   rk  
differencer  r1   r  )ry   Ztest_op_outputsrV   r   Ztest_external_inpinpr-   r-   r.   _CheckLookupTables  s    
zNet._CheckLookupTablesc                 C   s
   d| _ d S NT)r  r|   r-   r-   r.   r    s    zNet._InvalidateLookupTablesc                 C   sZ   t  | _| jjD ]}|jD ]}| j| qqt  | _| jjD ]}| j| q>d| _d S rQ   )	r6   r  r  rV   rX   r   r  rk  r  )ry   rV   r   r  r-   r-   r.   r    s    
zNet._RecreateLookupTablesr   c                 C   sF   t | jj|d |\}}t r8|D ]}t| q(| | |S )ao  Add the gradient for operators in the net.

        Inputs:
          ys: a list or a dictionary specifying what blobs we want to compute
              derivatives of. If the input is a list, we will automatically
              generate their gradients with all-one values; if the input is a
              dictionary, for any dictionary entries that are not None, we will
              take the corresponding blobs as their gradients; for all those
              that are None, we will auto-fill them with 1.
          skip: skips the first n operators. This is provided mainly because a
              lot of nets may use the first few operators for data generation
              like stuff which really do not need to have gradients.

        Outputs:
          returns a map from the blob name in the input network to a blob
          containing gradient or a GradientSlice in case of sparse gradient

        Currently, this is hard-coded for float operators if there are branches
        (i.e. a blob is used as input to multiple operators). This is because
        the gradient accumulation (Sum) is float only right now.
        N)rG  rP  r  rV   r   r   r   r  )ry   rA  skipZgrad_opsrB  rV   r-   r-   r.   AddGradientOperators  s     
zNet.AddGradientOperatorsc                 C   s   | j jt||g d S r5   )r  r   r<   r   r   )ry   Zarg_name	arg_valuer-   r-   r.   AddArgument  s    zNet.AddArgumentc                 G   s   t |dkstg }t }|D ]4}t|}|| jkr<||ksHtd| || q|D ]8}t|}| jj|g | j	|g |
t| qXt |dkr|d S |S )Nr   z&Net already contains an input named %sr   )rf   r1   r6   rv   r  r   r  rk  r<   rN  rj   r  )ry   r   refsZinput_name_setr]   r  r-   r-   r.   AddExternalInput
  s$    zNet.AddExternalInputc                 G   sT   |D ]*}t |tst| |std|q|D ]}|  jt|g q4d S )Nr  )	rs   rq   r1   r  r   rU   r  r<   rv   )ry   r   rX   r-   r-   r.   AddExternalOutput  s
    zNet.AddExternalOutputc                 G   s(   | j dd |D  }t|ts$|g}|S )Nc                 S   s   g | ]}t |qS r-   r   r  r-   r-   r.   r   &  s     z/Net.AddScopedExternalInputs.<locals>.<listcomp>)r  rs   r   )ry   r   resr-   r-   r.   AddScopedExternalInputs$  s    
zNet.AddScopedExternalInputsc                 G   s   | j dd |D  S )Nc                 S   s   g | ]}t |qS r-   r  r  r-   r-   r.   r   .  s     z0Net.AddScopedExternalOutputs.<locals>.<listcomp>r  ry   r   r-   r-   r.   AddScopedExternalOutputs,  s    zNet.AddScopedExternalOutputsc                 C   s   t | jj|S r5   )r=   Zadd_observer_to_netr  r2   )ry   Zobserver_typer-   r-   r.   AddObserver2  s    zNet.AddObserverc                 C   s   t | jj| d S r5   )r=   Zremove_observer_from_netr  r2   )ry   Zobserverr-   r-   r.   RemoveObserver5  s    zNet.RemoveObserverc                 C   s   t | jjS r5   )r=   Znum_observers_on_netr  r2   r|   r-   r-   r.   NumObservers8  s    zNet.NumObserversc                 C   s   dd | j jD S )Nc                 S   s   g | ]}t |qS r-   r  r$  r-   r-   r.   r   =  s     z'Net.external_inputs.<locals>.<listcomp>)r  rk  r|   r-   r-   r.   r  ;  s    zNet.external_inputsc                 C   s   dd | j jD S )Nc                 S   s   g | ]}t |qS r-   r  r$  r-   r-   r.   r   A  s     z(Net.external_outputs.<locals>.<listcomp>)r  r  r|   r-   r-   r.   r  ?  s    zNet.external_outputsc              	   C   s   ddl m} | jd ks@| r8t| t| j ks@td| spt|   |	| || _W 5 Q R X n|| _| j D ]}| 
|s| | q| jS )Nr   r  zInput schema cannot be reset)r  r  r  	has_blobsr6   r  r1   	NameScoper  Z	NewRecordis_external_inputr  )ry   r  r  r   r-   r-   r.   r  C  s     

zNet.set_input_recordc                 C   s    t | jj|}|r| | dS )z
        Tries to recover input record by taking a subset of external_inputs with
        a given prefix name and interpreting them as schema column names
        N)r  r  rk  r  ry   r  recordr-   r-   r.   recover_input_record_by_prefixT  s    z"Net.recover_input_record_by_prefixc                 C   s   | j d ks4| r,t| t| j  ks4td| D ]"}| |s<td||  q<| D ]}|| jkrh| 	| qh|| _ d S )NzOutput schema cannot be resetz{} is not defined in net {})
r  r  r6   r  r1   r  r   rU   r  r  )ry   r  r   r-   r-   r.   set_output_record]  s     

zNet.set_output_recordc                 C   s    t | jj|}|r| | dS )z
        Tries to recover out record by taking a subset of external_outputs with
        a given prefix name and interpreting them as schema column names
        N)r  r  r  r   r  r-   r-   r.   recover_output_record_by_prefixl  s    z#Net.recover_output_record_by_prefixc                 C   sv   ddl m} | jd k	std| D ]}| |s&td|q&| D ]}| | qL| j|||f | _d S )Nr   r  z(Tried to append to missing output recordr  )	r  r  r  r1   r  r  r   r  Struct)ry   
field_namer  r  r   r-   r-   r.   AppendOutputRecordFieldu  s    zNet.AppendOutputRecordFieldc                 C   s   | j S r5   )r  r|   r-   r-   r.   r    s    zNet.input_recordc                 C   s   | j S r5   )r  r|   r-   r-   r.   output_record  s    zNet.output_recordc                 G   s
   | j | S r5   )r  )ry   r   r-   r-   r.   AddExternalInputs  s    zNet.AddExternalInputsc                 G   s   | j |  d S r5   r  r  r-   r-   r.   AddExternalOutputs  s    zNet.AddExternalOutputssumc                 C   s   t |tst| j|jgddd\}}| dkrF| |j|gd}n.| dkrf| |j|gd}nt	d
|t||dS )	Nr   Z
SparseHashrB   r(  r   Zmeanz{} is not supportedr)  )rs   rn   r1   ZUniquero   lowerZUnsortedSegmentSumrp   ZUnsortedSegmentMeanr   r   )ry   r   Z
aggregatoruniqueZ	remappingZnew_gr-   r-   r.   DeduplicateGradientSlices  s    zNet.DeduplicateGradientSlicesc                 C   sz   t  }tj|_||_| j| |r8| jD ]
}d|_	q,| jD ]6}|j
dkrNq>|jD ]}|jdkrTt|j|| qTq>d S )NCUDNNr  r|  )r   rI   r   ZGpuDeviceTyperG   rJ   rW   r   rV   rC   r   r   r2   r   _RunAllOnGPUre  )rY   gpu_id	use_cudnnrW   rV   r   r-   r-   r.   r-    s    




zNet._RunAllOnGPUc                 C   s   |  | j|| dS )z3A convenient function to run everything on the GPU.N)r-  r  )ry   r.  r/  r-   r-   r.   RunAllOnGPU  s    zNet.RunAllOnGPUc                 C   s"   t  }t j|_| jj| dS )z5A convenient function to run everything using MKLDNN.N)r   rI   ZMKLDNNrG   r  rW   r   ry   rW   r-   r-   r.   RunAllOnMKL  s    zNet.RunAllOnMKLc                 C   s"   t  }t j|_| jj| dS )z4A convenient function to run everything using IDEEP.N)r   rI   ZIDEEPrG   r  rW   r   r1  r-   r-   r.   RunAllOnIDEEP  s    zNet.RunAllOnIDEEPc                    s  t |}|D ](}|s| ks*t| q|dkrLj d}n$t|tkrp fddt|D }t |d}t	 ||f|}
|g t tj  tjjd < t|jdkrdS t|jdkrt|jd S tfdd	|jD S dS )
zDA helper function to create an operator and add it to self.
        Nr  c                    s   g | ]}j  |d qS ))r  r  )r  r^   r   r-   r.   r     s   z+Net._CreateAndAddToSelf.<locals>.<listcomp>r   r   r   c                 3   s   | ]}t | V  qd S r5   r#  r   r|   r-   r.   r,    s     z*Net._CreateAndAddToSelf.<locals>.<genexpr>)r   r  r   r1   r  r  r   r   re   r   r  _extract_stacktracer   r  r  rf   r  rV   rX   rq   r   )ry   rE   r   r   r   r]   rV   r-   r   r.   _CreateAndAddToSelf  s.    
zNet._CreateAndAddToSelfc                    sb     drtd t sTt dsTtd  d d dtj  d  fd	d
S )Nr   r   r,  r   r   r   r   r   c                     s   j  f| |S r5   )r5  r   r   r-   r.   r     s   z!Net.__getattr__.<locals>.<lambda>)	r   r   r   rF   rD   r   r   r=   r   r   r-   r   r.   r     s    

zNet.__getattr__c                 C   s6   t   dd tD }ttttt| t| j|S )Nc                 S   s   g | ]}d |kr|qS )r   r-   r   r-   r-   r.   r     s   zNet.__dir__.<locals>.<listcomp>r   r   r-   r-   r.   r     s    
zNet.__dir__c           
         s   t dstdd }||}||}	|r<| |	 ks<tdi  |r^| d< |	 d< | d< nt||||d d	< pxg pg  fd
dS )a-  
        Registers and returns a python operator.

        `f` and `grad_f` can be one of the following:
            - a function with signature (inputs, outputs), where inputs and
              outputs are a list of CPUTensor objects. This function will be
              called from C++ everytime the operator is executed.
            - a tuple (func, args, kwargs), here `func` is a callable, args is
              an argument list, and kwargs is a dict list. The call:
                  f = func(*args, kwargs)
              will be performed locally at node initialization time, on all of
              the nodes of the job, returning `f`, a callable that will be used
              as the python operator function to be called during Net execution.
              This is to be used when using python operator in a distributed
              context, and allows to create and keep local python state across
              calls to the operator.

        `python_func_type` is a type of an object that constructed as
        python_func_type(f) and provides an implementation to forward and
        backward functions. Its useful in such a case where users needs
        a statefull PythonOp (ex: use autograd for computing grad_f).

        If `pass_workspace` is True, the signature is changed to
        (inputs, outputs, workspace) where `workspace` is the workspace the op
        is going to run on. This is potentially dangerous (as the op can
        manipulate the workspace directly), use on your own risk.

        If a gradient function is specified (`grad_f`), by default its inputs
        will be: (1) all inputs to `f`, (2) followed by all outputs of `f`, (3)
        and then all gradient outputs of `f`. The outputs of `grad_f` will be
        (by default) all gradient inputs to `f`. If a subset of the gradient
        outputs or gradient inputs is desired instead, then the subsets can be
        specified by providing `grad_output_indices` and/or `grad_input_indices`
        which identify the indices of `f`'s inputs and outputs which have
        gradients.
        r   c                 S   sH   t | tsdS t| dks"td| \}}}|t|t|f}t|S )Nr   r   z+Expected builder tuple (func, args, kwargs))rs   r   rf   r1   rL  pickledumps)r  r   r>   r   
normalizedr-   r-   r.   make_builder#	  s    

z Net.Python.<locals>.make_builderz9A tuple has to be passed to both f and grad_f or neither.Zpickled_builderZpickled_grad_builderr   r   r   c                     s,   j d| dttt|t S )Nr   )grad_output_indicesgrad_input_indices)r   )r5  rL  r	   r   r   Zcore_kwargsr;  r:  ry   r-   r.   r   <	  s    zNet.Python.<locals>.<lambda>)rF   r1   r   )
ry   r   r   r   r   r:  r;  r9  Z	f_builderZgrad_f_builderr-   r<  r.   r     s*    -
   
z
Net.Pythonc                 C   s    | j r|   t|}|| jkS r5   )r  r  rv   r  r  r-   r-   r.   r  D	  s    zNet.is_external_inputc                 C   s
   |  |S r5   )r  )ry   r  r-   r-   r.   
extend_opsK	  s    zNet.extend_ops)F)N)r  TF)NN)NNNTF)N)N)r   )r   )NN)r   )r(  )r   F)r   F)N)NNFNN)Cr&   r'   r(   r6   r  Zoperator_registry_staticmethodr  r  rz   r  r  r  r  r  r  r   r  r  r  r  r  r  r  rU   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  propertyr  r  r  r  r   r!  r$  r  r%  r&  r'  r+  r-  r0  r2  r3  r5  r   r   r   r  r=  r-   r-   r-   r.   r     s   


D




     
e
A
6
	


"

		


      
Pr   c                    s0    fdd| j D }| j d d = | j | d S )Nc                    s   g | ]}  ||qS r-   rD  r  blob_name_remappingr-   r.   r   P	  s     zremap_input.<locals>.<listcomp>)r]   r<   )rV   rA  r  r-   r@  r.   r  O	  s    r  c                    s   t j}tj}t j}j|kr2 j|kr2d S |rZ|rZj jkrJd S  fdd}|S |rx j|krxfdd}|S j|kr|r fdd}|S td f d S )Nc              
      s,   t   | j||W  5 Q R  S Q R X d S r5   )DeviceScopeZCopyrY   r>   kwdstr-   r.   funa	  s    
z&copy_func_between_devices.<locals>.func              
      s,   t   | j||W  5 Q R  S Q R X d S r5   )rB  ZCopyGPUToCPUrC  )srcr-   r.   rG  g	  s    
c              
      s,   t   | j||W  5 Q R  S Q R X d S r5   )rB  ZCopyCPUToGPUrC  rE  r-   r.   rG  m	  s    
z Non-supported devices: %s and %s)r   rM   rH   rG   rJ   r   )rH  rF  rM   Z
is_src_gpuZ
is_dst_gpurG  r-   )rF  rH  r.   copy_func_between_devicesU	  s"    

rI  c                 C   s   | j |j ko| j|jkS )z
    We are using this fucntion instead of == operator because optional-value
    comparison between empty device_options and {device_type:0, device_id:0}
    returns not equal in some cases.
    )rG   rJ   )rH  rF  r-   r-   r.   device_equalu	  s    rJ  c                 C   sV   g }| j D ].}||kr.|| jtjkr.|d7 }|| q
| j dd= | j | dS )zm
    Placeholder ops (for e.g. Recv) always runs on CPU. So ensure their
    output blobs reside on CPU.
    _cpuN)rX   rG   r   rM   rj   r<   )rV   blob_to_devicer   rX   r-   r-   r.   update_placeholder_op_output~	  s    
rM  c                   @   s$   e Zd Zdd Zdd Zdd ZdS )
RemapEntryc                 C   s   || _ || _d S r5   r   device)ry   r   rP  r-   r-   r.   rz   	  s    zRemapEntry.__init__c                 C   s   | j |j ko| j|jkS r5   rO  r~   r-   r-   r.   r   	  s    zRemapEntry.__eq__c                 C   s   t | jt| j S r5   )r{   r   rv   rP  r|   r-   r-   r.   r}   	  s    zRemapEntry.__hash__N)r&   r'   r(   rz   r   r}   r-   r-   r-   r.   rN  	  s   rN  c              
      s*  | j | jjd dd}|jjdd= |dkr0i }|dkr<i }i  | jjpNt }tt}|	 D ]\}}||j
 | q`g }	|jjD ]}
|	||
pg  q|jj|	 | jjD ]h}   d}d}|dk	r|j|krt|\}}nt|\}}t||jD ]\}}
| |
s&td|
|
|krT| |
rF|||
< ntd|
t||
 |st|
||kr||t|
|  |kr|t|
|  |
< nJt||
 |}dd }||
|}|||
| ||t|
|< | |
< |||< q|dk	r|j|krt|| t||jD ]H\}}||kr||jkrt|| |std	||| ||qt  }|!|  fd
d|jD }|jdd= |j| t|j}t"|jD ]@\}}z|#|}|j| |j|< W n t$k
r   Y nX q|%dd t||jD  |&|g q||fS )aE  
    Injecting Copy functions between device within a net. Users can provide
    a net with part of operators using different device_options. This method
    will automatically create a new net with Copy ops inserted in it.

    Inputs:
      blob_to_device: If not None, it is a map of blobs and their device locations.
      blob_remap: If not None, it is a map from a pair (blob, device) to
                  the name of the blob in the given device. Blobs found in this
                  map are assumed to be cached and don't need to be copied.
    Outputs:
      new_net: A new net with CopyCPUToGPU inserted with correct device option

      required_external_to_device:
               A mapping between unresolved external inputs and their
               required device options.
    Assumptions:
      1. every external inputs of this net is already in blob_to_device!
      2. if not, this function will use net device option
      3. InferOpBlobDevices might fail to get the correct inference for ops like
         EnsureCPUOutput that could take in input from multiple places.
    Z_cross_deviceTr  Nz&input {} should be defined in the net.z(No device information found for blob {}.c                 S   sH   t j}|j|krd}n*t|jr0dt|j }ntd|j| | S )NrK  Z_gpu_zUnknown device type: {})r   rM   rG   rH   rv   rJ   r   r   )r   rW   rM   r  r-   r-   r.   _gen_new_name	  s    

z.InjectCrossDeviceCopies.<locals>._gen_new_namezuIn-place blob: {} is not supported between operators with different device option previous:{} now: {}. Failed op:
 {}c                    s   g | ]}  ||qS r-   rD  r  Z
temp_remapr-   r.   r   
  s     z+InjectCrossDeviceCopies.<locals>.<listcomp>c                 S   s   i | ]\}}||qS r-   r-   )r_   r2  r   r-   r-   r.   rb   
  s      z+InjectCrossDeviceCopies.<locals>.<dictcomp>)'r  r  r2   rV   rW   r   rI   r   r   r/   r   rj   rk  r<   r   clearr   rm   rd   rI  r]   r  r1   r   r  r   rJ  rN  rI  rM  rX   r   r   r   r   r&  r   rN  r=  )rY   rL  rz  ZplaceHolderOpsr  Z
net_optionZ
all_remapsentryZmapped_blobZmapped_external_inputsr]   rV   rk   rl   devZ	copy_funcrQ  new_namerX   r  r  Zoriginal_inputsr`   r'  Z	input_idxr-   rR  r.   InjectCrossDeviceCopies	  s    

 


   



rW  c                 C   s~   t | tstdt| tdd | D s@tdt| |pFi }i }g }| D ] }t|||d\}}|| qT||fS )a-  
    Takes in a list of nets. They usually represent your whole execution graph.
    This function will insert cross device copy functions to all nets, and resolve
    inter-net external inputs dependencies. This method will insert Copy funcitons if
    external inputs of a net is produced on different device than it is required.
    Inputs:
      nets: a list of nets
    Outputs:
      new_nets: a list of new nets with device difference solved.

    Some notes from wyiming:
      1. You MUST pass nets in execution order. e.g. [train_init, train]
    z!nets {} should be a list of nets.c                 s   s   | ]}t |tV  qd S r5   rs   r   )r_   rY   r-   r-   r.   r,  3
  s     z.InjectDeviceCopiesAmongNets.<locals>.<genexpr>)rL  rz  )rs   r   r1   r   rv   r4  rW  rj   )netsblob_to_device_initrL  rz  new_netsrY   r  r-   r-   r.   InjectDeviceCopiesAmongNets#
  s"    
r\  c                 C   s   t | |\}}|S r5   )r\  )rY  rZ  r[  r  r-   r-   r.   %InjectDeviceCopiesAmongNetsWithoutB2DE
  s    r]  c                 C   s.   t | tr|  jS t | tjr&| jS | S d S r5   )rs   r   rU   r2   r   rj  )Znetliker-   r-   r.   get_net_nameJ
  s
    

r^  c                 C   s.   t | tttfkstt| tr&| gS t| S )aE  
    Ensures that the output of an operator is a list.
    Use when an operator has a variable number of outputs, but a list of
    outputs is desired even when number of outputs is 1.

    Args:
        op_output: Either a BlobReferenece or an iterable of BlobReferences.

    Returns:
        A list of BlobReferences.
    )r   r   r   rq   r1   rs   )Z	op_outputr-   r-   r.   output_to_listS
  s
    r_  c                 C   sV   t |}|| kr8| | d ks4|| | ks4td| dS t|trF|nd | |< dS d S )NzDifferent nets with same name: FT)r^  r1   rs   r   )Znet_dictrY   r2   r-   r-   r.   _add_net_to_dicte
  s    r`  c                   @   s   e Zd Ze Zedd Zd2ddZdd Zdd	 Z	d
d Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zed0d1 ZdS )3ExecutionStepc                 C   sD   | }d}|t jkr,| d t| }|d7 }qt  jt|gO  _|S )Nr   r  )ra  _step_names_usedrv   r6   r  r-   r-   r.   _get_next_step_names
  s    

z!ExecutionStep._get_next_step_nameNc                 C   s   t  | _|ptd| j_t | _d| _g | _|d k	rtt	|t
krJ|g}|D ]$}t| j|rN| jjt|g qN|d k	r|| j_d S )NstepF)r   ra  _steprc  r2   r   	_net_dict_is_used	_substepsr   r   r`  networkr<   r^  num_iter)ry   r2   rY  rj  rY   r-   r-   r.   rz   }
  s    
zExecutionStep.__init__c                 C   s
   | j | S r5   )rf  r  r-   r-   r.   get_net
  s    zExecutionStep.get_netc                 C   s   | j jS r5   re  r2   r|   r-   r-   r.   r  
  s    zExecutionStep.Namec                 C   s   | j jS r5   rl  r|   r-   r-   r.   r   
  s    zExecutionStep.__str__c                 C   s   | j rtdd S )Nz@Cannot mutate a step that has already been added to a plan/step.)rg  r1   r|   r-   r-   r.   _assert_can_mutate
  s    z ExecutionStep._assert_can_mutatec                 C   s
   d| _ d S r	  )rg  r|   r-   r-   r.   _notify_is_used
  s    zExecutionStep._notify_is_usedc                 C   s   | j S r5   )re  r|   r-   r-   r.   rU   
  s    zExecutionStep.Protoc                 C   s   | j jd k	ot| j jdkS r:  )re  ri  rf   r|   r-   r-   r.   HasNets
  s    zExecutionStep.HasNetsc                 C   s   | j jd k	ot| j jdkS r:  )re  substeprf   r|   r-   r-   r.   HasSubsteps
  s    zExecutionStep.HasSubstepsc                 C   s   t t| jS r5   r   r   rf  r|   r-   r-   r.   Nets
  s    zExecutionStep.Netsc                 C   s   | j S r5   )rh  r|   r-   r-   r.   Substeps
  s    zExecutionStep.Substepsc                 C   s   |    || j_d S r5   )rm  re  rj  )ry   rj  r-   r-   r.   SetIter
  s    zExecutionStep.SetIterc                 C   s   |    || j_d S r5   )rm  re  create_workspace)ry   rv  r-   r-   r.   SetCreateWorkspace
  s    z ExecutionStep.SetCreateWorkspacec                 C   s   |    || j_d S r5   )rm  re  num_concurrent_instances)ry   rx  r-   r-   r.   SetNumConcurrentInstances
  s    z'ExecutionStep.SetNumConcurrentInstancesc                 C   s   |    || j_d S r5   )rm  re  	only_once)ry   rz  r-   r-   r.   SetOnlyOnce
  s    zExecutionStep.SetOnlyOncec                 C   s4   t |tstdt||   t|| j_d S )Nz"expects BlobReference here, got {})	rs   rq   r1   r   r   rm  rv   re  should_stop_blob)ry   r|  r-   r-   r.   SetShouldStopBlob
  s
    zExecutionStep.SetShouldStopBlobc                 C   s   || j _dS )a	  
        Run this step every interval millisecods, as long as its
        siblings are still running. It is guaranteed that, after all
        siblings finish, this step will run at least one.

        This property is ignored for top-level ExecutionSteps.
        N)re  run_every_ms)ry   intervalr-   r-   r.   RunEveryMillis
  s    zExecutionStep.RunEveryMillisc                 C   s,   |    t| j| t|| j_|| j_dS )z) DEPRECATED. Use RunEveryMillis instead. N)rm  r`  rf  r^  re  
report_netreport_interval)ry   r  r  r-   r-   r.   SetReportNet
  s    zExecutionStep.SetReportNetc                 C   s   |    |  rtdt|trn|  | s>| s>| S | D ]}t| j	| qF| j
| | }n|}| jj | | S Nz&Cannot have both network and substeps.)rm  ro  r1   rs   ra  rn  rq  rs  r`  rf  rh  rj   rU   re  rp  r   r   )ry   rp  rY   rl  r-   r-   r.   
AddSubstep
  s    

zExecutionStep.AddSubstepc                 C   s$   |    |  rtd|| j_d S r  )rm  ro  r1   re  concurrent_substeps)ry   r  r-   r-   r.   SetConcurrentSubsteps
  s    z#ExecutionStep.SetConcurrentSubstepsc                 C   sJ   |    |  rtdt|ts&tt| j| | jj	t
|g | S r  )rm  rq  r1   rs   r   r`  rf  re  ri  r<   r^  )ry   rY   r-   r-   r.   AddNet
  s    zExecutionStep.AddNetc                    s    fddt | jD S )z
        Return the list of all attributes under the given `name`, present in
        all of the nets used in this execution step and its children.
        c                    s    g | ]}|  D ]}|qqS r-   r  r_   rY   attrr   r-   r.   r   
  s    z4ExecutionStep.get_all_attributes.<locals>.<listcomp>r   rf  r  r-   r   r.   get_all_attributes
  s    
z ExecutionStep.get_all_attributesc                 C   s  t |tjstt|jdkr,t|jdksLt|jdkrHt|jdksLtg }t|jdkr|jD ]}|t||| qdnT|jD ]L}||kr||kstt	|| }|||< || }t |t	st|| q|
dr|jnd}|
dr|jnd}	|
drt|jnd}
|
dr(|jnd}|
dr>|jnd}|
drT|jnd}|
d	rj|jnd}t|j||dd|	|
||||d
S )zN
        Create ExecutionStep from ExecutionStep protobuf recursively
        r   rj  Nr  r|  rz  rx  rv  r~  )	rj  r  r  r  r|  rz  rx  rv  r~  )rs   r   ra  r1   rf   ri  rp  rj   create_from_protor   ZHasFieldrj  r  rq   r|  rz  rx  rv  r~  execution_stepr2   )rS  
step_protonet_obj_dictnet_proto_dictsteps_or_netsZsubstep_protoZnet_namerY   rj  r  r|  rz  rx  rv  r~  r-   r-   r.   r  
  sx    
  





zExecutionStep.create_from_proto)NN)r&   r'   r(   r6   rb  r>  rc  rz   rk  r  r   rm  rn  rU   ro  rq  rs  rt  ru  rw  ry  r{  r}  r  r  r  r  r  r  ra  r  r-   r-   r-   r.   ra  p
  s6   
	

ra  c                 C   s^   |   }|  D ]}t|| q|jD ]}||kr&|| q&|jrZ|j|krZ||j d S r5   )rU   rt  add_nets_in_orderri  rj   r  )rd  net_listrl  rp  rY   r-   r-   r.   r  0  s    
r  c                   @   sX   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
edd ZdS )Planc                 C   sZ   t  | _t | _g | _t|tr:| | j_	| 
| nt|trN|| j_	ntdd S )Nz.name_or_step must be a string or ExecutionStep)r   PlanDef_planr   rf  _stepsrs   ra  r  r2   AddStepr   r   )ry   Zname_or_stepr-   r-   r.   rz   @  s    



zPlan.__init__c                 C   s   | j jS r5   )r  r2   r|   r-   r-   r.   r   L  s    zPlan.__str__c                 C   s   | j S r5   )r  r|   r-   r-   r.   rU   O  s    z
Plan.Protoc                 C   s>   |D ]4}t | j|rt|ts"t| jj |	  qd S r5   )
r`  rf  rs   r   r1   r  ri  r   r   rU   )ry   rY  rY   r-   r-   r.   AddNetsR  s    zPlan.AddNetsc                 C   s   t t| jS r5   rr  r|   r-   r-   r.   rs  X  s    z	Plan.Netsc                    sv   t  tst     s*  s*d S | jj 	 
  | j  g }t | |  fdd|D  d S )Nc                    s   g | ]}  |qS r-   )rk  rd  rd  r-   r.   r   e  s     z Plan.AddStep.<locals>.<listcomp>)rs   ra  r1   rn  ro  rq  r  r  r   r   rU   r  rj   r  r  )ry   rd  r  r-   r  r.   r  [  s    
zPlan.AddStepc                 C   s   | j S r5   )r  r|   r-   r-   r.   Stepsg  s    z
Plan.Stepsc                    s    fddt | jD S )z
        Return the list of all attributes under the given `name`, present in
        all of the nets used in this plan.
        c                    s    g | ]}|  D ]}|qqS r-   r  r  r   r-   r.   r   o  s    z+Plan.get_all_attributes.<locals>.<listcomp>r  r  r-   r   r.   r  j  s    
zPlan.get_all_attributesc                 C   s   t |tjstt|j}|j| |jjd d = |jj	d d = i }i }|jD ]}|j|ksbt|||j< qP|j	D ]}t
|||}|| qt|S r5   )rs   r   r  r1   r  r2   r  r   ri  r  ra  r  r  )rS  Z
plan_protoZplanr  r  Z	net_protor  rd  r-   r-   r.   r  u  s$    


  zPlan.create_from_protoN)r&   r'   r(   rz   r   rU   r  rs  r  r  r  ra  r  r-   r-   r-   r.   r  >  s   r  c                 C   sX   ddl m} t| tr| S d }|s2t| dr2| j}t| |rJ| j}|  } t|| |dS )Nr   r  r2   )r|  )	r  r  rs   ra  hasattrr2   Z
_stop_blobr   r  )Zstep_or_netsdefault_namer  Z	stop_blobr-   r-   r.   to_execution_step  s    

  r  c                 C   s\  |dks|dkst d|dkr,|dkr,d}t| }|dk	rF|| |dk	rX|| |dk	rj|| |dk	r||| |dk	r|dk	st ||| |dk	r|| |	r|d |
r|	|
 t
|tr|| nxt
|tr|| nbt
|trJtdd |D r.|D ]}|| qn|D ]}|t| q2n|rXtd|S )a  
    Helper for creating an ExecutionStep.
    - steps_or_nets can be:
      - None
      - Net
      - ExecutionStep
      - list<Net>
      - list<ExecutionStep>
    - should_stop_blob is either None or a scalar boolean blob.
      - This blob is checked AFTER every substeps/subnets.
      - If specified and true, then this step will return immediately.
      - Be sure to handle race conditions if setting from concurrent threads.
    - if no should_stop_blob or num_iter is provided, defaults to num_iter=1
    Nz.Cannot set both should_stop_blob and num_iter.r   Tc                 s   s   | ]}t |tV  qd S r5   rX  r$  r-   r-   r.   r,    s     z!execution_step.<locals>.<genexpr>z@steps_or_nets must be a step, a net, or a list of nets or steps.)r1   ra  r}  ru  r{  r  r  ry  rw  r  rs   r  r   r  r   r4  r  r   )r  r  rj  r  r  r  r|  rz  rx  rv  r~  rd  r%  r-   r-   r.   r    sL    








r  c                 O   s    | rt | n| }t|f||S )z=Same as execution_step() except that the step name is scoped.)r   r  )r2   r>   r   r  r-   r-   r.   scoped_execution_step  s    r  c                  C   s8   g } t d}|r4| |jj|j|jjf |j}q| S )aH  
    This function extracts stacktrace without file system access
    by purely using sys._getframe() and removes part that belongs to
    this file (core.py). We are not using inspect module because
    its just a wrapper on top of sys._getframe() whose
    logic is based on accessing source files on disk - exactly what
    we are trying to avoid here. Same stands for traceback module

    The reason for file system access avoidance is that
    if code is located on an NFS, file access might be slow

    Function returns a list of tuples (file_name, line_number, function)
    r   )sys	_getframerj   f_codeco_filenamef_linenoco_namef_back)resultframer-   r-   r.   r4    s    
r4  )T)r   NNNN)TT)N)r   NNNNN)NNF)NFN)N)NNT)r   )NNN)N)N)N)	NNNNNNNFN)tcollectionsr   r   r   Zpast.builtinsr   Zfuture.utilsr   r   r   	itertoolsr	   sixr
   r   r   Zcaffe2.protor   r  r   r   r   Zcaffe2.python.lazyr   Zcaffe2.python.control_ops_gradr   r   r   r   Z!caffe2.python._import_c_extensionpythonZ_import_c_extensionr=   r  r6  Znumpyr  r  r   r   platformZregistered_dbsprintrB  r  r   r4   r7   r8   r9   r;   r?   r@   rF   rD   rH   rI   rT   r\   rg   rd   rm   rn   objectrq   r   r   r   r   r   r   r   r   r   r   r   rG  rU  rc  ro  rq  rv  r  r  r  r~  r  r  r  r   r  rI  rJ  rM  rN  rW  r\  r]  r^  r_  r`  ra  r  r  r  r  r  r4  Zset_per_op_engine_prefZSetPerOpEnginePrefZset_global_engine_prefZSetGlobalEnginePrefZset_engine_prefZSetEnginePrefZset_op_engine_prefZSetOpEnginePrefr-   r-   r-   r.   <module>   s"  
     

r
$      
@     
   

       ob
)	  
>
       , 	  
 
"
	 AM
         
B