U d<@stddlZddlZddlmZmZmZddlmZddZ ddZ dd Z dd d Z Gd dde ZGdddZdS)N)core workspace net_drawer) caffe2_pb2cCstj|dd|jDS)NcSsg|] }|dqS)_grad).0srrB/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/gradient_checker.py sz$getGradientForOp..)rZGradientRegistryZGetGradientForOpoutput)oprrr getGradientForOpsrcCs^||}t|tjrtj|St|tjs.td}td|j|j |g|}t |tj|S)NZtmp_dense_gradZ SparseToDense) isinstancerZ BlobReferencerblobs GradientSliceAssertionErrorCreateOperatorindicesvaluesRunOperatorOnce)grad_mapinput_to_checkZ grad_blobZ dense_gradsparse_to_dense_oprrr _get_grad_blobs    rc s||d}|||p"iD]\}}|tj|<q(|D],}|ksZtd|t|tjks@tq@t |dd|D} fdd|D} | | |fS)NZ_copyz.{} has no gradient, cannot check net gradient.cSsg|]}|tj|fqSrrrrr rrr r 6sz_get_grad..csi|]}|t|qSr)r)rrrrr 7sz_get_grad..) ZCloneNameZAddGradientOperatorsitemsrrrformatstr RunNetOnce) netoutputsoutputs_with_grad input_valuesinputs_with_gradsZgrad_netnamevaluerZforward_resultsZgradsrrr _get_grad(s"     r+cCs:tjj|||||dt||}t|t|fS)N)atolrtolerr_msg)npZtestingZassert_allcloseabsflattenZmeanmax)Zvalue1Zvalue2 thresholdr/deltarrr _assert_close=sr6c@s(eZdZed ddZed d d ZdS) NetGradientCheckerNHz>Fc sfdd|rdt|D]J\}}t|} tdt||dd} | | W5QRXqfddt||D} |rt| \} } } t| D]J\}}t|} tdt||dd} | | W5QRXq| d \}}} | d dD]\}}} t |t |ks t tt||D]0\}\\}}\}}t |||d |||d q.| | ksvt |D]$\}}t ||||d |d q~qdS)NcsfddDS)Ncsg|] }|qSrr)ri net_outputsrr r NszWNetGradientChecker.CompareNets.._get_output_with_grad_names..rr:)outputs_with_grad_idsr:r _get_output_with_grad_namesMszCNetGradientChecker.CompareNets.._get_output_with_grad_namesZcaffe2_net_forward_z.pngwbcs$g|]\}}t|||qSr)r+)rr$r;)r=r'r(rr r Xs z2NetGradientChecker.CompareNets..Z caffe2_net_rzVDifferent forward pass results for output id {}. Corresponding output blobs: {} and {})r/z Different gradients for input {}) enumeraterZ GetPydotGraphZ create_pngopenr"rwriteziplenrr6r!keysr )netsr%r<r(r'r4Zprint_net_imagesr9r$Zpngfresults_Z backward_netsZfirst_net_resultsZfirst_net_grads net_results net_gradsidxZblob1Z blob_value1Zblob2Z blob_value2blobZblob_grad_valuer)r=r'r(r<r CompareNetsIsV  zNetGradientChecker.CompareNets-C6?皙?Tcst|gg\}}|} fdd} fdd} t} tjD]8} | | | |}| | | | }|||d| j| <qZd|}|r|d|7}t | | ||S)Ncs*|tj<ttddDS)NcSsg|]}tj|qSrrrrrr r sz=NetGradientChecker.Check..GetLoss..)rrr#sum) new_value)full_netrr&rr GetLosss   z)NetGradientChecker.Check..GetLosscs"}|j||7<|SN)copyflat)dimr5Z input_value)rr'rr GetValues z*NetGradientChecker.Check..GetValuez'Error in gradient check for net_copy {}z: {}) r+r0 zeros_likerangesizerWr!rZProtor6)r$r&r'rZ step_sizer4Z print_netrJrKZ analytic_gradrTrY grad_estimaterXpos_lossneg_lossr/r)rSrr'r&r Checkxs* zNetGradientChecker.Check)Nr8F)rOrPT)__name__ __module__ __qualname__ staticmethodrNrarrrr r7Hs .r7c@s4eZdZdZd ddZddZdd d Zd d ZdS)GradientCheckerzA gradient checker in Python. This is not the most efficient way to check gradients, as the Python interface will involve a lot of copies back and forth operations. Use at your own risk. Ngradient_checkcCs:||_||_|pt|_||_|dkr0i|_n||_dSrU) _stepsize _thresholdrZ DeviceOption_device_option_workspace_name_input_device_options)selfZstepsizer4 device_optionZworkspace_nameinput_device_optionsrrr __init__szGradientChecker.__init__c Csxtt|D]*}t|||||j|||jq ||} t|d} |D]<} |j| } t | } | | d 7} t| d| |jqR| d} t |t |t jrftdtj| tjdtdtjdtjdt jd |j|jd |jd }t jd |j|jd |jd }t d dd|jd |jd dgd}t|t|t|t d}n t |}| |fS) NgrZrg@zeros)dtypeonesr?ZEnsureCPUOutputZ_cpu)rnZScatterWeightedSum)r\rDrFeedBlobrlgetrjrr FetchBlobrQZRunOperatorsOncerrrr0r[float32rsrrr)rmr grad_opsinputsZ input_namesr grad_nameoutputs_with_gradsr9xlossrLr)arrZ gv_cpu_opZ gi_cpu_oprgradrrr GetLossAndGrads`         zGradientChecker.GetLossAndGradFc Cslt}|j|kr t|jd|j|j|dkrBt|\}} |pRt |d} t |D],\} } t |j | | | |j | |jq\| |} |||||j || |\}}t||}|j|jkrtd|j|j|r|||tddk}||j}t|D]}|sB|dkrB|d|krB|j||j|<q ||j||j7<|||||j || |\}}||j||jd8<|||||j || |\}}||j||j7<|||jd|j|<q tj|||j|jd }t|r@t|}td tt ||j||j|gj!d }nd}|j|krbt"t||||fS) aChecks the operator in a very simple fashion by stacking a sum of squares on the top. Inputs: op: the operator to be checked. inputs: the input data in numpy arrays. input_to_check: an index specifying which input blob we should check. outputs_with_grads: indices specifying which output blobs will we need to check gradients with. For these outputs, we will collect a squared sum and also feed in their gradients. grad_operator: the gradient operator. If not given, we will get the gradient operator from the gradient registry. input_device_options: an optional mapping from input names to DeviceOptions (to override the default DeviceOption) ensure_outputs_are_inferred: if set will assert that the gradient output shapes matches the inferred shapes Outputs: boolean: True if it passes, False if it does not pass. TNrz5Mismatched gradient shapes: estimated ({}), grad ({})ZCAFFE2_FULL_GRAD_CHECK1rZ)r-r.z'Failed. [idx, grad, grad_estimate] are:F)#rZCurrentWorkspacerkZSwitchWorkspacernZCopyFromrjrrZInferOpBlobDevicesAsDictr@rtinputrurr0r[shape Exceptionr!_assertInferTensorChecksosgetenvr]r\rWrhiscloserianyZ flatnonzeroprintZvstackTZResetWorkspace)rmr ryrr{rxroZensure_outputs_are_inferredZ old_ws_nameZg_inputrlr9r~rzr}rr^Zfull_grad_checkZ dims_to_checkZ current_dimr_rIr`Zfail_matrLretrrr CheckSimples          "  zGradientChecker.CheckSimplecCsft}|j|g|j|tj|gdd\}}t}|D]}||jq@|D]}||krrt d |t |} t | j } t ||} | | krt d | | t| tjkr4| jtdkrtjj} n^| jtdkrtjj} nD| jtdkr tjj} n(| jtdkr&tjj} n d tj} n tt| } ||} | | krVt d | | qVdS) NT)Z nets_protoz!expected output {} to be inferredz,Mismatched inferred shape: want({}), got({})Zfloat64rwZint32Zint64z unknown {}z+Mismatched inferred type: want({}), got({}))rZNetDefr extendrZInferShapesAndTypessetupdater rr!rvlistrtyper0ZndarrayrrZ TensorProtoZDOUBLEFLOATZINT32ZINT64r")rmr rxZtmp_netZinferred_shapesZinferred_typesr%Zgrad_opr rMZ correct_shapeZinferred_shapeZ correct_typeZ inferred_typerrr rYsX           z(GradientChecker._assertInferTensorChecks)NrgN)NNF)rbrcrd__doc__rprrrrrrr rfs  7 srf)r,)rZnumpyr0Z caffe2.pythonrrrZ caffe2.protorrrr+r6objectr7rfrrrr s  T