U dA@sddlZddlZddlZddlZddlmZmZddlm Z ddl Z ddl Z ddl m Z mZddlmmZe dZee jedddd gZd[d d Zd\ddZddZd]ddZddZddZddZddZ ddZ!ddZ"d d!Z#d"d#Z$d$d%Z%d&d'Z&d^d(d)Z'd*d+Z(d,d-Z)d.d/Z*d0d1Z+d_d2d3Z,d4d5Z-d`d6d7Z.dad8d9Z/d:d;Z0dd?Z2ed@dAdBdCgZ3dDdEZ4dFdGZ5GdHdIdIe j6Z7dJdKZ8e&de7j9fdLdMZ:dNdOZ;dPdQZdWdXZ?dYdZZ@dS)bN) workspacecore) caffe2_pb2) viewitems viewvaluesZmemonger LiveRangedefinedusedsizeFc sfddfdd}tddkr>ds>d7t|}g} t|j} |jD]0} | j D]$} | d| j krr| | krr| | qrqht| d d } g} t |jD]\}} || r| |qt}|jD]>} t | j t | j D]$} | s|r| | kr|| qqt}t|d d |D| td d|Dd|d krltn||d kr|in|}tdt|t}||t||stdt||std|S)a$ Implements similar optimization as Torch's shareGradInput(): for the gradients that are passed between layers, share blobs between operators when possible. This yields significant memory savings with deep networks. Returns an optimized protobuf (assign to net._net) cs2t|}|do0|s*|do0|kS)NZ_grad_)strendswith startswith)bname) namescope param_grads:/tmp/pip-unpacked-wheel-ua33x9lu/caffe2/python/memonger.py is_grad_blob)s  z&share_grad_blobs..is_grad_blobcs,t|jt|jD]}|rdSqdS)NTF)listinputoutput)opr)rrr is_grad_op0sz$share_grad_blobs..is_grad_opz4NOTE: Executing memonger to optimize gradient memory/_wNcSsg|]}t|dqSutf-8r encode.0srrr Xsz$share_grad_blobs..css|]}t|dVqdSr Nr!r#rrr Zsz#share_grad_blobs..r )Memonger memory optimization took {} secs(Memonger graph is not equal to original.+Inplace assignments differ in memonger net.)logwarnr copydeepcopyProtosetexternal_outputrrrappend enumerateraddtimeC'memonger_compute_blob_recycling_for_dagSerializeToStringr"infoformatrNetDefParseFromStringverify_graph_equalityAssertionErrorverify_inplace_blobs)netZlossesrrZdont_share_blobsZshare_activationsZ blob_shapesrnetprotoZ activationsr2rrZgrad_op_indicesidxZ shared_blobs start_time optim_stroptimr)rrrrshare_grad_blobssX        rGrc st|}t|jt|jfdd}t}t}t|j}ddt|jD}|D]t} | j D],} || rx| | | |krxt d | qx| j D]} || r| | q|t| j }| jrnt dqnt} t|dd|D|tdd |D|d ti} td t| t} | | t|| sht d t|| st d | S)Ncs|ko|kSNr)rexternal_inputr2rris_activation_blobrsz6optimize_inference_for_dag..is_activation_blobcSsg|] \}}|qSrr)r$indexrrrrr&xsz.optimize_inference_for_dag..z{} not in external inputzCYou can only pass inference-only nets to optimize_inference_for_dagcSsg|]}t|dqSrr!r#rrrr&scss|]}t|dVqdSr'r!r#rrrr(sz-optimize_inference_for_dag..r r)r*r+)r.r/r0r1rJr2rrr4rr5r?r;runionZis_gradient_opr6r7r8r9r"r,r:rr<r=r>r@)rA input_blobsrrBrKZactivation_blobsZseen_as_outputopsZ op_indicesrrrDrErFrrIroptimize_inference_for_dagmsV        rPcsbddltjjdtjjdtjjdtjjdtjjdtjjdtjj dtjj dtjj dtjj di fddfdd }fd d |D}t d d }d}d}d}t} |D]} | jD]} | jdks| jdkr| jD]"} | | kr||| 8}| | qq| jD]N} | | kr|| }||7}||7}t||}| | || j|7<qqq|||fS)Nrcs0fdd|jD}|jdd=|j||S)Ncs$g|]}|jks|jdkr|qS)>FreeAlias) device_optiontype)r$r devicescoperrr&s z.split_net..)rextend)protorOrYrr split_nets  z(estimate_memory_usage..split_netcsB|ks|kr$td|dS|}||S)NzUnknown blob encountered: {}r)r,warningr;prod)blobsizeof)npshapessizeofstypesrr num_bytess  z(estimate_memory_usage..num_bytescsg|] }|qSrr)r$r\)r]rrr&sz)estimate_memory_usage..cSsdSNrrrrrrz'estimate_memory_usage..rUrV)ZnumpyrZ TensorProtoZDOUBLEFLOATZFLOAT16ZINT32ZINT8ZUINT8ZUINT16ZINT16ZBOOLZINT64 collections defaultdictr1rrXrremovemaxr5)protosrcrerZrfZ allocs_by_opsZcurrent_allocatedZ max_allocatedZtotal_allocatedZ allocatedr\rornbytesr)rZrbrcrdr]rerestimate_memory_usages\         rrc Cs6t}t}t}t|}|jD]d}|jdkrB||jdq"|jD]}||qH|jD]&}||kr^|dksz||r^||q^q"|t|j}| |}||}||}t |j} t t dt |jD]J} |j| }|jD]4}||kr||| | dtd|g|gqq|jdd=|j| |S)a Insert Free-ops after a blob has been used the last time, so that its memory can be reclaimed. Use this only with efficient caching memory managers (such as CUB, --caffe2_cuda_memory_pool=cub). Blobs used with Alias op won't be freed. @dont_free_blobs: is a set of blobs that should not be freed @selector_fun: optional lambda that return True if blob name can be released. Use for easy special filtering, like excluding blobs with "loss" in the name. Returns a new protobuffer. To use with a model, use: model.net._net = memonger.release_blobs_when_used(..) rVrNrTrU)r1r.r/rrXr5rrr2 intersectionrreversedrangelenrminsertrZCreateOperatorr[) rBZdont_free_blobsZ selector_funrNZ can_releaseZ alias_blobsrinpZoutprOjrrrrelease_blobs_when_useds8           "  rzcCs.g}|D] }t||}|s||q|S)z# Return nodes without predecessors )rZ predecessorsr3)gretcnZcur_predrrr_find_source_nodess  r~cCs.g}|D] }t||}|s||q|S)z! Return nodes without successors )r successorsr3)r{r|r}Zcur_succrrr_find_target_nodes$s  rcCsft|}t|dkstt|dkr(|St|}dd}||}|||D]}|||qP|S)NrTcSs&d}|D]}||kr|}q|d7}|S)NrTr)r{r|r}rrr_next_available_idx5s z8_add_single_target_ifneeded.._next_available_idx)rrvr?r.r/add_nodeadd_edge)r{targetsr|rZtarget_node_idxr}rrr_add_single_target_ifneeded.s   rcstfddDsttfddd}g}|}|dk r||z||r^||dnd}Wq6tk r||}Yq6Xq6tt|S)z. Get the path from nx.bellman_ford()'s output c3s|]}|dkVqdS)rNrr$x dist_listrrr(Isz_get_path..cs|SrHrrrrrrhKriz_get_path..keyNr)allr?minr3 TypeErrorrrt)Z pred_listrtargetr|currrr _get_pathEs rc Cst|}|D]\}}d|||d<qi}|D]X}tj||dd\}}t||} | d|ksftt| d|| d kst| ||<q4|S)zn Get the longest path for nodes in 'source_nodes' Find with bellman_ford() by setting weight = -1 rweight)rrrT)r.r/edgesnxZ%bellman_ford_predecessor_and_distancerr?rv) r{ source_nodesZnguvr|r}preddistpathrrr_get_longest_paths\s   rcstfddDstt}ddD}||D]8}t|dd|ddD]}||d|dq\q>dd}t||||fS) z Build a tree for given paths based on common elements. Last elements of all paths are the same, which is the root of the tree. c3s"|]}|dddkVqdS)rrNr)r$cppathsrrr(tsz_build_tree..cSsh|]}|D]}|q qSrrr$ryrrr vsz_build_tree..rrrTN)rr?rDiGraphZadd_nodes_fromzipr_compute_tree_height)rr{Znode_setrZcerootrrr _build_treeps   rcsfdd|dS)zR Compute the heights of the tree for all nodes Height of leaves are 0 csFt|}d}|r4fdd|D}t|d}|j|d<|S)Nrcsg|] }|qSrrr _get_heightrrr&sz=_compute_tree_height.._get_height..rTheight)rrrnnodes)rchildrenr child_heightsrr{rrrs z)_compute_tree_height.._get_heightNrr{rrrrrs rcs$fddfdd|S)z For each node, sort its child nodes based on the height of the nodes. Return the leaf nodes of the tree after sorting. csj|dS)Nr)r)rr{rrrsz&_sort_tree_leaves.._get_heightcslt|}|s|gSfdd|Dttt|fddd}g}|D]}||}||7}qN|S)Ncsg|] }|qSrrrrrrr&szA_sort_tree_leaves.._get_sorted_leaves..cs|SrHrrrrrrhriz?_sort_tree_leaves.._get_sorted_leaves..r)rrsortedrurv)rrorderr|coZcrr_get_sorted_leavesr{rrrsz-_sort_tree_leaves.._get_sorted_leavesrrrrr_sort_tree_leavess  rc st|}t|}t||}ttt|\}}t||}t|t|ksLtt j dkrdt ||}nt|t |}|D]6} t || } | D] } | |kr|| | qqxtfddtDt jjj|fddd}t|}t|t|jkst|S)a5 The graph 'g' may contain several source nodes (nodes without incoming edge), which could be in any order and still be a valid topological sorting result. We would like to arrange these source nodes so that the average live spans of the computed blobs are shorter. The idea is to sort the source nodes based on the length of their path to the target node so that the one with longer path is used first. This is done by: - Add a single target node if there are multiple target nodes in 'g'. - Find the longest path between each source and the target node. - Convert the longest paths to a tree with the target node being the root and source nodes being the leaves. - Sort the nodes of the tree based on the height of the tree. z2.0c3s"|]\}}|t|fVqdSrHrv)r$ir)dependency_orderrrr(sz:topological_sort_traversal_longest_path..cs|SrHrr)sort_keyrrrhriz9topological_sort_traversal_longest_path..r)rr~rrrrrrr?r __version__topological_sortr1Z descendantsr5r3dictr4Z algorithmsZdagZ lexicographical_topological_sortrvr) r{gtrZlpathstreerZsorted_sourcesr|Z seen_nodesr%descdr)rrr'topological_sort_traversal_longest_paths0      rcCstt|SrH)rrrrrrrtopological_sort_traversalsrc Cs"|stdtdd}t|D]\}}|jD]n}||j}|dkrN|}n t||}||j|d||<|rx||nd}|r|dk st ||j|d||<q2|j D]r}||j }|dkr|}n t ||}||j|d||<|r||nd}|r|dk st ||j|d||<qq$|S)Nz4Provide blob sizes to get more accurate assignments.cSstddddS)N)rr r )rrrrrrhriz compute_ranges..r r r) r,r^rkrlr4rr rn_replacer?rrr) linearized_ops blob_sizesblobsrrr`r Z blob_sizerrrrcompute_rangess2       rcCsF|d\}}||krdS|jdks6|jdks6|jdkr:dS|j|jkS)NrF)rr )candidate_range assignment static_blobsrrange_rrr is_compatibles rcCsBi}|D]4}t|dkrq|d\}}|D]\}}|||<q*q|S)NrTrr) assignmentsblob_assignmentsrZ last_blobr r`rrrcompute_blob_assignmentss    rcCs.|sdStdd|D}|dkr&dn|}|S)NrcSsg|]}|djqSrTrrrrrr& sz!_get_max_size..rn)rr|rrr _get_max_size s rcCsd}|D]}|t|7}q|Srg)r)rr|rrrrget_memory_usagesrc Cs|pg}dd|D}|D]\}}||kr,qd}d}td}|jpDd} t|D]8\} } t|| grNd}tt| | } | |krN| }| }qN|r||} | ||fq|||fgq|S)NcSsh|]}|D] }|dq qSrrrrrrrsz-compute_assignments_greedy..FrinfT)floatr r4rabsrr3) ranges_sortedZinit_assignmentsrvisitedrrassignedbest_assignmentZmin_distZcandidate_sizerCrrrrrcompute_assignments_greedys*   rcCs|rtdd|DSdS)z' Return number of blobs in assignments cSsg|] }t|qSrrrrrrr&6sz_get_count..r)sum)rrrr _get_count3srcCsBdd}dd}|sdg}|dd7<|rz|dddkrz|ddj|ddjg}td |d|d|d|pg}g}t|D]\}}||||} | dkrt|n t|| } || d|d} || | |r|dn||} t| t| t | kst | t| qt |t |ks6t |d} | S) aw Compute assignment for blobs in 'ranges_sorted' on top of 'init_assignment' using dynamic programming + recursion. ranges_sorted: blobs sorted by 'used' init_assignment: assignment to start with, blobs in 'ranges_sorted' should not be used in 'init_assignment' Using f(b, k, init) to represent the best assignment for blobs b[0:k] given initial assignment 'init', we have f(b, k, init) = f(b, j, init) + find_best(b[j:k], f(b, j, init)) where j is the index of the last best assignment that is independent of blob b[k - 1] (b[k - 1] is compatible with all assignments in f(b, j, init)), and find_best(b1, init1) gives the best assignment for blobs in 'b1' based on the initial assignment 'init1', and blobs b1[0:-1] should be incompatible with b1[-1]. f(b, len(b), []) gives the best assignment for blobs 'b'. For find_best(b, init), since b[0:-1] are not compatible with b[-1], we could reduce it to a smaller problem to find best assignment for b[0:-1] as find_best(b, init) = min { f(b[0:-1], len(b) - 1, init - x) + [x, b[-1]] for x in init, or f(b[0:-1], len(b) - 1, init) + [b[-1]] } where min{} gives the assignment with minimum memory usage. cSs<dd}|d}|dkr8||}|||r.|S|d8}qdS)z Find closest position k of best_assignments that is independent of candidate_range that candiate_range is compatible with all assignments in best_assignments[k]. Return -1 if not found. cstfdd|DS)z> return true if compatible for all assignments in assignments csg|]}td|gqSrrrrrrr&_szccompute_assignments_dp.._get_compatible_prev..is_compatible_all..)r)rrrrris_compatible_all]szOcompute_assignments_dp.._get_compatible_prev..is_compatible_allrTrrr)rbest_assignmentsZcur_idxriiZcbarrr_get_compatible_prevWs  z4compute_assignments_dp.._get_compatible_prevc s|dtfdd|ddDs*tt|}g}t|D]td|gsXq>t|}|t|dkrfddt|D}t |dd||}||g}||q>||ggt |d d d }|S) a* Find the best assignment for blobs 'ranges' given an initialized assignment 'init_assignment'. Blobs in ranges[0:-1] should be incompatible with blob range[-1]. 'prev_best_assignment': best assignment for blobs in ranges[:-1] By assigning ranges[-1] to each assignment k in 'init_assignment' or in a new assignment, the problem becomes a smaller problem to find the best assignment for ranges[0:-1] given the initial assignment init_assigment[0:k, (k+1):-1]. rc3s"|]}t|dgg VqdSrTNrr) find_rangerrr(ysz=compute_assignments_dp.._find_best..rrTcsg|]\}}|kr|qSrr)r$rr)rrrr&sz>compute_assignments_dp.._find_best..NcSst|SrH)rrrrrrhriz._find_best..r) rr?rvrurr.r/r3r4compute_assignments_dpr) rangesinit_assignmentZprev_best_assignmentcounterszZbest_candidatescur_bestZ cur_best_tmpr|r)rrr _find_bestis* "     z*compute_assignments_dp.._find_bestrrTirz$Finding assignments {} ({} -> {})...) rr r,r:r;r4r.r/rrvr?r3)rrrrrrsrrZ cur_rangeZprev_idxZ prev_bestZ ranges_partrbestrrrr:s>'  rcs8dd}dddkr ||fdd|D}|S)z Set LiveRange.defined = -1 if it is None Set LiveRange.used = max_live if it is None Set LiveRanee.size = 1 if it is None cSstdd|Dd}|S)Ncss"|]}|djr|djVqdSrrrrrrr(s z._get_max_live..rTr)rmax_liverrr _get_max_livesz)get_updated_ranges.._get_max_livecSsz|}|djdkr*|d|djddf}|djdkrP|d|dj|df}|djdkrv|d|dj|df}|S)NrTrrrrr)rrr r )rrr Zcxrrr _update_rangesz)get_updated_ranges.._update_rangeNcsg|]}|dqSrrrrrrrr&sz&get_updated_ranges..r)rrrrrrget_updated_rangess  rcstt|ddd}t|}fdd|D}fdd|D}tdt|g}|tjkrnt |g}n$|tj krt |g}nd|st |d d|D7}|S) a] algo: Method used to find assignments (AssignmentAlgorithm.GREEDY or AssignmentAlgorithm.DYNAMIC_PROGRAMMING). AssignmentAlgorithm.DYNAMIC_PROGRAMMING gives optimal solution at the cost of more computation. AssignmentAlgorithm.GREEDY may be better in the case 'blob_sizes' is not provided. cSs|djdk|djfS)NrTr)prrrrhriz%compute_assignments..rcsg|]}|dkr|qSrrrrrrr&s z'compute_assignments..csg|]}|dkr|qSrrrrrrr&s zTotal sharable blobs {}zInvalid algo name {}cSsg|] }|gqSrrrrrrr&s) rrrr,r:r;rvAssignmentAlgorithmDYNAMIC_PROGRAMMINGrGREEDYrr?)rralgoZranges_sharableZ ranges_staticrrrrcompute_assignmentss     rcCsJ|D]@}t|dd|ddD] \}}|dj|djks"tq"qdS)NrrrT)rr rr?)rrrrrrrverify_assignmentss"rcst}t|D]\}}|j||dqt|D]p\}}t|D]^\}||krRq@tfdd|jDr@tj|j}|j |||dt |s@t q@q0|S)Nrc3s|]}|jkVqdSrH)r)r$rZchild_oprrr(sz-compute_interference_graph..)deps) rrr4ranyrr1rrsrZis_directed_acyclic_graphr?)rOr{rrZ parent_opryrrrrcompute_interference_graphsr OptimizationrArrcsxfdd}|jD]`}|jdr.t||t|jD]\}}|||j|<q8t|jD]\}}|||j|<qZqdS)Ncs|kr |S|SrHr)r`rrrcanonical_namesz)apply_assignments..canonical_nameZRecurrentNetwork)rrXr apply_recurrent_blob_assignmentsr4rr)rArr rrZinput_rrr rapply_assignmentss    r cCsLtd|jdd|jD}|D].}t|jD]\}}|||j|<q4q&dd|jD}|D].}t|jD]\}}|||j|<qxqjdd|jD} | D]>} t | j |t| j j D] \}} | |kr|| | j j |<qqt |D]P\}} |t |jt |jkrt} |d| _t| d| _|j| gqdS)Nz(Applying assignments to recurrent op: {}cSsg|]}|jdr|qS) alias_dstrr r$arrrr&%s z4apply_recurrent_blob_assignments..cSsg|]}|jdr|qS) link_externalrrrrrr&+s cSsg|]}|jdr|qS)Zstep_netrrrrrr&1s z.renameascii)r,debugr;rXargr4stringsdecoder"r nrJrrrrrZArgumentrr r%r[)rrr Zalias_dst_argsr rr`Zlink_external_argsrZ step_argsZstep_argZeinpZrenamedrrrrr !s*  r c@seZdZdZdZdS)rrrTN)__name__ __module__ __qualname__rrrrrrrAsrcCs0t}t|dd|D}|||S)NcSsg|]}t|dqSrr!r#rrrr&Jsz+optimize_inference_fast..)rr<r7Zmemonger_optimize_inference_netr9r=)rArrFrErrroptimize_inference_fastFs  rc s|ttj}||}fdd|D}jdd=j|t||}t|||} t| } t| t | | dS)a_ ordering_function: topological_sort_traversal or topological_sort_traversal_longest_path. topological_sort_traversal_longest_path gives better results but needs a bit more computation. algo: Method used to find assignments (AssignmentAlgorithm.GREEDY or AssignmentAlgorithm.DYNAMIC_PROGRAMMING). AssignmentAlgorithm.DYNAMIC_PROGRAMMING gives optimal solution at the cost of more computation. AssignmentAlgorithm.GREEDY may be better in the case 'blob_sizes' is not provided. csg|]}j|qSrr)r$rrArrr&msz)optimize_interference..N)rArr) r.r/rrr[rrrr r) rArZordering_functionrrr{Zorderingrrrrrrroptimize_interferencePs       rcCsLdd}t|j|jD]0\}}|j|jkr0dS||||krdSqdS)z Verifies that net_a and net_b have the same in-place blob assignments. Particularly, that memonger did not add an in-place assignment when that did not exist before. cSsBt|j}g}t|jD]$\}}||kr||||gq|SrH)rrr4rr3rL)routZinplacesryrxrrr get_inplacess  z*verify_inplace_blobs..get_inplacesFT)rrrX)net_anet_brop_aop_brrrr@s r@c Csdd}t|jt|jkr dSt|j|jD]2\}}|j|jksZ|j|jksZ|j|jkr.dSq.||j}||j}||krd}t||D]F\}} || krtd||j||j|td|| |d7}q||kS)a Determines if the execution of two graphs are identical. That is, all inputs blobs are mapped to the same output blobs for each operator in their respective positions. This is meant to check the output of memonger with the original graph. It assumes that the nets have same external input and output. O(E) runtime + O(1) amortized cost to hash for python dict cSshdd|D}i}t|D]H\}}|jD]$}||}|dk r(|||q(|jD] }|||<qTq|S)NcSsg|]}gqSrr)r$r rrrr&sz>verify_graph_equality..parent_list..)r4rgetr3r)rO parent_listZ edge_ownerrrr`Z parent_idrrrr%s    z*verify_graph_equality..parent_listFrzDifference {} vs {} {}zParents: {} vs {}rT)rvrrrXrWZengineprintr;) r r!r%r"r#Z parent_list_aZ parent_list_bryrrrrrr>s0       r> Statisticsbaseline_nbytesoptimized_nbytescCs>d}zt|j}Wn$tk r8td|YnX|S)NrzError when fetching blob {})rZ FetchBlobrq Exceptionr,r^r;)r`rrrr blob_nbytess r+cs<dd|Dtt}tfdd|D}t||dS)NcSs$i|]}|D]\}}|t|q qSr)r+)r$rr`r rrr s z&compute_statistics..c3s$|]}tfdd|DVqdS)c3s|]\}}|VqdSrHr)r$r`r Z blob_bytesrrr(sz/compute_statistics...Nr)r$rr-rrr(sz%compute_statistics..)r(r))rrr')rr(r)rr-rcompute_statisticss  r.cCsDi}|jD]4}|jD]}t|||<q|jD]}t|||<q,q |SrH)rrr+r)rArrr`rrrcollect_blob_sizess   r/)NFN)r)N)N)N)N)N)AZnetworkxrrkr6r.Z caffe2.pythonrrZ caffe2.protorenumloggingZ future.utilsrrZ!caffe2.python._import_c_extensionpythonZ_import_c_extensionr7 getLoggerr,setLevelINFO namedtuplerrGrPrrrzr~rrrrrrrrrrrrrrrrrrrrrrr r Enumrrrrr@r>r'r+r.r/rrrrs|    U 4D 5  )     x *   03