U
    9%eM                     @   s  d dl mZmZ d dlmZmZmZmZmZm	Z	 d dl
Z
d dlmZ d dlZd dlZd dlm  mZ d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ ddlmZmZ dd	l m!Z! d dl"Z"e!j#Z$d
d Z%dd Z&dd Z'dd Z(G dd dZ)e) Z*dd Z+dd Z,dd Z-dd Z.dd Z/ej0ddd Z1ej0dd!d"Z2ej0eej0ej0f d#d$d%Z3d&d' Z4d(d) Z5ej6e7d*d+d,Z8d-d. Z9e":dd/d0 Z;d1d2 Z<d3d4 Z=d5d6 Z>d7d8 Z?d9d: Z@dFej0eej0ej0f d#d<d=ZAdGe
jj0eBeBd@dAdBZCdHdDdEZDdS )I    )is_sym_nodepy_sym_types)hint_intmagic_methodsmethod_to_operatorfree_symbolsis_symbol_binding_fx_nodefind_symbol_binding_fx_nodesNdefaultdict)graph_drawer)Tuple   )fx_graph_cseget_aten_target)configc                 C   s   | j ddS )N	recomputeF)metagetnode r   \/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/torch/_functorch/partitioners.pymust_recompute   s    r   c                 C   s$   d}| j jD ]}t|r dS qdS )NFT)graphnodesr   )fx_gfoundr   r   r   r   has_recomputable_ops   s
    r   c                 C   s<   | j jD ].}t|rt|jdrtjj|jjkr dS qdS )NtagsTF)	r   r   r   hasattrtargettorchTagnondeterministic_seededr   )r   r   r   r   r   has_recomputable_rng_ops#   s    $r%   c                 C   s6   t | jd tjtjfrdS t | jd tjs2tdS )Nvalr      )
isinstancer   r"   SymIntZSymBoolZSymFloatAssertionErrorr   r   r   r   sym_node_size)   s    r+   c                   @   s   e Zd Zdd ZdS )InvalidNodeBasec                 C   s   dS )NzInvalid Noder   )selfr   r   r   __repr__0   s    zInvalidNodeBase.__repr__N)__name__
__module____qualname__r.   r   r   r   r   r,   /   s   r,   c           	         sx  t  }i  |D ] }||j}|j|_| |< q| jD ]}||krHq8q8|jdkr\t |< q8|jdkrt	|j
|jfd } fdd|D }t|rt |< q8|| fdd |< q8|jdkr|| fd	d |< q8|jd
kr8q8g }|D ]f}t|t jrN| krtd| dt | tr>td| d| |  q|| q|| |  |  |S )a  
    Given a graph, extracts out a subgraph that takes the specified nodes as
    inputs and returns the specified outputs.

    This includes specifying non-placeholder nodes as inputs.

    The general strategy is to initialize all inputs with proxies as we
    encounter them, and trace through the graph, only keeping values which take
    in valid proxies. Then, all dead code is eliminated.
    placeholdercall_functionr   c                    s&   g | ]}t |tjrt  | tqS r   )r(   fxNoder,   ).0xenvr   r   
<listcomp>S   s      z6_extract_graph_with_inputs_outputs.<locals>.<listcomp>c                    s    |  S Nr   r7   r8   r   r   <lambda>W       z4_extract_graph_with_inputs_outputs.<locals>.<lambda>Zget_attrc                    s    |  S r;   r   r<   r8   r   r   r=   Y   r>   outputzNode z couldn't be found in envz was invalid, but is output)r4   Graphr2   namer   r   opInvalidNodepytreetree_flattenargskwargsany	node_copyr(   r5   RuntimeErrorr,   r*   appendr?   eliminate_dead_codeZlint)	Zjoint_graphinputsoutputs	new_graphr   new_nodeZall_argsZoutput_valuesr7   r   r8   r   "_extract_graph_with_inputs_outputs7   sD    







 
rQ   c                 C   s(   | j dko&d| jko&t|  o&t|  S Nr2   tangents)rB   r!   _is_bwd_seed_offset_is_fwd_seed_offsetr   r   r   r   
_is_primall   s    
rV   c                 C   s   | j dkod| jkS rR   rB   r!   r   r   r   r   _is_tangentt   s    rX   c                 C   s   | j dkod| jkpd| jkS )Nr2   Zbwd_seedZbwd_base_offsetrW   r   r   r   r   rT   w   s    rT   c                 C   s   | j dkod| jkpd| jkS )Nr2   Zfwd_seedZfwd_base_offsetrW   r   r   r   r   rU   z   s    rU   )joint_modulec                C   s<   t dd | jjD d }|d | }||d  }||fS )Nc                 S   s   g | ]}|j d kr|jqS r?   )rB   rF   r6   r   r   r   r   r:      s     
 z,_extract_fwd_bwd_outputs.<locals>.<listcomp>r   )rD   rE   r   r   )rY   num_fwd_outputsrN   fwd_outputsbwd_outputsr   r   r   _extract_fwd_bwd_outputs~   s    r_   c                C   s   t | |d\}}ttt| jj}ttt| jj}ttt| jj}ttt| jj}	t	| j|| || | }
t	| j|| | |	 |}|jD ]\}|j
dkr|js|D ]}|j|jkr||  qq|D ]}|j|jkr||  qqqt }g }g }|D ]4}t|}|r*|| || n
|| qt| j}t|||D ]d}d|jkrdqPt|jd | }t|dd dD ]"}||krq|||  q||O }qP|  |||  t	| j|| || | }
t	| j|| | |	 |}t| |
}t| |}||fS )Nr\   r2   r&   c                 S   s   | j S r;   rA   )sr   r   r   r=      r>   z*_extract_fwd_bwd_modules.<locals>.<lambda>key)r_   listfilterrV   r   r   rX   rU   rT   rQ   rB   usersrA   removesetr   addrK   r	   	itertoolschainr   r   sortedclearextendr4   GraphModule)rY   saved_valuessaved_sym_nodesr\   r]   r^   primal_inputstangent_inputsfwd_seed_offset_inputsZbwd_seed_offset_inputsZ	fwd_graphZ	bwd_graphr   Zsaved_valueZ	saved_symZsaved_symbolsZsaved_sym_nodes_bindingZsaved_sym_nodes_derivedsymbolZsymbol_bindingsZnew_symbolsrb   Z
fwd_moduleZ
bwd_moduler   r   r   _extract_fwd_bwd_modules   sv    







rw   )rY   returnc                   s~  t | rt| ||dS ttt| jj}ttt| jj}|| }t| |d\}}t	| j||}dd |jD  g }	g }
| jjD ]}|j
 krqt|r|
| qd|jkr|jdkr|j}tdd |D st|D ]}|	| qq fdd	|jD }d|jkr6td
d |D r6|D ]}|
| q"q|	| qtdd |	D  }	tdd |
D  }
t| |	|
|dS )a  
    Partitions the :attr:`joint_module` in a manner that closely resembles the
    behavior observed in the original ``.forward()`` and ``.backward()`` of the
    callable, i.e., the resulting forward graph contains those operators that
    are executed in the original ``.forward()`` callable passed to
    :func:`aot_function`.

    The default partitioner collects the operators that are between the forward
    inputs and the forward outputs. This helps in finding the tensors which have
    to be stashed for the backward pass. These stashed tensors become the output
    of the generated forward graph. The remaining operators are then placed in
    the backward graph.

    .. warning::
        This API is experimental and likely to change.

    Args:
        joint_module(fx.GraphModule): The joint forward and backward graph. This
            is the result of AOT Autograd tracing.

    Returns:
        Returns the generated forward and backward Fx graph modules.
    r`   c                 S   s   h | ]}|j d kr|jqS rZ   rB   rA   r[   r   r   r   	<setcomp>  s     
 z$default_partition.<locals>.<setcomp>tensor_metar3   c                 s   s   | ]}|j tjkV  qd S r;   )r!   operatorgetitemr6   userr   r   r   	<genexpr>  s     z$default_partition.<locals>.<genexpr>c                    s   g | ]}|j  kr|qS r   ra   r6   nZforward_node_namesr   r   r:     s     
 z%default_partition.<locals>.<listcomp>c                 s   s   | ]}t |V  qd S r;   r   r   r   r   r   r     s     c                 S   s   i | ]
}|d qS r;   r   r6   kr   r   r   
<dictcomp>&  s      z%default_partition.<locals>.<dictcomp>c                 S   s   i | ]
}|d qS r;   r   r   r   r   r   r   '  s      rr   r\   )r   #min_cut_rematerialization_partitionre   rf   rV   r   r   rU   r_   rQ   rA   r   rK   r   rB   rg   allr*   keysrw   )rY   _joint_inputsr\   rs   ru   rM   r]   r^   forward_only_graphrq   rr   r   rg   r   Zbackward_usagesr   r   r   default_partition   s>    
 	r   c                 C   s   d}| D ]}||9 }q|S Nr   r   )r7   rb   ir   r   r   _prod,  s    
r   c                 C   s
   | |j  S r;   )itemsize)numeldtyper   r   r   _tensor_nbytes2  s    r   )r   rx   c                 C   s   d| j kr| j d }t|tr4t|tjr.dS dS n@t|ttfrTtdd |D S t|tjrtt	t
| |jS tdt| d| j kr| j d }ttt|j}|j}ndS t	||S )	Nr&   r   i?B c                 s   s.   | ]&}t |tjrtt| |jV  qd S r;   )r(   r"   Tensorr   r   r   r   r   r   r   r   r   >  s      z_size_of.<locals>.<genexpr>zUnknown metadata type r{   r   )r   r(   r   r"   r)   re   tuplesumr   r   r   r   r   rJ   typer   mapZto_size_hintshape)r   r&   metadatar   r   r   r   r   _size_of5  s"    




r   c                 C   s\   ddl m} |t}| jD ]"}|jdkr||jj  d7  < qtt|	 dd dd d S )	Nr   r
   r3   r   c                 S   s   | d S r   r   r<   r   r   r   r=   V  r>   z_count_ops.<locals>.<lambda>Trd   reverse)
collectionsr   intr   rB   r!   r/   printrm   items)r   r   Zcntr   r   r   r   
_count_opsP  s    

r   c                  C   sl   g } t tjjD ]V}ttjj|}t|tjjs2q| D ]*}t||}tj	j
|jkr:| |  qq:q| S r;   )dirr"   opsatengetattrr(   Z_opsZOpOverloadPacketZ	overloadsr#   Z	pointwiser   rK   )r   	attr_nameZopoverloadpacketoverloadZop_overloadr   r   r   pointwise_opsY  s    

r   c                    s   |  kr |  S | j dkr*d | <  |  S | j dkrf| jd }|D ]}t|tjjjrBt|  qBd S  fdd| jD }t	|dkrdg}t
|d  | <  |  S )Nr2   r   r?   c                    s&   g | ]}t |tjjjrt| qS r   )r(   r"   r4   r   r5   	get_depthr6   arg	depth_mapr   r   r:   |  s      zget_depth.<locals>.<listcomp>r   )rB   rF   r(   r"   r4   r   r5   r   all_input_nodeslenmax)r   r   rF   r   
arg_depthsr   r   r   r   j  s     


r   c                    s(    fdd| D }t | dd ddS )Nc                    s&   i | ]}t |tjjjr| | qS r   )r(   r"   r4   r   r5   r   r   r   r   r     s       zsort_depths.<locals>.<dictcomp>c                 S   s   | d S r   r   r<   r   r   r   r=     r>   zsort_depths.<locals>.<lambda>Tr   )rm   r   )rF   r   r   r   r   r   sort_depths  s    r   c                    s(  t  i | jjD ]*}|jdkr|j}|j|_||< qi }t| jjD ]\}}|||< qPi  dd | jjD d }t	|   fddt
tt| jj}d}tj}|D ](}	|	jD ]}
||
 |k r||
 }|
}qq|dk	stt
| jj|| d D ]}| qtj | }|S )a  
    This pass finds the first bwd node in the graph (by looking at users of
    tangents) and then reorders the graph by walking from this node to all the
    way to the end of the graph. At each op in this traveral, we insert this op
    in a new graph and try to bring only the relevant subgraph from the other
    non-bwd edges relevant for this op. This closely mimics the behavior of
    autograd engine.

    Why is this pass required in the first place?

    This is an artifact of how partitioners work today. The starting point of
    partitioner is a joint graph, which is fwd and then bwd graph. In the case
    of checkpointing, we keep portions of fwd graph in their original place in
    the joint graph, while obtaining a bwd graph. As a result, the resulting bwd
    graph has copies of recomputed fwd subgraphs followed by the original bwd
    graph. If we run this naively, this leads to bad memory footprint, because
    the fwd subgraphs are live for way longer duration than necessary. This pass
    reorders the operations such that we prioritize the ops for the original bwd
    graph while only realizing those ops from the fwd graph that are necessary
    at any given point in the graph.
    r2   c                 S   s   g | ]}|j d kr|qS rZ   rB   r[   r   r   r   r:     s     
 z7reordering_to_mimic_autograd_engine.<locals>.<listcomp>r   c                    sR   | kr|  S t | j D ]\}}||< q| fdd| < |  S )Nc                    s    |  S r;   r   r<   r8   r   r   r=     r>   zSreordering_to_mimic_autograd_engine.<locals>.insert_node_in_graph.<locals>.<lambda>)r   r   rI   )r   r   _Zdepthsr9   insert_node_in_graphrO   r   r   r     s    zAreordering_to_mimic_autograd_engine.<locals>.insert_node_in_graphN)r4   r@   r   r   rB   r2   rA   r   	enumerater   re   rf   rX   mathinfrg   r*   r"   rp   )Zgmr   rP   orderidxZoutput_nodert   Zfirst_node_in_bwdZminimum_ordertangentr   Znew_gmr   r   r   #reordering_to_mimic_autograd_engine  s6    




r   c               
   C   s  t  }dd }dd }dd }|| }||}	||}
t }| jjD ]T}t|rFt|jdrFtj	j
|jjkrF||j }|	|j }|
|j }||d||< qFtjjj}tjjj}|jjD ] }|jd	krd
|jkr|} qqg }| D ]0\}}|d }|d }|j}||r |jd||jf|j|jd}|jdtj|dfi d}|jdtj|dfi d}|| || || W 5 Q R X |j}||0 dt| }||}||||jd< W 5 Q R X ||: |jd|||jf|j|jd}|| || W 5 Q R X qdd |jjD d }|jd }t|| }|d | | ||d   }|j | |j| |!  |!  ||fS )Nc                 S   sF   i }| j jD ]4}|jdkrt|jdrtjj|jjkr|||j	< q|S )Nr3   r   )
r   r   rB   r    r!   r"   r#   r$   r   rA   )ZgmodZrandom_nodesr   r   r   r   get_rng_ops  s    
z*functionalize_rng_ops.<locals>.get_rng_opsc                 S   sT   d| j krdS | j d }t|ts(|f}|D ]"}t|tjr,|jjdkr, dS q,dS )zV
        Check the example value of the node outputs to find the device type.
        r&   Ncudacpu)r   r(   r   r"   r   devicer   )r   
candidates	candidater   r   r   
get_device  s    


z)functionalize_rng_ops.<locals>.get_devicec                 S   s   | dkrt j S t  S )Nr   )r"   r   Zget_rng_state)r   r   r   r   get_sample_rng_state  s    
z3functionalize_rng_ops.<locals>.get_sample_rng_stater   )fwdbwdr2   r   r   r   r3   )rF   rG   r   r   Zrng_state_output_r&   c                 S   s   g | ]}|j d kr|qS rZ   r   r[   r   r   r   r:   O  s     
 z)functionalize_rng_ops.<locals>.<listcomp>)"rk   countdictr   r   r   r    r!   r"   r#   r$   r   rA   Z_primsZ	rng_primsZrun_and_save_rng_staterun_with_rng_staterB   r   Zinserting_beforeZcreate_noderF   rG   r|   r}   Zreplace_all_uses_withZ
erase_noderK   nextr2   r   r   r?   	recompile) rY   	fw_module	bw_moduleZnum_sym_nodesuidr   r   r   Zjoint_graph_rng_opsZfw_graph_rng_opsZbw_graph_rng_opsZrecomputable_rng_ops_mapr   Z	base_nodeZfw_nodeZbw_nodeZrun_and_save_rngr   Zbw_tangent_start_nodeZfw_rng_state_outputsZ	node_pairZfw_graphZfunctional_fw_nodestateZ
rng_outputZbw_graphZ
state_nameZbw_rng_state_nodeZfw_output_nodeZ
fw_outputsZsym_node_start_idxrN   r   r   r   functionalize_rng_ops  s    










r   c                 C   sL   | j jD ]>}t|r|jD ]*}t|r|jd |jd krd|jd< qq| S )a  
    If there are two consecutive checkpointed blocks with no operator in
    between, we would still want to stash the tensor at the boundary of
    checkpointed blocks. The following pass makes the last output node
    non-recomputable to allow for that.
    r   r   )r   r   r   rg   r   )rY   r   r   r   r   r   cleanup_recompute_tagsZ  s    
r   inductorc          ,   K      s  zddl }W n, tk
r8 } ztd|W 5 d}~X Y nX | j  |   | j}tjrft|}|| _| j}	t	| t
| }
rt| } i | jjD ]}||j< qfdd}|| \}}}t|dkrt| |dS t| jjD ]@}|krd|_n,td|_|jD ]}t|j|jd |_qqtjjtjj}jjjjjjjjj j!j"j#j$j%j&j'j(j)j*j+j,j-j.j/j0j1j2j3j4j5j6j7j8j9j:j;j<j=j>j?j@jAjBjCjDjEjFjGjHjIjJjKjLjMjNjOjPjQjRjSjTjUjVjWjXjYjZj[j\j]t^j_j`jajbjcgK}j`jajdgd	kr<||j|jejfjcjg|jh|jWjhji|jjjkjljmjnjojpjqjrjsjtjujvjwjxjyjzj{j|j}j~jjg 7 }jnjpjlj|jjjoj}g7 |jg7 }|7 }|t 7 }|jg7 }|d
d tD 7 }dk	r~tnt|jjjg}jjjjjjjjjjjjjg}|| t|B trdd | jjD }|dd D  }td| t  d fdd fdd}fddfddtdfdd}| |	jD ]>}|jdkrq||krj|jd dtjd qt|st|rjd |jd tjd ||r|krjd |jd tjd d!|jko(d"|jkpJd!|jkoJt|jd! tj }t|r`t|}n|rntj}n||}j|jd |jd# |d |jD ]$}j|jd# |jd tjd qqz|d d\}}W n8 tk
r   td$ td%|jj  Y nX |\}
t }fd&d'|D D ]$\}|
fd(d'|D  q2t } |D ]>\}!}"|!dd) |"dd* kst|!dd) }#| |# qbd+d, t| jjD 	tfd-d'| D 	fd.d/d0}$ttd1d/ |$}%ttd2d/ |$}$t| |$|%d3\}&}'r<|
r4t| |&|'t|%\}&}'t|'}'trtd4tWd5d |$D d  d6d |&jjD }(d7d |'jjD })|(|)@ }*tt}+|&jjD ]8}|j|*krt|jd8r|+t|jj  d7  < qtd9t|* d:t|( d:t|)  td;t|+ d<d/ d=d> |&|'fS )?ay  
    Partitions the joint graph such that the backward recomputes the forward.
    Recomputing helps in trading off memory bandwidth with computation.

    To create the fwd and bwd graph, we copy the joint graph, manually set the
    outputs to just original forward or backward outputs. And then we run the
    resulting graphs through dead code elimintation.

    .. warning::
        This API is experimental and likely to change.

    Args:
        joint_module(fx.GraphModule): The joint forward and backward graph. This
            is the result of AOT Autograd tracing.
        _joint_inputs: The inputs to the joint graph. This is unused.
        compiler: This option determines the default set of recomputable ops.
            Currently, there are two options: ``nvfuser`` and ``inductor``.
        recomputable_ops: This is an optional set of recomputable ops. If this
            is not None, then this set of ops will be used instead of the
            default set of ops.
        num_fwd_outputs: The number of outputs from the forward graph.

    Returns:
        Returns the generated forward and backward Fx graph modules.
    r   NzANeed networkx installed to perform smart recomputation heuristicsc           
         s   t   | jjD ]@}|jdkr0d|jkr0 | | kr|jD ]} | q>qttt	| jj}ttt
| jj}|| }t| d\}} dd |D  t| j||}fdd|jD  fdd| jjD }	| |	fS )	Nr2   rS   r`   c                 s   s   | ]}|d k	r|V  qd S r;   r   )r6   or   r   r   r     s      zNmin_cut_rematerialization_partition.<locals>.classify_nodes.<locals>.<genexpr>c                    s    h | ]}|j d kr |j qS rZ   ry   r[   name_to_noder   r   rz     s    
zNmin_cut_rematerialization_partition.<locals>.classify_nodes.<locals>.<setcomp>c                    s    h | ]}|kr| kr|qS r   r   r[   required_bw_nodesrequired_fw_nodesr   r   rz     s     )ri   r   r   rB   r!   rj   rg   re   rf   rV   rU   r_   updaterQ   )
rY   r   r   rs   ru   rM   r]   r^   r   unclaimed_nodes)r   r\   r   r   classify_nodes  s     

z;min_cut_rematerialization_partition.<locals>.classify_nodesr`   g    eAr   r   c                 S   s   g | ]}t |qS r   )r   )r6   mr   r   r   r:     s   z7min_cut_rematerialization_partition.<locals>.<listcomp>c                 S   s.   h | ]&}|j d krt|jdrt|jjqS )r3   _overloadpacket)rB   r    r!   strr   r[   r   r   r   rz     s   
 z6min_cut_rematerialization_partition.<locals>.<setcomp>c                 S   s   h | ]}t |qS r   )r   r6   r   r   r   r   rz     s     z#Ops banned from rematerialization: Fc                    sb   | h}t |dkr^| }|jD ]:}|kr< ||s< dS |kr t|kr || q qdS )Nr   TF)r   poprg   r   rj   )r   Z	cur_nodescurr   )
is_fusibler   view_opsr   r   is_materialized_backwards  s    
zFmin_cut_rematerialization_partition.<locals>.is_materialized_backwardsc                    s   d| j kr| j d dkS  r2| jdko0t| kS | jdkr@dS t| krPdS | jtjkr`dS | jjjjjfkrzdS | rdS sdkr| j	t
jkrdS tdd | jD }t| }|d	 |k S d S )
Nr   r   r3   FTr   c                 s   s"   | ]}t |tjrt|V  qd S r;   )r(   r4   r5   r   r   r   r   r   r      s      zQmin_cut_rematerialization_partition.<locals>.ban_recomputation.<locals>.<genexpr>r'   )r   rB   r   r!   r|   r}   lift_fresh_copydefaultZ
lift_freshdist_from_bwr   Zmax_dist_from_bwr   rF   r   )r   Zinput_tensors_sizeZoutput_size)AGGRESSIVE_RECOMPUTATIONr   compilergraph_has_recomputable_opsr   recomputable_opsunrecomputable_opsr   r   ban_recomputation  s(    

	z>min_cut_rematerialization_partition.<locals>.ban_recomputationc                    s   t |  kot | kS r;   )r   )ab)fusible_opsr   r   r   $  s    z7min_cut_rematerialization_partition.<locals>.is_fusiblec                    s*    j dkrdS t fdd jD  S )Nr2   Tc                 3   s   | ]} |V  qd S r;   r   r~   )r   r   r   r   r   +  s     zOmin_cut_rematerialization_partition.<locals>.is_materialized.<locals>.<genexpr>)rB   r   rg   r   )r   r   r   is_materialized'  s    
z<min_cut_rematerialization_partition.<locals>.is_materialized)rx   c                    s>   t | }t|dtt| jdd  } | r2|S |d S d S )Ng?d   r      )r   r   r   minr   )r   Zmem_sz)r   r   r   get_node_weight-  s
    z<min_cut_rematerialization_partition.<locals>.get_node_weightr?   Z_inZsink)capacitysourcer&   r{   Z_outz-Failed to compute min-cut on following graph:
c                 3   s   | ]}| | fV  qd S r;   r   r   )nx_graphr   r   r   e  s     z6min_cut_rematerialization_partition.<locals>.<genexpr>c                 3   s   | ]}| kr|fV  qd S r;   r   )r6   v)non_reachableur   r   r   f  s      c                 S   s   i | ]\}}||qS r   r   )r6   r   r   r   r   r   r   o  s      z7min_cut_rematerialization_partition.<locals>.<dictcomp>c                 3   s   | ]} | V  qd S r;   r   r[   r   r   r   r   p  s     c                    s    |  S r;   r   r<   )node_idxr   r   r=   p  r>   z5min_cut_rematerialization_partition.<locals>.<lambda>rc   c                 S   s   t | S r;   r   r   r   r   r   r=   r  r>   c                 S   s
   t |  S r;   r   r  r   r   r   r=   s  r>   r   z Theoretical Activations Stored: c                 S   s   g | ]}t |qS r   )r   r   r   r   r   r:     s     c                 S   s   h | ]}|j d kr|jqS r3   ry   r[   r   r   r   rz     s     
 c                 S   s   h | ]}|j d kr|jqS r  ry   r[   r   r   r   rz     s     
 r   z# remat/fw/bw: /zCount of Ops Rematerialized: c                 S   s   | d S r   r   r<   r   r   r   r=     r>   Tr   )ZnetworkxImportErrorrJ   r   rL   r   r   Zcser   r   r%   r   r   rA   r   r   reversedr   r   rg   r   r"   r   r   primsrj   subdivatan2mulr   pow	remainderfmod__and____or____xor__
__lshift__
__rshift__eqnegegtleltabsZbitwise_notceilfloorfracnegZreluroundZsilutruncloglog10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtZrsqrtZ
reciprocalZsigmoidZsoftplus	thresholdZthreshold_backwardclampwhereZlerpZaddcmulZgeluZgelu_backwardr   ZmeanZ_grad_sum_to_sizeZsum_to_sizeZamaxtoZtype_asr|   r}   ZsqueezeZ	unsqueezeZrsubZ_to_copyaliasZconvert_element_typecloneZ	full_likevarZstdZbroadcast_in_dimselectZpermuteZ_unsafe_viewviewexpandsliceZreshapeZbroadcast_tensorsZscalar_tensorZonesZ	new_zerosr   ZarangeZtriuZvar_meanisinfrH   fullZ
as_stridedZzerosZargmaxmaximumtindexr   Z
zeros_liker   ri   Znative_dropoutZ	rand_likeZ
randn_likemmZconvolutionZconvolution_backwardZbmmZaddmmZupsample_bilinear2dZ_softmaxZ_softmax_backward_dataZnative_layer_normZnative_layer_norm_backwardZnative_batch_normZnative_batch_norm_backwardZ_native_batch_norm_legitAOT_PARTITIONER_DEBUGr   ZDiGraphrB   Zadd_edger   r   rV   rU   r   r(   r   r   r+   Zminimum_cut	ExceptionjoinZ	readwriteZedgelistZgenerate_edgelistr   r*   r   rm   re   rf   rw   r   r   r   r    r!   r   r   r   ),rY   r   r   r   r\   nxer   Z	cse_graphZfull_bw_graphZgraph_has_recomputable_rng_opsr   r   Zorig_fw_outputsr   r   r   r	  Zdefault_recomputable_opsZ
random_opsZcompute_intensive_opsZjoint_module_opsZops_ignoredr   r   Zis_non_tensor_nodeweightZ	cut_value	partitionZ	reachableZcutsetZnbrsZ	cut_nodesZnode_inZnode_outZ	node_namerq   rr   r   r   Zfw_module_nodesZbw_module_nodesZremat_nodescountsr   )r   r   r   r   r   r   r   r   r   r  r   r\   r   r   r   r   r   r   r   r   i  s
   


 1
$
8
'



&"   
   &r   fx_graphT)tracedfnamefignamec           
      C   s   |r0t | j}t| |} | jjD ]
}i |_q$tj	|\}}|sHd}t
d| |  t| |}| }	t|	d|d | |  d S )Nz.svgzWriting FX graph to file: Zwrite_.)copydeepcopyr   r4   rp   r   r   ospathsplitextr   r   ZFxGraphDrawerZget_main_dot_graphr   lstrip)
rQ  rR  rS  Z
clear_metarO   r   baseextgr7   r   r   r   
draw_graph  s    r^  full_graph.pngc                 C   s   t | | t| |S r;   )r^  r   )r   Zjoint_inputs	file_namer   r   r   draw_joint_graph  s    
ra  )r   N)rP  T)r_  )EZ"torch.fx.experimental.proxy_tensorr   r   Z%torch.fx.experimental.symbolic_shapesr   r   r   r   r   r	   r"   Ztorch.fxr4   r|   r   Ztorch.utils._pytreeutilsZ_pytreerD   rU  rW  rk   Zsympyr   r   Ztorch.fx.passesr   typingr   Zcompile_utilsr   r    r   	functoolsZdebug_partitionerrH  r   r   r%   r+   r,   rC   rQ   rV   rX   rT   rU   rp   r_   rw   r   r   r   r5   r   r   r   	lru_cacher   r   r   r   r   r   r   r   r^  ra  r   r   r   r   <module>   sn    5^J	
K      '