U 9%e@s8UddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z ddlmZmZmZmZmZmZmZmZmZmZddlZddlmmZddlmZddl m!Z!ddl"m#Z#m$Z$m%Z%dd l&m'Z'dd l(m)Z)m*Z*m+Z+dd l,m-Z-m.Z.m/Z/m0Z0m1Z1ej2ej3BZ4ej5j6j7Z8ee9d <ej5j6j:Z;ee9d <iZe!ddGddde?Z@e eAe dddZBe!ddGdddZCeCZDe!ddGdddeCZEe!ddGddde/ZFiZGeeeAeHfeIfe9d<gZJeee?eHfe9d <ejKLd!d"kr8eJMejNd#fd$d%ZOd&d'ZPd(d)ZQGd*d+d+eZRGd,d-d-eRZSGd.d/d/eRZTGd0d1d1eRZUGd2d3d3ZVeVd4d5d6ZWeVeeHefeeAd7d8d9ZXe!ddeeHefd:d;d<ZYe!dddCeej5j6ed=effeeeHefe'd>d?d@ZZeYdAdBZ[dS)DN)chain)CodeType FunctionType ModuleType) AnyCallableDictList NamedTupleOptionalSetTupleTypeUnion) ScriptObject) compatibility)_PyTreeCodeGen _PyTreeInfoGraph) GraphModule)Argument base_types map_aggregate)ParameterProxyProxy TracerBaseScopeScopeContextManager_orig_module_call_orig_module_getattr_proxyable_classesFcCstSN)_is_fx_tracing_flagr$r$W/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/torch/fx/_symbolic_trace.py is_fx_tracing-sr&TZis_backward_compatiblecs(eZdZdZfddZddZZS)ProxyableClassMetaa ProxyableClassMeta allows you to make construction of a given Python class symbolically traceable. For example:: import torch import torch.fx class TensorPair(metaclass=torch.fx.ProxyableClassMeta): def __init__(self, left, right): self.left, self.right = left, right def add(self, other): l = self.left + other.left r = self.right + other.right return TensorPair(l, r) def mul(self, other): l = self.left * other.left r = self.right * other.right return TensorPair(l, r) def use_tensor_pair_ctor(x : TensorPair, y : torch.Tensor): s = x.add(TensorPair(y, y)) return s.mul(x) x = TensorPair(torch.randn(5, 3), torch.randn(5, 3)) y = torch.randn(5, 3) ref_out = use_tensor_pair_ctor(x, y) traced = torch.fx.symbolic_trace(use_tensor_pair_ctor) print(traced.code) ''' def forward(self, x : __main___TensorPair, y : torch.Tensor): tensor_pair = __main___TensorPair(y, y); y = None add = x.add(tensor_pair); tensor_pair = None mul = add.mul(x); add = x = None return mul ''' From this example, we can see that construction of a class (``TensorPair``) defined with ``ProxyableClassMeta`` as metaclass can be recorded in symbolic tracing. cst|t|||dSr")r! setdefaultsuper__init__)clsnamebasesattrs __class__r$r%r+^s zProxyableClassMeta.__init__csn||}gfdd}t||t||tdkrTdj}|d|||S|j|f|||SdS)Ncst|tr|dSr") isinstancerappend)aZ found_proxiesr$r% check_proxygs z0ProxyableClassMeta.__call__..check_proxyr call_function)__new__rlentracer create_proxyr+)r,argskwargsinstancer6r:r$r5r%__call__bs      zProxyableClassMeta.__call__)__name__ __module__ __qualname____doc__r+r? __classcell__r$r$r0r%r(0s, r()fnnargsreturncCs|j}|jt@}t|drb|dd|j|j||j|j|j|j |j |j |j |j |j|j|j|jf}nt|dr|dd|j|j||j|j|j|j |j |j |j |j|j|jf}n:|d|j|j||j|j|j|j |j |j |j |j|j|jf}t|}t||j|j|j|jS)N co_qualnamerco_posonlyargcount)__code__co_flags HAS_VARSTUFFhasattr co_nlocals co_stacksizeco_code co_constsco_names co_varnames co_filenameco_namerHco_firstlineno co_lnotabZco_exceptiontable co_freevars co_cellvarsrr __globals__r@ __defaults__ __closure__)rErFcorKZco_argsZnew_coder$r$r%_patch_functionvs   r^c@seZdZdZddZdS)PHBaseE Object representing an input placeholder to `concrete_args` cCsdS)NPHr$selfr$r$r%__repr__szPHBase.__repr__N)r@rArBrCrdr$r$r$r%r_sr_cs,eZdZdZdeedfdd ZZS) PHWithMetar`N)ph_keycst||_dSr")r*r+rf)rcrfr0r$r%r+s zPHWithMeta.__init__)N)r@rArBrCr strr+rDr$r$r0r%resrecsbeZdZdZeddefddfeeeedfe ddfd d Z edde d d fd d Z edde jjee dddZedde jjedddZedde jjede fee dfeee fe dddZeddee eee fdddZeddd"ddZeddd#ee jjede ffeeee fedddZd d!ZZS)$TraceraTracer(autowrap_modules=(math,), autowrap_functions=()) ``Tracer`` is the class that implements the symbolic tracing functionality of ``torch.fx.symbolic_trace``. A call to ``symbolic_trace(m)`` is equivalent to ``Tracer().trace(m)``. Tracer can be subclassed to override various behaviors of the tracing process. The different behaviors that can be overridden are described in the docstrings of the methods on this class. Tr'r$F.N)autowrap_modulesautowrap_functionsparam_shapes_constantrGcsztddtdd|DD|_|jdd|Dt||_||_d|_d|_ t dd|_ t |_i|_dS)a Construct a Tracer object. Args: autowrap_modules (Tuple[ModuleType]): defaults to `(math, )`, Python modules whose functions should be wrapped automatically without needing to use fx.wrap(). Backward-compatibility for this parameter is guaranteed. autowrap_functions (Tuple[Callable, ...]): defaults to `()`, Python functions that should be wrapped automatically without needing to use fx.wrap(). Backward compatibility for this parameter is guaranteed. param_shapes_constant (bool): When this flag is set, calls to shape, size and a few other shape like attributes of a module's parameter will be evaluated directly, rather than returning a new Proxy value for an attribute access. Backward compatibility for this parameter is guaranteed. cSs*h|]"\}}|dst|rt|qS)_) startswithcallableid).0r-valuer$r$r% s z"Tracer.__init__..cSsg|]}|jqSr$)__dict__items)rpmr$r$r% sz#Tracer.__init__..cSsh|] }t|qSr$)ro)rpfr$r$r%rrsN)r*r+r_autowrap_function_idsupdatelist_autowrap_searchrksubmodule_pathsroot_module_namerscope collections OrderedDict module_stackZnode_name_to_scope)rcrirjrkr0r$r%r+s!    zTracer.__init__r)r4rGc st|tjjrHjD]$\}}||krd|diSqtdn|t|tjrj D]$\}}||kr^d|diSq^n>t|tjj rĈj D]$\}}||krd|diSqt|t rt |drt fdd|D}d|j|iSt|tjtfr|j|}|sld}d |}t j|sHqT|d 7}q,|j|<tj||d|diSt|tkrd}d |jjd |}t j|sq|d 7}qtj||d|diSt|S) a A method to specify the behavior of tracing when preparing values to be used as arguments to nodes in the ``Graph``. By default, the behavior includes: #. Iterate through collection types (e.g. tuple, list, dict) and recursively call ``create_args`` on the elements. #. Given a Proxy object, return a reference to the underlying IR ``Node`` #. Given a non-Proxy Tensor object, emit IR for various cases: * For a Parameter, emit a ``get_attr`` node referring to that Parameter * For a non-Parameter Tensor, store the Tensor away in a special attribute referring to that attribute. This method can be overridden to support more types. Args: a (Any): The value to be emitted as an ``Argument`` in the ``Graph``. Returns: The value ``a`` converted into the appropriate ``Argument`` get_attrr$z(parameter is not a member of this module_fieldsc3s|]}|VqdSr") create_arg)rpelemrbr$r% Wsz$Tracer.create_arg..r7rZ_tensor_constantrrlZ _constant_)r2torchnn Parameterrootnamed_parameters create_node NameErrorTensor named_buffersModule named_modulestuplerMr1r tensor_attrsgetsetattrtyper!r@r*r) rcr4npZn_Zp_r<qualnameir0rbr%r'sH         zTracer.create_arg)rumodule_qualified_namerGcCs(|jds|jdo&t|tjj S)aF A method to specify whether a given ``nn.Module`` is a "leaf" module. Leaf modules are the atomic units that appear in the IR, referenced by ``call_module`` calls. By default, Modules in the PyTorch standard library namespace (torch.nn) are leaf modules. All other modules are traced through and their constituent ops are recorded, unless specified otherwise via this parameter. Args: m (Module): The module being queried about module_qualified_name (str): The path to root of this module. For example, if you have a module hierarchy where submodule ``foo`` contains submodule ``bar``, which contains submodule ``baz``, that module will appear with the qualified name ``foo.bar.baz`` here. ztorch.nnz torch.ao.nn)rArmr2rrZ Sequential)rcrurr$r$r%is_leaf_moduleszTracer.is_leaf_module)modrGcCsd|jr4|j|}|dkr"tdt|ts0t|S|jD]\}}||kr>|Sq>tddS)a~ Helper method to find the qualified name of ``mod`` in the Module hierarchy of ``root``. For example, if ``root`` has a submodule named ``foo``, which has a submodule named ``bar``, passing ``bar`` into this function will return the string "foo.bar". Args: mod (str): The ``Module`` to retrieve the qualified name for. Nz&module is not installed as a submodule)r}rrr2rgAssertionErrorrr)rcrpathrrr$r$r%path_of_modules   zTracer.path_of_module)ruforwardr<r=rGc Cs||}t|jt|t|f}|j|j|j<|||sH|||}n| d|||}|jj dd\}} ||jkst d|W5QRX|S)a Method that specifies the behavior of this ``Tracer`` when it encounters a call to an ``nn.Module`` instance. By default, the behavior is to check if the called module is a leaf module via ``is_leaf_module``. If it is, emit a ``call_module`` node referring to ``m`` in the ``Graph``. Otherwise, call the ``Module`` normally, tracing through the operations in its ``forward`` function. This method can be overridden to--for example--create nested traced GraphModules, or any other behavior you would want while tracing across ``Module`` boundaries. Args: m (Module): The module for which a call is being emitted forward (Callable): The forward() method of the ``Module`` to be invoked args (Tuple): args of the module callsite kwargs (Dict): kwargs of the module callsite Return: The return value from the Module call. In the case that a ``call_module`` node was emitted, this is a ``Proxy`` value. Otherwise, it is whatever value was returned from the ``Module`` invocation. call_moduleT)lastz Unexpected key ) rrrrr module_typer module_pathrr;popitemr) rcrurr<r=rZ_scopeZret_valkeyrlr$r$r%rs"   "zTracer.call_module)attrattr_valparameter_proxy_cachecslfdd}t|tjjr8||j|}|dk r8|Sjrht|tjrh||j|}|dk rh|S|S)a Method that specifies the behavior of this ``Tracer`` when we call getattr on a call to an ``nn.Module`` instance. By default, the behavior is to return a proxy value for the attribute. It also stores the proxy value in the ``parameter_proxy_cache``, so that future calls will reuse the proxy rather than creating a new one. This method can be overridden to --for example-- not return proxies when querying parameters. Args: attr (str): The name of the attribute being queried attr_val (Any): The value of the attribute parameter_proxy_cache (Dict[str, Any]): A cache of attr names to proxies Return: The return value from the getattr call. cs~|D]t\}|kr|krli}dtjjkrPjs s zBTracer.getattr..maybe_get_proxy_for_attr..rr$)inspect signaturer; parametersrk)rZcollection_to_searchrrr=Z val_proxyrb)rrr%maybe_get_proxy_for_attrs   z0Tracer.getattr..maybe_get_proxy_for_attrN) r2rrrrrZproxy_buffer_attributesrr)rcrrrrZmaybe_parameter_proxyZmaybe_buffer_proxyr$rbr%getattrs$ zTracer.getattrc stj}|j|j}t|j}t|jg}d}|rd|dkrLtdd}t | j t t dfdd fddt||D} ttrt| tkrtd t| d td tt| |fd d | D|jdks|jt@rj|jtj@r8| dt |jtj@r\| dt tt|tt|\} tdd jDrtt|d|dj _!fdd} | | fS|fS)z Create ``placeholder`` nodes corresponding to the signature of the ``root`` Module. This method introspects root's signature and emits those nodes accordingly, also supporting ``*args`` and ``**kwargs``. rz4``self`` argument cannot be part of *args expansion!r)r-c sdk r6kr6dfdd}t|SddkrHd}n$j}|jtjjkrddn|jf}jd|ij ddS)Nrcsd7j}|jtjjkr$dn|jf}ddt|i}t|trvdd}|t krr|||j d|St |t kst |t krt |tjkrt||kd|d nBt |t dkr|d f}d t|intd d |S)Nrr$ placeholderrlcSsDt|D]6}t||}t|s|dst||st|||qdS)N__)dirrrnrmrMr)frto attr_namerr$r$r%transfer_attrsTs  zbTracer.create_args_for_root..proxy_placeholder..replace_ph..transfer_attrs)rrz$ has been specialized to have value z but got another valuez> has been specialized to have value None but got another valuer7z9Was not able to add assertion to guarantee correct input z to specialized function. It is up to the user to make sure that your inputs match the inputs you specialized the function with.)rdefaultrremptyr;rgr2r_rarrboolrrrZ_assert_assert_is_nonewarningswarn)xparamroutrr<)cntr-rcsigr$r% replace_phGsH        zJTracer.create_args_for_root..proxy_placeholder..replace_ph*r$rZ type_expr) pytreeZtree_maprrrrrr;__annotations__r)r-rrr) concrete_argsfn_for_analysisrcr)rr-r%proxy_placeholderCs;   z6Tracer.create_args_for_root..proxy_placeholdercsg|] }tqSr$)next)rpidx) names_iterr$r%rvsz/Tracer.create_args_for_root..zTracing expected z arguments but got z concrete argumentsc3s|]}|VqdSr"r$)rpnames)rr$r%rsz.Tracer.create_args_for_root..rz**css|]}t|tj VqdSr")r2rZLeafSpec)rprr$r$r%rsNcsTtt|}|}t|\}}tjjts8tjjj j |djj_ |S)N)out_spec) rZtree_unflattenr{ tree_flattenr2graph_codegenrrZ pytree_info_replace)r<Z tree_argsZtree_outZout_argsr)in_specroot_fnrcr$r% flatten_fnsz/Tracer.create_args_for_root..flatten_fn)"runwraprJ co_argcountco_kwonlyargcountr{rSiter RuntimeErrorrr3rrrgranger2rr9dictzipextendrKrL CO_VARARGSCO_VARKEYWORDSr^rranyZchildren_specsrrrr) rcrZ is_modulerr]Z total_args orig_argsr<Z skip_arg_idx arg_namesZ flat_argsrr$)rrrrrrrcrr%create_args_for_root&sN      M  zTracer.create_args_for_rootrrrGc st}dazt|tjjr||_tt|jsLt djdt|j t t|j}| _ dd|D_ntj_|}t dd}t|d_t|d r|j}|j|j|jd j_i_tjjttd fd d jgt|ts t |j}|t|tjj|\}}ittfdd} tt fdd} t!j"tjjd| ddj"tjjd| ddt#t$|j%j&D]} t$| j'j%qj(dd)||fi|j*+dddW5QRXd_W5|aXjS)a Trace ``root`` and return the corresponding FX ``Graph`` representation. ``root`` can either be an ``nn.Module`` instance or a Python callable. Note that after this call, ``self.root`` may be different from the ``root`` passed in here. For example, when a free function is passed to ``trace()``, we will create an ``nn.Module`` instance to use as the root and add embedded constants to. Args: root (Union[Module, Callable]): Either a ``Module`` or a function to be traced through. Backwards-compatibility for this parameter is guaranteed. concrete_args (Optional[Dict[str, any]]): Concrete arguments that should not be treated as Proxies. This parameter is experimental and its backwards-compatibility is *NOT* guaranteed. Returns: A ``Graph`` representing the semantics of the passed-in ``root``. Tztraced_func_name=z doesn't exist in cSsi|]\}}||qSr$r$)rpr-rr$r$r% sz Tracer.trace..r1N) tracer_clsrJ)rUrTrV)ru prefix_atomscs`|jD].\}}t|tjtfr d||gj|<q |D]\}}|||gqBdS)N.) rsrtr2rrrjoinrZnamed_children)rurkv)collect_tensor_attrsrcr$r%rs z*Tracer.trace..collect_tensor_attrscst||}||Sr")r r)rrr)rrcr$r%module_getattr_wrapper s z,Tracer.trace..module_getattr_wrappercs:fdd}tttddij|||S)Ncstf||Sr")r)r<r=rr$r%rsz:Tracer.trace..module_call_wrapper..forwardrrZ)_autowrap_checkrryr)rr<r=r)patcherrcrr%module_call_wrappers z)Tracer.trace..module_call_wrapper __getattr__F) deduplicater?outputrGr),r#r2rrrrrMrZtraced_func_namerr@rZ _get_namer~rr}rrrJrUrTrVZ _co_fieldsrr rgrrZr functoolswrapsr r_Patcher patch_method_patch_wrapped_functionsrryr|rsrrrr) rcrrZold_is_fx_tracing_flagrErcodeZ fn_globalsr<rrmoduler$)rrrrcr%traces            z Tracer.tracecCsLtt}|jD]2\}}|dkr0t|}n t||}||j|<q|S)N>r|)rhr8rsrtcopydeepcopy)rcmemoZ new_tracerrrZnew_objr$r$r% __deepcopy__;s    zTracer.__deepcopy__)N)N)r@rArBrCrmathr rrrr+rrrrrrgrrrrrrrr rrrrDr$r$r0r%rhsL 9[   /> rh_wrapped_fns_to_patch_wrapped_methods_to_patchZFX_PATCH_GETITEM1 __getitem__csdfdd}t||S)zh Recursively search a data structure for a Proxy() and return it, return None if not found. Ncst|tr|dSr")r2r)rproxyr$r% find_proxyds z_find_proxy..find_proxy)r)Zobjects_to_searchr r$r r% _find_proxy]s  r cstfdd}|S)Ncs>t||}|dk r4|jd||}d|jjd<|S||S)aN Given an closed-over ``orig_function`` to invoke, search the args and kwargs for a Proxy object. If there is one, emit a ``call_function`` node to preserve the call to this leaf function directly. Otherwise, just return the results of this function call, as this function is not being traced. Nr7TZ is_wrapped)r r:r;rmeta)r<r=r Z return_proxyorig_fnr$r%wrappedns  z%_create_wrapped_func..wrapped)rr)rrr$rr%_create_wrapped_funcmsrcs&t|tfdd}|S)Ncs.t||}|dk r$|jd||S||S)a Search the args and kwargs for a Proxy object. If there is one, emit a ``call_method`` node to preserve the call to this method directly. Otherwise, just return the results of this function call, as this function is not being traced. NZ call_method)r r:r;)r<r=r r-rr$r%rs z'_create_wrapped_method..wrapped)rrr)r,r-rr$rr%_create_wrapped_methods  rc@s.eZdZUeed<eed<eed<ddZdS) _PatchedFn frame_dictfn_namercCs tdSr")NotImplementedErrorrbr$r$r%revertsz_PatchedFn.revertN)r@rArBrrrgrr$r$r$r%rs rc@seZdZddZdS)_PatchedFnSetItemcCs|j|j|j<dSr")rrrrbr$r$r%rsz_PatchedFnSetItem.revertNr@rArBrr$r$r$r%rsrc@seZdZddZdS) _PatchedFnDelcCs|j|j=dSr")rrrbr$r$r%rsz_PatchedFnDel.revertNrr$r$r$r%rsrc@seZdZddZdS)_PatchedFnSetAttrcCst|j|j|jdSr")rrrrrbr$r$r%rsz_PatchedFnSetAttr.revertNrr$r$r$r%rsrcsneZdZfddZdeeefeeedddZ de eeeddd Z ed d d Z d dZ ddZZS)rcstg|_t|_dSr")r*r+ patches_madesetvisitedrbr0r$r%r+s z_Patcher.__init__T)rr-new_fnrcCsf||_||kr.tt|r.|jt||dn,t||ddrBdS|jt|||||||<dS)zY Replace frame_dict[name] with new_fn until we exit the context manager. N__fx_already_patchedF)_Patcher__fx_already_patchedrMbuiltinsrr3rrr)rcrr-r!rr$r$r%patchs z_Patcher.patch)r,r-r!rcCsD||_t||}t|ddr dS|jt|||t|||dS)z\ Replace object_or_dict.name with new_fn until we exit the context manager. r"FN)r#rrr3rr)rcr,r-r!rrr$r$r%rs   z_Patcher.patch_method)thingcCs&t|}||jkrdS|j|dS)zz0wrap must be called at the top level of a module)rnr2rgrrr@r currentframef_backf_coderUr f_globalsrro)r1rr2rwr$r$r%wrap s,"   r6.rcCs<t}|||}t|tjjr(|jjn|j}t|j ||S)a Symbolic tracing API Given an ``nn.Module`` or function instance ``root``, this function will return a ``GraphModule`` constructed by recording operations seen while tracing through ``root``. ``concrete_args`` allows you to partially specialize your function, whether it's to remove control flow or data structures. For example:: def f(a, b): if b == True: return a else: return a*2 FX can typically not trace through this due to the presence of control flow. However, we can use `concrete_args` to specialize on the value of `b` to trace through this:: f = fx.symbolic_trace(f, concrete_args={'b': False}) assert f(3, False) == 6 Note that although you can still pass in different values of `b`, they will be ignored. We can also use `concrete_args` to eliminate data-structure handling from our function. This will use pytrees to flatten your input. To avoid overspecializing, pass in `fx.PH` for values that shouldn't be specialized. For example:: def f(x): out = 0 for v in x.values(): out += v return out f = fx.symbolic_trace(f, concrete_args={'x': {'a': fx.PH, 'b': fx.PH, 'c': fx.PH}}) assert f({'a': 1, 'b': 2, 'c': 4}) == 7 Args: root (Union[torch.nn.Module, Callable]): Module or function to be traced and converted into a Graph representation. concrete_args (Optional[Dict[str, any]]): Inputs to be partially specialized Returns: GraphModule: a Module created from the recorded operations from ``root``. ) rhrr2rrrr1r@rr)rrr:rr-r$r$r%symbolic_traceIs 4 r7cCs|dkst|dSr")r)rqmsgr$r$r%rsr)N)\r$rrrrosrr itertoolsrtypesrrrtypingrrrr r r r r rrrZtorch.utils._pytreeutilsZ_pytreerZtorch._CrZ_compatibilityrrrrrZ graph_modulerrrrrr rrrrrrrrLrrr?rrrr r!r#r&rr(intr^r_rarerhrrgrrenvironrr3rr rrrrrrrrrr6r7rr$r$r$r%s 0     EI  y =  =;