U
    *-ey                     @   s   d dl Z d dlmZmZmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ ddlmZmZ dZe dejeee	jj  ejddd	Zejed
ddZG dd dZdS )    N)AnyDictIterableListOptionalSetTuple)DistributedDataParallel   )_get_registrycontract )moduleignored_modulesreturnc                 K   sV   t jd d|kr>t|d tt jfs>tdt|d  t|dj	| f| | S )zReplicates a module

    Args:
        module (torch.nn.Module): module to replicate

    Example::
        >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d)
        >>> module = nn.Linear(3, 3)
        >>> replicate(module)
    ztorch.distributed.replicate	device_idz6Expected device_id to be int or torch.device, but got )r   )
torchZ_CZ_log_api_usage_once
isinstanceintdeviceRuntimeErrortype_ReplicateStatemark_module)r   r   kwargs r   h/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/torch/distributed/_composable/replicate.py	replicate   s    r   r   r   c                 C   s   dt | kS )z+Check if module is marked with fully_shard.Zfully_shard)r   )r   r   r   r   _is_fully_sharded(   s    r   c                   @   s   e Zd Zeeejj  ddddZejddddZ	e
fejeddd	d
ZddddZejeedf eeef edddZejeej ejejdddZdS )r   N)r   r   c                 C   sP   d | _ d| _t | _i | _|d k	r,t|nt | _dd | jD | _g | _	d S )NFc                 S   s   h | ]}|  D ]}|qqS r   )
parameters).0mpr   r   r   	<setcomp>6   s    
  z+_ReplicateState.__init__.<locals>.<setcomp>)
r   has_initializednnZParameterList_param_listr   setr   ignored_params_param_names)selfr   r   r   r   __init__.   s    
z_ReplicateState.__init__r   c                 K   sH   t |rtd|| _dt|_|j| jdd || j	 || _
d S )NzGCannot apply `replicate()` on a Module already managed by `fully_shard`FT)Zwith_kwargs)r   AssertionErrorr   r   stateZ_params_collectedZregister_forward_pre_hookforward_pre_hookZregister_forward_hookforward_post_hookr   )r+   r   r   r   r   r   r   <   s    z_ReplicateState.mark_module)r   prefixr   c                 C   s   t |rd S || jkrd S |tkr,| dnt}|jddD ]2\}}|| jkr<| j| | j| |  q<| D ]\}}| j	|| | d qxd S )N.F)recurse)r   r1   )
r   r   _ROOT_MODULE_PREFIXZnamed_parametersr)   r'   appendr*   Znamed_children_collect_params)r+   r   r1   Zrecurse_prefixnr#   nameZchild_moduler   r   r   r6   H   s    

z_ReplicateState._collect_params)r   c                 C   s   | j r
d S d| _ | | j | jt| j_d| jkr| jd d k	r~| jd }t|t	j
rp|jdkrpd | jd< q|g| jd< n
d | jd< | jd t| jf| j| _t| jt| j_d S )NTr   cpuZ
device_ids)r%   r6   r   r*   r   r.   Z_replicate_param_namesr   r   r   r   r   popr	   r'   _ddpweakrefrefZ_ddp_weakref)r+   r   r   r   r   init_helper^   s    


z_ReplicateState.init_helper.)r   argsr   r   c                 C   s"   |    | jj||\}}||fS N)r>   r;   Z_pre_forward)r+   r   r?   r   r   r   r   r/   }   s    z _ReplicateState.forward_pre_hook)r   inputoutputr   c                 C   s   | j |S r@   )r;   Z_post_forward)r+   r   rA   rB   r   r   r   r0      s    z!_ReplicateState.forward_post_hook)__name__
__module____qualname__r   r   r   r&   Moduler,   r   r4   strr6   r>   r   r   r   r/   ZTensorr0   r   r   r   r   r   -   s&      
 
	r   )N)r<   typingr   r   r   r   r   r   r   r   Ztorch.nnr&   Ztorch.nn.parallelr	   r   r   r4   rF   r   boolr   r   r   r   r   r   <module>   s   $ 