U
    9%e%                  
   @   s   d dl Z d dlZd dlmZmZmZmZmZmZm	Z	m
Z
 ddlmZ d dlmZ d dlmZ d dlmZ dd	gZe	ejee eed
f eeef f eej dddZdee ee eeeeef   eeee	eejf    ee ddd	ZdS )    N)AnyDictListOptionalSequenceTupleUnioncast   )Module_get_device_index)autocast)ExceptionWrapper	get_a_varparallel_apply.)objreturnc                 C   sx   t | tjr| S t | ttfrBtt| D ]}t |tjr(|  S q(t | trttt|  D ]}t |tjrZ|  S qZd S )N)	
isinstancetorchTensorlisttuplemapr   dictitems)r   result r   _/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/torch/nn/parallel/parallel_apply.pyr      s    


)modulesinputs
kwargs_tupdevicesr   c           
   	      s  t | t |ks,tdt |  dt | |dk	rJt | t |ksftnttttf i ft |  }|dk	rt | t |kstndgt |  }dd |D }dd |D }t i t	 t
  dttttttf ttttjf  ttjj ddfdd	 t | d
krj fddtt| ||||D }|D ]}|  qD|D ]}|  qXn& d| d |d |d |d |d  g }tt |D ],}| }	t|	tr|	  ||	 q|S )a~  Applies each `module` in :attr:`modules` in parallel on arguments
    contained in :attr:`inputs` (positional) and :attr:`kwargs_tup` (keyword)
    on each of :attr:`devices`.

    Args:
        modules (Module): modules to be parallelized
        inputs (tensor): inputs to the modules
        devices (list of int or torch.device): CUDA devices

    :attr:`modules`, :attr:`inputs`, :attr:`kwargs_tup` (if given), and
    :attr:`devices` (if given) should all have same length. Moreover, each
    element of :attr:`inputs` can either be a single object as the only argument
    to a module, or a collection of positional arguments.
    zThe number of modules z& is not equal to the number of inputs Nc                 S   s   g | ]}t |d qS )Tr   .0xr   r   r   
<listcomp>6   s     z"parallel_apply.<locals>.<listcomp>c                 S   s   g | ]}t j|qS r   )r   cudacurrent_streamr#   r   r   r   r&   7   s     )imoduleinputkwargsdevicestreamr   c                    s.  t  |d krTt|}|d krL td|  dd| < W 5 Q R X d S | }|d krht j|}zt j|R t j|: t	 d$ t
|ttfs|f}|||}W 5 Q R X W 5 Q R X W 5 Q R X  || < W 5 Q R X W n@ tk
r(     td|  d| d| < W 5 Q R X Y nX d S )Nzin replica zQ, no device was provided and no tensor input was found; device cannot be resolved)where)enabledz on device )r   Zset_grad_enabledr   r   Z
get_devicer'   r(   r-   r.   r   r   r   r   	Exception)r)   r*   r+   r,   r-   r.   toutput)autocast_enabledgrad_enabledlockresultsr   r   _worker<   s0    

((zparallel_apply.<locals>._worker   c              
      s4   g | ],\}\}}}}}t j ||||||fd qS ))targetargs)	threadingThread)r$   r)   r*   r+   r,   r-   r.   )r8   r   r   r&   ^   s   r   )NN)lenAssertionErrorr	   r   strr   r<   Lockr   Zis_grad_enabledZis_autocast_enabledintr   r   r   r-   r'   ZStream	enumeratezipstartjoinranger   r   reraiseappend)
r   r    r!   r"   streamsthreadsthreadoutputsr)   r3   r   )r8   r4   r5   r6   r7   r   r      sN    ,  

!
&)NN)r<   r   typingr   r   r   r   r   r   r   r	   r   r   Ztorch.cuda._utilsr   Ztorch.cuda.ampr   Ztorch._utilsr   __all__r   r   r@   rB   r-   r   r   r   r   r   <module>   s"   (8  