U
    0-e^                  	   @   s  d dl Z d dlZd dlZd dlmZ d dlmZmZmZm	Z	 d dl
Z
ddlmZ ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZ d
dlmZmZ g fddZe je	ee eeef f dddZ e jeeef dddZ!e je	ee eeef f dddZ"d#e jeeef e#e	e jeeef f dddZ$ee eeef dddZ%ee je	e jeeef f dddZ&dd  Z'G d!d" d"Z(dS )$    Nliteral_eval)AnyDictListTuple   )SageMakerConfig)DYNAMO_BACKENDS)DynamoBackendPrecisionTypeis_ipex_availableis_npu_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)is_port_in_usemerge_dicts   )DistributedTypeSageMakerDistributedTypec                 C   sD   | |\}}t|  D ]$\}}|t| krt||| q|S )z4
    Filters out all `accelerate` specific args
    )parse_known_argsvarsitemskeyssetattr)argsparserZdefault_argsnew_args_keyvalue r!   X/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/accelerate/utils/launch.py_filter_args%   s
    r#   )r   returnc              	   C   s6  g }| j r| jrtd| j s:|tj | jr:|d || j || j t	j
 }t| jph| j|d< | jr~d|d< | jdkr| jdk	rt r| j|d< nt r| j|d	< n
| j|d
< | jdkr| j|d< t| j|d< nD| jdkr(| jdk	r| jnd|d< | jdk	r t| jnd|d< zt| j }W n6 tk
rp   td| j  dt  dY nX t||d< zt| j }W n2 tk
r   td| j  dt dY nX |j|d< | j |d< t| j!|d< t| j"|d< t| j#|d< t$ r.t| j% |d< t| j& |d< ||fS )zz
    Prepares and returns the command list and an environment with the correct simple launcher environment variables.
    0--module and --no_python cannot be used togetherz-mZACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallNZE_AFFINITY_MASKASCEND_RT_VISIBLE_DEVICESCUDA_VISIBLE_DEVICESr   ZMASTER_ADDRZMASTER_PORTz	127.0.0.1Z29500Unknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONUnknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMICOMP_NUM_THREADSZACCELERATE_USE_IPEXZACCELERATE_USE_XPU)'	no_pythonmodule
ValueErrorappendsys
executabletraining_scriptextendtraining_script_argsosenvironcopystrcpuZuse_cpudebuggpu_idsr   r   num_machinesmain_process_ipmain_process_portnum_processesr   mixed_precisionlowerlistr   dynamo_backendupperr
   r    dynamo_modedynamo_use_fullgraphdynamo_use_dynamicnum_cpu_threads_per_processr   ZipexZuse_xpu)r   cmdcurrent_envrJ   rM   r!   r!   r"   prepare_simple_launcher_cmd_env0   sZ    





"

rU   c           
   	   C   s  t | d}t | d}t | d}t | d}|dkrt| dt||  t| dt| t| dt| j t | d	d
rt| dt| t| dt| qt| d| d|  n(t| dt| |dk	rt| dt| |dkrd}t|rtd| d| jr| jrt	dn*| jr(t| dd n| jr<t| dd t
j }| jrVd|d< t | dd}|dkr| jdk	rt r||d< nt r||d< n||d< | j }zt|}W n0 t	k
r   t	d| dt  d Y nX t||d!< zt| j }W n2 t	k
r<   t	d"| j  dt d Y nX |j|d#< | j|d$< t| j|d%< t| j|d&< | jrJd|d'< t| j|d(< t| j |d)< t| j |d*< | j!dk	rt| j!|d+< | j"dk	rt| j"|d,< | j#dk	rt| j#|d-< | j$dk	rt| j$|d.< t| j% |d/< t| j& |d0< t| j' |d1< | j(rd2}	d|d3< t| j)||	d4 < t| j*||	d5 < t| j+||	d6 < | j,dk	rt| j,||	d7 < | j-dk	rt| j-||	d8 < | j.dk	rt| j.||	d9 < | j/dk	rt| j/||	d: < t| j0|d;< |S )<z_
    Prepares and returns an environment with the correct multi-GPU environment variables.
    rI   rF   rG   rH   r   nproc_per_nodennodes	node_ranksame_networkFmaster_addrmaster_portrdzv_endpoint:N<s  3Tried to launch distributed communication on port `&  `, but another process is utilizing it. Please specify a different port (such as using the `----main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.r%   r7   Tr6   r&   r'   rE   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   ZACCELERATE_USE_FSDPZFSDP_SHARDING_STRATEGYZFSDP_OFFLOAD_PARAMSZFSDP_MIN_NUM_PARAMSZFSDP_AUTO_WRAP_POLICYZFSDP_TRANSFORMER_CLS_TO_WRAPZFSDP_BACKWARD_PREFETCHZFSDP_STATE_DICT_TYPEZFSDP_FORWARD_PREFETCHZFSDP_USE_ORIG_PARAMSZFSDP_SYNC_MODULE_STATESZMEGATRON_LM_ZACCELERATE_USE_MEGATRON_LMZ	TP_DEGREEZ	PP_DEGREEZGRADIENT_CLIPPINGZNUM_MICRO_BATCHESZSEQUENCE_PARALLELISMZRECOMPUTE_ACTIVATIONSZUSE_DISTRIBUTED_OPTIMIZERr5   )1getattrr   rB   intmachine_rankr   ConnectionErrorr7   r6   r8   r?   r@   rA   rD   rE   r   r   rJ   rK   r   rL   r   rM   rN   r
   r    rO   rP   rQ   Zuse_fsdpZfsdp_sharding_strategyZfsdp_offload_paramsZfsdp_min_num_paramsZfsdp_auto_wrap_policyZ"fsdp_transformer_layer_cls_to_wrapZfsdp_backward_prefetch_policyZfsdp_state_dict_typeZfsdp_forward_prefetchZfsdp_use_orig_paramsZfsdp_sync_module_statesZuse_megatron_lmZmegatron_lm_tp_degreeZmegatron_lm_pp_degreeZmegatron_lm_gradient_clippingZmegatron_lm_num_micro_batchesZ megatron_lm_sequence_parallelismZ!megatron_lm_recompute_activationsZ%megatron_lm_use_distributed_optimizerrR   )
r   rI   rF   rG   rH   rT   rE   rJ   rM   prefixr!   r!   r"   prepare_multi_gpu_envi   s    









 "

rf   c           	   	   C   sB  t | d}t | d}t | d}t | d}d}| jdkrFt| dtd  |dkrL| jtd krLd	d
g}|dt| jdt| jg | jdk	r|dt| jg n:| jdk	r|dt| jg n|dt| j	| j
 g |dt|g | jr| jrtdn&| jr |d n| jr2|d || j || j n|dkr| jtd krt| dt||  t| dt| t| dt| j t | ddrt| dt| t| dt| nt| d| d|  n*t| dt| |dk	rt| dt| |dkrd}t|r6td| d| jrP| jrPtdn*| jrft| d d! n| jrzt| d"d! tj }| jrd#|d$< t | d%d&}|d&kr| jdk	rt s||d'< n||d(< zt| j }W n6 tk
r   td)| j  d*t  d+Y nX td,tj !d+|d,< t||d-< t| j" |d.< d#|d/< | j#dk	rpt| j#|d0< | j$dk	rt| j$|d1< | j%dk	rt| j% |d2< | j&dk	rt| j& |d3< | j'dk	rt| j' |d4< | j(dk	rt| j( |d5< | j)dk	r t| j) |d6< | j*dk	r:t| j*|d7< ||fS )8zt
    Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
    rI   rF   rG   rH   Ndeepspeed_multinode_launcherr   r   Z	deepspeedz--no_local_rankz
--hostfilez
--launcherz	--excludez	--includez
--num_gpusz--master_portr%   z--modulez--no_pythonrV   rW   rX   rY   FrZ   r[   r\   r]   r^   r_   r`   r7   Tr6   r&   r'   rE   r(   r+   r)   r,   r-   r.   
PYTHONPATHr/   ZACCELERATE_CONFIG_DS_FIELDSZACCELERATE_USE_DEEPSPEEDZACCELERATE_DEEPSPEED_ZERO_STAGEZ&ACCELERATE_GRADIENT_ACCUMULATION_STEPSZACCELERATE_GRADIENT_CLIPPINGZ-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICEZ)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEZACCELERATE_DEEPSPEED_ZERO3_INITZ+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODELZ ACCELERATE_DEEPSPEED_CONFIG_FILE)+ra   rg   r   r   r=   rB   Zdeepspeed_hostfileZdeepspeed_exclusion_filterZdeepspeed_inclusion_filterrI   rF   r7   r6   r8   r9   r<   r>   rb   rc   r   rd   r?   r@   rA   rD   rE   r   r   rJ   rK   rL   env_var_path_addpathabspathZ'deepspeed_fields_from_accelerate_configZ
zero_stageZgradient_accumulation_stepsZgradient_clippingZoffload_optimizer_deviceZoffload_param_deviceZzero3_init_flagZzero3_save_16bit_modelZdeepspeed_config_file)	r   rI   rF   rG   rH   rS   rT   rE   rJ   r!   r!   r"   prepare_deepspeed_cmd_env   s    
















rl   F)r   rT   podr$   c                 C   sL   | j dkr"| jrd|d< nd|d< | jr0d|d< |rD| j| _| j| _| |fS )zY
    Prepares and returns an environment with the correct TPU environment variables.
    Zbf161ZXLA_DOWNCAST_BF16ZXLA_USE_BF16r&   r'   )rJ   Zdowncast_bf16rD   Ztpu_vmvmZtpu_nameZtpu)r   rT   rm   r!   r!   r"   prepare_tpuA  s    

rp   )nargsr$   c                 C   s   t | dk ri S dd }t }|| \}}t|D ]p\}}|dr6d }|d t |k rx||d  drtdntd|d kr|j||d q6|j||d q6d	d
 || j	
 D S )Nr   c                 S   sB   z$t | } | d | kr t| W S | W S  tk
r<   |  Y S X d S )Nr   )floatrb   r8   )sr!   r!   r"   _infer_typeZ  s    
z+_convert_nargs_to_dict.<locals>._infer_type)-z--r   ul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)actionc                 S   s&   i | ]\}}||d krt |n|qS ))TrueFalser   ).0r   r    r!   r!   r"   
<dictcomp>y  s    z*_convert_nargs_to_dict.<locals>.<dictcomp>)lenargparseArgumentParserr   	enumerate
startswithr8   add_argument
parse_args__dict__r   )rq   rt   r   r   unknownindexargumentrw   r!   r!   r"   _convert_nargs_to_dictU  s,    

r   )sagemaker_configr   r$   c                 C   s
  t d | jtjd< | jd k	r,| jtjd< n6|jd k	rZ|jd k	rZ|jtjd< |jtjd< ntdtj	|j
}|sxd}tj|j
}|dstd	| d
t d t|j}zt|j }W n4 tk
r   td|j  dt  dY nX zt|j }W n2 tk
r>   td|j  dt dY nX dt||j|jt|jt|j| jjd}d }| jtjkrddddiii}d }	| j d k	rt d| j  d i }	t!| j F}
t"|
D ]6\}}|dkrސq|#d}|d $ |	|d < qW 5 Q R X t d|	  d }| j%d k	rt d| j% d g }t!| j%R}
t"|
D ]B\}}|dkrhqR|#d}|d |d $ d}|&| qRW 5 Q R X t d|  t d | j'||| j(| j)| j*| j+| j,| j-| j.d||||d }| j/d k	rt0| j/|}||	fS )!Nz(Configuring Amazon SageMaker environmentZAWS_DEFAULT_REGIONZAWS_PROFILEZAWS_ACCESS_KEY_IDZAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profiler.   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersr,   r-   r0   r&   )ZACCELERATE_USE_SAGEMAKERr/   r1   r2   r3   r4   Z%ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPEZsmdistributedZdataparallelenabledTzLoading SageMaker Inputs from z filer   	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF)	image_urientry_point
source_dirZroletransformers_versionpytorch_version
py_versionbase_job_nameZinstance_countZinstance_typeZdebugger_hook_configdistributionhyperparametersenvironmentZmetric_definitions)1printregionr?   r@   ZprofileZaws_access_key_idZaws_secret_access_keyEnvironmentErrorrj   dirnamer<   basenameendswithr8   r   r>   r   rJ   rK   rL   r   rM   rN   r
   rB   r    rO   rP   rQ   distributed_typer   ZDATA_PARALLELZsagemaker_inputs_fileopenr   splitstripZsagemaker_metrics_filer9   r   Ziam_role_namer   r   r   r   rF   Zec2_instance_typeZadditional_argsr   )r   r   r   r   r   rJ   rM   r   r   Zsagemaker_inputsfileilinelZsagemaker_metricsZmetric_dictr!   r!   r"   prepare_sagemager_args_inputs  s    



"


"


r   c                 C   s6   dd t j| ddD }|t| d|S )z
    Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
    caller to set it in os.environ.
    c                 S   s   g | ]}t |d kr|qS )r   )r|   )rz   pr!   r!   r"   
<listcomp>  s      z$env_var_path_add.<locals>.<listcomp> r]   )r?   r@   getr   r9   rB   join)Zenv_var_nameZpath_to_addpathsr!   r!   r"   ri     s    ri   c                   @   s"   e Zd ZdZd	ddZdd ZdS )
PrepareForLaunchai  
    Prepare a function that will launched in a distributed setup.

    Args:
        launcher (`Callable`):
            The function to launch.
        distributed_type ([`~state.DistributedType`]):
            The distributed type to prepare for.
        debug (`bool`, *optional*, defaults to `False`):
            Whether or not this is a debug launch.
    NOFc                 C   s   || _ t|| _|| _d S )N)launcherr   r   rD   )selfr   r   rD   r!   r!   r"   __init__  s    
zPrepareForLaunch.__init__c                 G   s   | j rBttjd}tjd}tjjd|tj|||d nb| j	t
jt
jt
jt
jfkrt|tjd< ttjdd}ttjdd	}t|| | tjd
< tdtjd< | j|  d S )NZ
WORLD_SIZEZACCELERATE_DEBUG_RDV_FILEZgloo)Zrankstore
world_sizeZ
LOCAL_RANKZNPROCr   Z	NODE_RANKr   ZRANKZFORK_LAUNCHED)rD   rb   r?   r@   r   torchdistributedZinit_process_groupZ	FileStorer   r   Z	MULTI_GPUZ	MULTI_NPUZ	MULTI_XPUZ	MULTI_CPUrB   r   )r   r   r   r   Zrdv_fileZnprocrX   r!   r!   r"   __call__
  s*    zPrepareForLaunch.__call__N)r   F)__name__
__module____qualname____doc__r   r   r!   r!   r!   r"   r     s   
r   )F))r}   r?   r:   astr   typingr   r   r   r   r   Zcommands.config.config_argsr	   Zcommands.config.config_utilsr
   utilsr   r   r   r   r   Zutils.constantsr   Zutils.otherr   r   dataclassesr   r   r#   	NamespacerB   rU   rf   rl   boolrp   r   r   ri   r   r!   r!   r!   r"   <module>   s:   &9g&r  
 + o
