U
    0-e&                     @   s   d dl Z d dlZddlmZ ddlmZmZmZ ddlm	Z	 ddl
mZ edere	 rd dlm  mZ d d	lmZmZ d d
lmZ d dlmZ d dlmZ eeZdddZdddZdddZdddZdS )    N   )
get_logger   )FSDP_PYTORCH_VERSION
MODEL_NAMEOPTIMIZER_NAME)is_torch_distributed_available)is_torch_versionz>=)DefaultLoadPlannerDefaultSavePlanner)!load_sharded_optimizer_state_dict)FullyShardedDataParallel)StateDictTypec           
   	   C   s  t j|dd | jtjkr4|jdk}|| j_|| j_t	|| j| j| j
r | }| jtjkr|dkrtt dnt d| d}t j||}|jdkrtd|  t|| td|  n| jtjkrF|dkrt d	|j dnt d| d	|j d}t j||}td|  t|| td|  nt| jtjkrt j|t d| }	t j|	dd td|	  d
|i}tj|t|	t d td|	  W 5 Q R X d S )NTexist_okr   r   .bin_zSaving model to zModel saved to _rankmodel
state_dictZstorage_writerplanner)osmakedirsstate_dict_typer   FULL_STATE_DICTnum_processesstate_dict_configoffload_to_cpu
rank0_onlyFSDPoptim_state_dict_configr   r   pathjoinprocess_indexloggerinfotorchsaveLOCAL_STATE_DICTSHARDED_STATE_DICTdist_cpsave_state_dictFileSystemWriterr   )
fsdp_pluginacceleratorr   
output_dirmodel_indexis_multi_processr   weights_nameZoutput_model_fileckpt_dir r5   \/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/accelerate/utils/fsdp_utils.pysave_fsdp_model#   sN    
   "
r7   c              	   C   s  |   | jtjkr.|jdk}|| j_|| j_t|| j| j| j	 | jtjkrt
|tkr|jdkr| jsxtdW 5 Q R  d S |dkrt dnt d| d}tj||}td|  t|}td|  n| jtjkr^|dkrt d|j dnt d| d|j d}tj||}td|  t|}td|  n| jtjkrt |krtj|t d| n|}	td|	  d	| i}tj|t|	t d
 |d	 }td|	  ||}
W 5 Q R X |
S )Nr   r   zzSet the `sync_module_states` flag to `True` so that model states are synced across processes when initializing FSDP objectr   r   zLoading model from zModel loaded from r   r   )r   storage_readerr   )wait_for_everyoner   r   r   r   r   r   r   r    r!   typer$   Zsync_module_states
ValueErrorr   r   r"   r#   r%   r&   r'   loadr)   r*   r   r+   load_state_dictFileSystemReaderr
   )r.   r/   r   	input_dirr1   r2   r3   Zinput_model_filer   r4   Zload_resultr5   r5   r6   load_fsdp_modelP   s`    
   "

r@   c           
   	   C   s  t j|dd t|| j| j| j t||}| jtjkr|j	dkr|dkrZt
 dnt
 d| d}t j||}td|  t|| td|  nbt j|t
 d| }	t j|	dd td|	  tjd|it|	t d	 td|	  W 5 Q R X d S )
NTr   r   r   r   zSaving Optimizer state to zOptimizer state saved in 	optimizerr   )r   r   r    r   r   r!   optim_state_dictr   r   r$   r   r"   r#   r%   r&   r'   r(   r+   r,   r-   r   )
r.   r/   rA   r   r0   optimizer_indexoptim_stateZoptim_state_nameZoutput_optimizer_filer4   r5   r5   r6   save_fsdp_optimizer   s2       
 rE   c              	   C   s  |   t|| j| j| j | jtjkrd }|dkrBt dnt d| d}tj	
||}td|  t|}td|  nft |krtj	
|t d| n|}	td|	  t| dt|	d}|d }td	|	  tj|||d
}
||
 W 5 Q R X d S )Nr   r   r   zLoading Optimizer state from zOptimizer state loaded from zLoading Optimizer from rA   )Zmodel_state_dictZoptimizer_keyr8   zOptimizer loaded from )r   ZoptimrB   )r9   r    r   r   r!   r   r   r   r   r"   r#   r%   r&   r'   r<   r   r   r+   r>   Zoptim_state_dict_to_loadr=   )r.   r/   rA   r   r?   rC   rD   Zoptimizer_nameZinput_optimizer_filer4   Zflattened_osdr5   r5   r6   load_fsdp_optimizer   s:        
rF   )r   )r   )r   )r   )r   r'   loggingr   	constantsr   r   r   Zimportsr   versionsr	   Ztorch.distributed.checkpointdistributed
checkpointr+   Z,torch.distributed.checkpoint.default_plannerr
   r   Z&torch.distributed.checkpoint.optimizerr   Z2torch.distributed.fsdp.fully_sharded_data_parallelr   r    r   __name__r%   r7   r@   rE   rF   r5   r5   r5   r6   <module>   s    
-
5
