U
    0-e                     @   sp  d dl mZmZmZmZmZmZmZmZm	Z	m
Z
mZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z" d dl#m$Z$m%Z%m&Z&m'Z' d dl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZC d dlDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZY d dlZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZa d dlbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZw d dlxmyZymzZz e4 r4d d	l{m|Z|m}Z}m~Z~mZmZmZ d d
lmZmZ d dlmZmZmZmZ d dlmZmZmZmZmZmZmZ d dlmZmZmZmZmZmZmZmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlmZmZmZ d dlmZ d dlmZ d dlmZmZ dS )   )
MODEL_NAMEOPTIMIZER_NAMERNG_STATE_NAMESAFE_WEIGHTS_INDEX_NAMESAFE_WEIGHTS_NAMESCALER_NAMESCHEDULER_NAME!TORCH_DISTRIBUTED_OPERATION_TYPESTORCH_LAUNCH_PARAMSWEIGHTS_INDEX_NAMEWEIGHTS_NAME)AutocastKwargsBnbQuantizationConfigComputeEnvironmentCustomDtypeDeepSpeedPluginDistributedDataParallelKwargsDistributedTypeDynamoBackendFP8RecipeKwargsFullyShardedDataParallelPluginGradientAccumulationPluginGradScalerKwargsInitProcessGroupKwargsKwargsHandler
LoggerTypeMegatronLMPluginPrecisionTypeProjectConfigurationRNGTypeSageMakerDistributedTypeTensorInformationTorchDynamoPlugin)get_int_from_envparse_choice_from_envparse_flag_from_envstr_to_bool)get_ccl_versionis_4bit_bnb_availableis_8bit_bnb_availableis_aim_availableis_bf16_availableis_bnb_availableis_boto3_availableis_ccl_availableis_comet_ml_availableis_cuda_availableis_datasets_availableis_deepspeed_availableis_fp8_availableis_ipex_availableis_megatron_lm_availableis_mlflow_availableis_mps_availableis_npu_availableis_rich_availableis_safetensors_availableis_sagemaker_availableis_tensorboard_availableis_timm_availableis_tpu_availableis_transformers_availableis_wandb_availableis_xpu_available)calculate_maximum_sizescheck_device_mapcheck_tied_parameters_in_config$check_tied_parameters_on_same_devicecompute_module_sizesconvert_file_size_to_intdtype_byte_sizefind_tied_parametersget_balanced_memoryget_max_layer_sizeget_max_memory#get_mixed_precision_context_managerid_tensor_storageinfer_auto_device_mapload_checkpoint_in_modelload_offloaded_weightsload_state_dictnamed_module_tensorsretie_parametersset_module_tensor_to_deviceshard_checkpoint)OffloadedWeightsLoaderPrefixedDatasetextract_submodules_state_dictload_offloaded_weightoffload_state_dictoffload_weightsave_offload_index)	broadcastbroadcast_object_listconcatenateconvert_outputs_to_fp32convert_to_fp32find_batch_sizefind_devicegathergather_objectget_data_structure
honor_typeinitialize_tensorsis_namedtupleis_tensor_informationis_torch_tensorlistifypad_across_processesrecursively_applyreducesend_to_deviceslice_tensors)compare_versionsis_torch_version)DeepSpeedEngineWrapperDeepSpeedOptimizerWrapperDeepSpeedSchedulerWrapper
DummyOptimDummySchedulerHfDeepSpeedConfig)has_4bit_bnb_layersload_and_quantize_model)load_fsdp_modelload_fsdp_optimizersave_fsdp_modelsave_fsdp_optimizer)PrepareForLaunch_filter_argsprepare_deepspeed_cmd_envprepare_multi_gpu_envprepare_sagemager_args_inputsprepare_simple_launcher_cmd_envprepare_tpu)AbstractTrainStepBertTrainStepGPTTrainStepMegatronEngineMegatronLMDummyDataLoaderMegatronLMDummySchedulerMegatronLMOptimizerWrapperMegatronLMSchedulerWrapperT5TrainStep%avg_losses_across_data_parallel_group"gather_across_data_parallel_groups)
initialize)prepare_data_loader)prepare_model)prepare_optimizer)prepare_scheduler)find_executable_batch_sizerelease_memory)
clear_environmentconvert_bytesextract_model_from_parallelget_pretty_nameis_port_in_usemerge_dictspatch_environmentsavewait_for_everyonewrite_basic_config)set_seedsynchronize_rng_statesynchronize_rng_states)install_xla)tqdm)convert_modelhas_transformer_engine_layersN)	constantsr   r   r   r   r   r   r   r	   r
   r   r   dataclassesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   environmentr#   r$   r%   r&   Zimportsr'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   ZmodelingrB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   ZoffloadrW   rX   rY   rZ   r[   r\   r]   
operationsr^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   versionsrs   rt   Z	deepspeedru   rv   rw   rx   ry   rz   Zbnbr{   r|   Z
fsdp_utilsr}   r~   r   r   Zlaunchr   r   r   r   r   r   r   Zmegatron_lmr   r   r   r   r   r   r   r   r   r   r   r   Zmegatron_lm_initializer   Zmegatron_lm_prepare_data_loaderr   Zmegatron_lm_prepare_modelr   Zmegatron_lm_prepare_optimizerr   Zmegatron_lm_prepare_schedulerZmemoryr   r   otherr   r   r   r   r   r   r   r   r   r   randomr   r   r   Z	torch_xlar   r   Ztransformer_enginer   r    r   r   Z/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/accelerate/utils/__init__.py<module>   s0   4`t\$	\ 	$	40