U
    9%ec                     @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZmZmZmZmZmZmZ ddlZddlmZmZmZmZmZmZmZmZmZmZ e rddlZdd Z dLe!e"d	d
dZ#e!dddZ$G dd dZ%G dd deZ&G dd deZ'G dd deZ(dZ)e*de) d Z+dd Z,G dd deZ-G dd deZ.G d d! d!eZ/G d"d# d#eZ0ee1e2f e2d$d%d&Z3ee1e2f d'd(d)Z4ee1e2f d'd*d+Z5d,d- Z6ee1e2f d'd.d/Z7G d0d1 d1eZ8d2d3 Z9d4d5 Z:dMd6d7Z;G d8d9 d9eZ<G d:d; d;Z=d<d= Z>d>d? Z?d@dA Z@G dBdC dCeZAdNeBe!e"dEdFdGZCG dHdI dIeZDG dJdK dKZEdS )Ozc
Utilities for the Trainer and TFTrainer class. Should be independent from PyTorch and TensorFlow.
    N)AnyDictList
NamedTupleOptionalTupleUnion   )
ExplicitEnumis_psutil_availableis_tf_availableis_torch_availableis_torch_cuda_availableis_torch_mps_availableis_torch_npu_availableis_torch_tpu_availableis_torch_xpu_availablerequires_backendsc                 C   s   t  d }t| dS )zN
    Helper function to set worker seed during Dataloader initialization.
    l        N)torchZinitial_seedset_seed)_Zworker_seed r   Y/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/trainer_utils.pyseed_worker2   s    r   F)seed	warn_onlyc                 C   sb   t |  t rDdtjd< dtjd< tjd|d dtjj_dtjj_	t
 r^dd	l}|jj  d	S )
a	  
    Helper function for reproducible behavior during distributed training. See
    - https://pytorch.org/docs/stable/notes/randomness.html for pytorch
    - https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_op_determinism for tensorflow
    1ZCUDA_LAUNCH_BLOCKINGz:16:8ZCUBLAS_WORKSPACE_CONFIGT)r   Fr   N)r   r   osenvironr   Zuse_deterministic_algorithmsbackendsZcudnnZdeterministicZ	benchmarkr   
tensorflowconfigZexperimentalZenable_op_determinism)r   r   tfr   r   r   enable_full_determinism:   s    



r#   )r   c                 C   st   t |  tj |  t r2t|  tj|  t rDtj	|  t
 rVtj|  t rpddl}|j |  dS )z
    Helper function for reproducible behavior to set the seed in `random`, `numpy`, `torch` and/or `tf` (if installed).

    Args:
        seed (`int`): The seed to set.
    r   N)randomr   npr   r   Zmanual_seedcudaZmanual_seed_allr   Znpur   xpur   r    r   )r   r"   r   r   r   r   U   s    

r   c                   @   sh   e Zd ZdZd
eejeej f eejeej f eeejeej f  dddZ	dd Z
dd	 ZdS )EvalPredictiona  
    Evaluation output (always contains labels), to be used to compute metrics.

    Parameters:
        predictions (`np.ndarray`): Predictions of the model.
        label_ids (`np.ndarray`): Targets to be matched.
        inputs (`np.ndarray`, *optional*)
    Npredictions	label_idsinputsc                 C   s   || _ || _|| _d S Nr)   )selfr*   r+   r,   r   r   r   __init__v   s    zEvalPrediction.__init__c                 C   s2   | j d k	rt| j| j| j fS t| j| jfS d S r-   )r,   iterr*   r+   r.   r   r   r   __iter__   s    
zEvalPrediction.__iter__c                 C   s`   |dk s|dkrt d|dkr2| jd kr2t d|dkr@| jS |dkrN| jS |dkr\| jS d S )Nr      ztuple index out of ranger	   )
IndexErrorr,   r*   r+   )r.   idxr   r   r   __getitem__   s    zEvalPrediction.__getitem__)N)__name__
__module____qualname____doc__r   r%   ndarrayr   r   r/   r2   r6   r   r   r   r   r(   l   s    
r(   c                   @   sb   e Zd ZU eejeej f ed< eeejeej f  ed< ee	e
ef  ed< ee ed< dS )EvalLoopOutputr*   r+   metricsnum_samplesN)r7   r8   r9   r   r%   r;   r   __annotations__r   r   strfloatintr   r   r   r   r<      s   
r<   c                   @   sV   e Zd ZU eejeej f ed< eeejeej f  ed< ee	e
ef  ed< dS )PredictionOutputr*   r+   r=   N)r7   r8   r9   r   r%   r;   r   r?   r   r   r@   rA   r   r   r   r   rC      s   
rC   c                   @   s.   e Zd ZU eed< eed< eeef ed< dS )TrainOutputZglobal_stepZtraining_lossr=   N)r7   r8   r9   rB   r?   rA   r   r@   r   r   r   r   rD      s   
rD   
checkpoint^z\-(\d+)$c                    sF   t  } fdd|D }t|dkr,d S t j t|dd dS )Nc                    s4   g | ],}t |d k	rtjtj |r|qS r-   )_re_checkpointsearchr   pathisdirjoin).0rI   folderr   r   
<listcomp>   s    z'get_last_checkpoint.<locals>.<listcomp>r   c                 S   s   t t|  d S )Nr   )rB   rG   rH   groups)xr   r   r   <lambda>       z%get_last_checkpoint.<locals>.<lambda>)key)r   listdirlenrI   rK   max)rN   contentZcheckpointsr   rM   r   get_last_checkpoint   s    

rY   c                   @   s   e Zd ZdZdZdZdS )IntervalStrategynostepsepochNr7   r8   r9   NOZSTEPSEPOCHr   r   r   r   rZ      s   rZ   c                   @   s   e Zd ZdZdZdZdS )EvaluationStrategyr[   r\   r]   Nr^   r   r   r   r   ra      s   ra   c                   @   s   e Zd ZdZdZdZdZdS )HubStrategyendZ
every_saverE   Zall_checkpointsN)r7   r8   r9   ZENDZ
EVERY_SAVE
CHECKPOINTZALL_CHECKPOINTSr   r   r   r   rb      s   rb   c                   @   sN   e Zd ZU dZeed< eeee f ed< e	ee
f ed< dZee
 ed< dS )BestRunac  
    The best run found by a hyperparameter search (see [`~Trainer.hyperparameter_search`]).

    Parameters:
        run_id (`str`):
            The id of the best run (if models were saved, the corresponding checkpoint will be in the folder ending
            with run-{run_id}).
        objective (`float`):
            The objective that was obtained for this run.
        hyperparameters (`Dict[str, Any]`):
            The hyperparameters picked to get this run.
        run_summary (`Optional[Any]`):
            A summary of tuning experiments. `ray.tune.ExperimentAnalysis` object for Ray backend.
    Zrun_id	objectiveZhyperparametersNrun_summary)r7   r8   r9   r:   r@   r?   r   rA   r   r   r   rg   r   r   r   r   r   re      s
   
re   )r=   returnc                 C   sf   t | } | dd}| dd}dd |  D }|D ]}| |d}q8t| dkrZ|S t|  S )aj  
    The default objective to maximize/minimize when doing an hyperparameter search. It is the evaluation loss if no
    metrics are provided to the [`Trainer`], the sum of all metrics otherwise.

    Args:
        metrics (`Dict[str, float]`): The metrics returned by the evaluate method.

    Return:
        `float`: The objective to minimize or maximize
    Z	eval_lossNr]   c                 S   s.   g | ]&}| d s&| ds&| dr|qS )_runtimeZ_per_secondZ_compilation_time)endswithrL   mr   r   r   rO      s
   
 
 
z-default_compute_objective.<locals>.<listcomp>r   )copydeepcopypopkeysrV   sumvalues)r=   Zlossr   speed_metricssmr   r   r   default_compute_objective   s    
ru   )rh   c                 C   s\   ddl m} | std| jddddd| d	dd
| ddd| ddddddgdS )Nr	   )is_optuna_availablez:This function needs Optuna installed: `pip install optuna`learning_rateư>-C6?T)lognum_train_epochs   r   (   per_device_train_batch_size             @   rw   r{   r   r~   )integrationsrv   AssertionErrorZsuggest_floatZsuggest_intZsuggest_categorical)trialrv   r   r   r   default_hp_space_optuna   s    r   c              
   C   sd   ddl m} | stdddlm} |dd|ttdd|	dd	|d
ddddgdS )Nr	   )is_ray_tune_availablez:This function needs ray installed: `pip install ray[tune]`r   )tunerx   ry      r}   r   r   r   r   r   r   )
r   r   r   rayr   Z
loguniformchoicelistrangeuniform)r   r   r   r   r   r   default_hp_space_ray  s    

r   c                 C   sJ   ddddddddd	dd
dddddddddddddgdddgS )Nrx   ry   )minrW   rw   doublerz   )boundsnametypeZtransformamtionr	   r   r{   rB   )r   r   r   r}   r   48Z16Z32Z64r~   Zcategorical)Zcategorical_valuesr   r   r   )r   r   r   r   default_hp_space_sigopt  s    r   c                 C   s\   ddl m} | stddddddd	d
dddddddddddddddgiddS )Nr	   )is_wandb_availablez8This function needs wandb installed: `pip install wandb`r$   rf   Zminimize)r   Zgoalr   rx   ry   )distributionr   rW   Zint_uniformr   r}   rr   r   r   r   r   r   r   )methodZmetric
parameters)r   r   ImportError)r   r   r   r   r   default_hp_space_wandb  s    


r   c                   @   s   e Zd ZdZdZdZdZdS )HPSearchBackendZoptunar   ZsigoptZwandbN)r7   r8   r9   ZOPTUNAZRAYZSIGOPTZWANDBr   r   r   r   r   1  s   r   c                 C   s0   t ddr(ddlm  m} | dkS | dkS )z
    Whether or not the current process is the local process, based on `xm.get_ordinal()` (for TPUs) first, then on
    `local_rank`.
    TZcheck_devicer   N)r   )r   torch_xla.core.xla_modelcore	xla_modelZget_ordinal)
local_rankxmr   r   r   is_main_process8  s    
r   c                 C   sH   t ddr$ddlm  m} | S | dkrDt rDddl}|j S dS )zg
    Return the number of processes launched in parallel. Works with `torch.distributed` and TPUs.
    Tr   r   Nr   r	   )	r   r   r   r   Zxrt_world_sizer   r   distributedZget_world_size)r   r   r   r   r   r   total_processes_numberD  s    

r   c           
      C   s   t   | }|  dt|di}|dkr,|S |dk	rP|| }t|d||  d< |dk	rt|| }t|d||  d< |dk	r|| }	t|	d||  d< |S )	a  
    Measure and return speed performance metrics.

    This function requires a time snapshot `start_time` before the operation to be measured starts and this function
    should be run immediately after the operation to be measured has completed.

    Args:
    - split: name to prefix metric (like train, eval, test...)
    - start_time: operation start time
    - num_samples: number of samples processed
    - num_tokens: number of tokens processed
    ri   r   r   N   Z_samples_per_secondZ_steps_per_secondZ_tokens_per_second)timeround)
split
start_timer>   Z	num_stepsZ
num_tokensZruntimeresultZsamples_per_secondZsteps_per_secondZtokens_per_secondr   r   r   rs   S  s    rs   c                   @   s,   e Zd ZdZdZdZdZdZdZdZ	dZ
d	S )
SchedulerTypeZlinearZcosineZcosine_with_restartsZ
polynomialZconstantZconstant_with_warmupZinverse_sqrtZreduce_lr_on_plateauN)r7   r8   r9   ZLINEARZCOSINEZCOSINE_WITH_RESTARTSZ
POLYNOMIALZCONSTANTZCONSTANT_WITH_WARMUPZINVERSE_SQRTZREDUCE_ON_PLATEAUr   r   r   r   r   p  s   r   c                   @   sd   e Zd ZdZddddddZddd	Zd
d Zdd Zdd Zdd Z	dd Z
dd ZdddZdS )TrainerMemoryTrackera  
    A helper class that tracks cpu and gpu memory.

    This class will silently skip unless `psutil` is available. Install with `pip install psutil`.

    When a stage completes, it can pass metrics dict to update with the memory metrics gathered during this stage.

    Example :

    ```python
    self._memory_tracker = TrainerMemoryTracker(self.args.skip_memory_metrics)
    self._memory_tracker.start()
    # code ...
    metrics = {"train_runtime": 10.5}
    self._memory_tracker.stop_and_update_metrics(metrics)
    ```

    At the moment GPU tracking is only for `pytorch`, but can be extended to support `tensorflow`.

    To understand this class' intricacies please read the documentation of [`~Trainer.log_metrics`].
    inittrainevaltest)r/   r   Z_inner_training_loopevaluateZpredictFc                 C   s   || _ t sd| _ | j rd S dd l}t r@dd l}|| _i | _n>t r\dd l}|| _i | _n"t rxdd l}|| _i | _nd | _| | _	d | _
i | _d| _d S )NTr   F)skip_memory_metricsr   psutilr   r   gpur   r   Processprocess	cur_stagecpuinit_reported)r.   r   r   r   r   r   r   r/     s.    
zTrainerMemoryTracker.__init__c                 C   sB   t  jjjj}|| jkr$| j| S td| d| j  dS )z+derives the stage/caller name automaticallyzwas called from z+, but only expect to be called from one of N)inspectcurrentframef_backf_codeco_namestages
ValueErrorrp   )r.   Zcallerr   r   r   derive_stage  s    

z!TrainerMemoryTracker.derive_stagec                 C   s   | j  jS )z4get resident set size memory for the current process)r   Zmemory_infoZrssr1   r   r   r   cpu_mem_used  s    z!TrainerMemoryTracker.cpu_mem_usedc                 C   s&   d| _ t|  | j | _ | jsq"qd S )Nr   )cpu_mem_used_peakrW   r   peak_monitoringr1   r   r   r   peak_monitor_func  s    z&TrainerMemoryTracker.peak_monitor_funcc                 C   s   | j r
dS |  }| jdk	r*| j|kr*dS || _t  | jdk	rtj rf| jj  | jj	  nt
 r| jj  | jj	  | jdk	rtj r| jj | _nt
 r| jj | _|  | _d| _tj| jd}d|_|  dS )z%start tracking for the caller's stageNT)target)r   r   r   gccollectr   r&   is_availableZreset_peak_memory_statsempty_cacher   r'   memory_allocatedgpu_mem_used_at_startr   cpu_mem_used_at_startr   	threadingThreadr   daemonstart)r.   stageZpeak_monitor_threadr   r   r   r     s0    




zTrainerMemoryTracker.startc                 C   s,  | j dk	r| j |krdS d| _t  | jdk	rZtj rH| jj  nt rZ| jj	  | jdk	rtj r| jj
 | _| jj | _n,t r| jj	
 | _| jj	 | _ntd| j| j| j| j td| j| j d| j| j < |  | _| j| j| j| j td| j| j d| j| j < d| _ dS )z"stop tracking for the passed stageNFzNo available GPU device found!r   )beginrc   allocpeaked)r   r   r   r   r   r&   r   r   r   r'   r   Zgpu_mem_used_nowZmax_memory_allocatedZgpu_mem_used_peakr   r   rW   r   r   Zcpu_mem_used_nowr   r   r   )r.   r   r   r   r   stop  s:    






zTrainerMemoryTracker.stopc                 C   s  | j r
dS | jdk	r"| j|kr"dS |g}| js@|dd d| _|D ]}dD ]z}|| jkr|| j| kr| j| | || d| d< | jdk	rL|| jkrL|| j| krL| j| | || d| d< qLqD|d dkr| jd d	 |d
< | jdk	r| jd d	 |d< dS )zupdates the metricsNr   r   T)r   r   Z	_mem_cpu_Z_deltaZ	_mem_gpu_r   Zbefore_init_mem_cpuZbefore_init_mem_gpu)r   r   r   insertr   r   r   )r.   r   r=   r   tr   r   r   update_metrics4  s$    ""
z#TrainerMemoryTracker.update_metricsNc                 C   s4   | j r
dS |  }| | |dk	r0| || dS )z<combine stop and metrics update in one call for simpler codeN)r   r   r   r   )r.   r=   r   r   r   r   stop_and_update_metrics]  s    
z,TrainerMemoryTracker.stop_and_update_metrics)F)N)r7   r8   r9   r:   r   r/   r   r   r   r   r   r   r   r   r   r   r   r   {  s   
$
%6)r   c                 C   s*   zt | dk	W S  tk
r$   Y dS X dS )zR
    Checks if the dataset implements __len__() and it doesn't raise an error
    NF)rV   	TypeError)Zdatasetr   r   r   
has_lengthj  s    r   c                 C   s   t | ttfr$t| dd | D S t | trHt| dd |  D S t | tjr\|  S t	 rt | t
jr|  dkr|  S | S )zM
    Recursively calls `.item()` on the element of the dictionary passed
    c                 s   s   | ]}t |V  qd S r-   denumpify_detensorizerk   r   r   r   	<genexpr>z  s     z(denumpify_detensorize.<locals>.<genexpr>c                 S   s   i | ]\}}|t |qS r   r   rL   kvr   r   r   
<dictcomp>|  s      z)denumpify_detensorize.<locals>.<dictcomp>r	   )
isinstancer   tupler   dictitemsr%   Zgenericitemr   r   ZTensorZnumel)r=   r   r   r   r   u  s    
r   c                 C   sF   t | tjr6tt| jj}|t| j t| j	 S tt| jS )za
    Return the number of arguments of the passed function, even if it's a partial function.
    )
r   	functoolspartialrV   r   	signaturefuncr   argskeywords)r   Z
total_argsr   r   r   number_of_arguments  s    r   c                   @   s    e Zd ZdZdZdZdZdZdS )ShardedDDPOptionsimpleZ	zero_dp_2Z	zero_dp_3offload	auto_wrapN)r7   r8   r9   SIMPLEZ	ZERO_DP_2Z	ZERO_DP_3OFFLOAD	AUTO_WRAPr   r   r   r   r     s
   r      )functionstarting_batch_sizeauto_find_batch_sizec                 C   sL   | dkrt jt||dS |r>ttd ddlm} || |dS t j| |dS )a%  
    Args:
    A basic decorator that will try to execute `function`. If it fails from exceptions related to out-of-memory or
    CUDNN, the batch size is cut in half and passed to `function`. `function` must take in a `batch_size` parameter as
    its first argument.
        function (`callable`, *optional*)
            A function to wrap
        starting_batch_size (`int`, *optional*)
            The batch size to try and fit into memory
        auto_find_batch_size (`bool`, *optional*)
            If False, will just execute `function`
    N)r   r   Z
accelerater   )find_executable_batch_size)r   r   )Z
batch_size)r   r   r  r   Zaccelerate.utils)r   r   r   Z%accelerate_find_executable_batch_sizer   r   r   r    s    
r  c                   @   s    e Zd ZdZdZdZdZdZdS )
FSDPOptionZ
full_shardZshard_grad_opZno_shardr   r   N)r7   r8   r9   Z
FULL_SHARDZSHARD_GRAD_OPZNO_SHARDr   r   r   r   r   r   r    s
   r  c                   @   sL   e Zd ZdZdee ee dddZeedddZe	e d	d
dZ
dS )RemoveColumnsCollatorzWWrap the data collator to remove unused columns before they are passed to the collator.N)
model_namedescriptionc                 C   s(   || _ || _|| _|| _|| _d| _d S )NF)data_collatorsignature_columnsloggerr  r  message_logged)r.   r  r  r  r  r  r   r   r   r/     s    zRemoveColumnsCollator.__init__)featurerh   c                    s   t |ts|S  js jr jrtt| t j }t	|dkr j
d krTdnd j
 d} jd| d j dd| d	d| d
 j d d _ fdd| D S )Nr    zin the z setzThe following columns z) don't have a corresponding argument in `z!.forward` and have been ignored: z, z. If z are not expected by `z/.forward`,  you can safely ignore this message.Tc                    s    i | ]\}}| j kr||qS r   )r  r   r1   r   r   r     s     
  z9RemoveColumnsCollator._remove_columns.<locals>.<dictcomp>)r   r   r	  r  r  r   setrp   r  rV   r  inforK   r   )r.   r
  Zignored_columnsZdset_descriptionr   r1   r   _remove_columns  s    
2z%RemoveColumnsCollator._remove_columns)featuresc                    s    fdd|D }  |S )Nc                    s   g | ]}  |qS r   )r  )rL   r
  r1   r   r   rO     s     z2RemoveColumnsCollator.__call__.<locals>.<listcomp>)r  )r.   r  r   r1   r   __call__  s    zRemoveColumnsCollator.__call__)NNN)r7   r8   r9   r:   r   r@   r/   r   r  r   r  r   r   r   r   r    s      r  )F)NNN)Nr   F)Fr:   rm   r   r   r   r   r$   rer   r   typingr   r   r   r   r   r   r   numpyr%   utilsr
   r   r   r   r   r   r   r   r   r   r   r   rB   boolr#   r   r(   r<   rC   rD   ZPREFIX_CHECKPOINT_DIRcompilerG   rY   rZ   ra   rb   re   r@   rA   ru   r   r   r   r   r   r   r   rs   r   r   r   r   r   r   callabler  r  r  r   r   r   r   <module>   sl   $0'
 p
	       