U 9%e{@sDdZddlZddlZddlmZddlmZmZmZm Z m Z ddl Z ddl m Z ddl mZddlmZmZdd lmZdd lmZdd lmZeeZdEd d ZdFeedddZedddZeedddZdGeeedddZ eeedddZ!dHddZ"eeee#d d!d"Z$dIeeee#ed$d%d&Z%eeeed d'd(Z&dJeeeeed$d)d*Z'eeee#e#ed+d,d-Z(dKd0d1Z)dd2eeed3d4d5Z*dLeeeed6d7d8Z+ej,e"ej-e%ej.e'ej/e)ej0eej1e ej2e+ej3eiZ4dMe e5efeeeeed9d:d;Z6Gdd?d?eZ8Gd@dAdAeZ9dNdCdDZ:dS)Oz$PyTorch optimization for BERT model.N)partial)CallableIterableOptionalTupleUnion)nn) Optimizer)LambdaLRReduceLROnPlateau) SchedulerType)logging)require_versioncCsdSNr _rrX/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/optimization.py_get_constant_lambda#sr optimizer last_epochcCst|t|dS)a Create a schedule with a constant learning rate, using the learning rate set in optimizer. Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. last_epoch (`int`, *optional*, defaults to -1): The index of the last epoch when resuming training. Return: `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. r)r rrrrrget_constant_schedule'srrcCst|S)aL Create a schedule with a constant learning rate that decreases when a metric has stopped improving. Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. Return: `torch.optim.lr_scheduler.ReduceLROnPlateau` with the appropriate schedule. )r rrrrget_reduce_on_plateau_schedule8s r current_stepnum_warmup_stepscCs"||krt|ttd|SdS)N?floatmaxrrrr,_get_constant_schedule_with_warmup_lr_lambdaGsr%)rr rcCstt|d}t|||dS)ad Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate increases linearly between 0 and the initial lr set in the optimizer. Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. num_warmup_steps (`int`): The number of steps for the warmup phase. last_epoch (`int`, *optional*, defaults to -1): The index of the last epoch when resuming training. Return: `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. r r)rr%r )rr r lr_lambdarrr!get_constant_schedule_with_warmupMs r(rr num_training_stepscCsB||krt|ttd|Stdt||ttd||S)Nr r"r)rrr*_get_linear_schedule_with_warmup_lr_lambdabsr,cCstt||d}t|||S)a Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer. Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. num_warmup_steps (`int`): The number of steps for the warmup phase. num_training_steps (`int`): The total number of training steps. last_epoch (`int`, *optional*, defaults to -1): The index of the last epoch when resuming training. Return: `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. r r*)rr,r )rr r*rr'rrrget_linear_schedule_with_warmuphs r.)rr r* num_cyclesc Csf||krt|ttd|St||ttd||}tdddttjt|d|S)Nr r+?r!g@r#r$mathcospirr r*r/progressrrr*_get_cosine_schedule_with_warmup_lr_lambdasr7r0)rr r*r/rcCstt|||d}t|||S)a Create a schedule with a learning rate that decreases following the values of the cosine function between the initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the initial lr set in the optimizer. Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. num_warmup_steps (`int`): The number of steps for the warmup phase. num_training_steps (`int`): The total number of training steps. num_cycles (`float`, *optional*, defaults to 0.5): The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0 following a half-cosine). last_epoch (`int`, *optional*, defaults to -1): The index of the last epoch when resuming training. Return: `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. r r*r/)rr7r rr r*r/rr'rrrget_cosine_schedule_with_warmupsr:c Csr||krt|ttd|St||ttd||}|dkrHdStdddttjt||dS)Nr r!r+r0r1r5rrr=_get_cosine_with_hard_restarts_schedule_with_warmup_lr_lambdas r;cCstt|||d}t|||S)a Create a schedule with a learning rate that decreases following the values of the cosine function between the initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases linearly between 0 and the initial lr set in the optimizer. Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. num_warmup_steps (`int`): The number of steps for the warmup phase. num_training_steps (`int`): The total number of training steps. num_cycles (`int`, *optional*, defaults to 1): The number of hard restarts to use. last_epoch (`int`, *optional*, defaults to -1): The index of the last epoch when resuming training. Return: `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. r8)rr;r r9rrr2get_cosine_with_hard_restarts_schedule_with_warmupsr<)rr r*lr_endpowerlr_initc Csj||krt|ttd|S||kr.||S||}||}d|||}||||} | |SdSrr") rr r*r=r>r?Zlr_rangeZ decay_stepsZ pct_remainingdecayrrr4_get_polynomial_decay_schedule_with_warmup_lr_lambdas rAHz>r!cCsH|jd}||ks(td|d|dtt|||||d}t|||S)a Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the optimizer to end lr defined by *lr_end*, after a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer. Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. num_warmup_steps (`int`): The number of steps for the warmup phase. num_training_steps (`int`): The total number of training steps. lr_end (`float`, *optional*, defaults to 1e-7): The end LR. power (`float`, *optional*, defaults to 1.0): Power factor. last_epoch (`int`, *optional*, defaults to -1): The index of the last epoch when resuming training. Note: *power* defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT implementation at https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37 Return: `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. lrzlr_end (z&) must be be smaller than initial lr ())r r*r=r>r?)defaults ValueErrorrrAr )rr r*r=r>rr?r'rrr)get_polynomial_decay_schedule_with_warmups rG) timescale)rr rHcCs@||krt|ttd|S||}dt|||}|S)Nr r!)r#r$r2sqrt)rr rHshiftr@rrr$_get_inverse_sqrt_schedule_lr_lambdas rK)rr rHrcCs(|dkr |}tt||d}t|||dS)a Create a schedule with an inverse square-root learning rate, from the initial lr set in the optimizer, after a warmup period which increases lr linearly from 0 to the initial lr set in the optimizer. Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. num_warmup_steps (`int`): The number of steps for the warmup phase. timescale (`int`, *optional*, defaults to `num_warmup_steps`): Time scale. last_epoch (`int`, *optional*, defaults to -1): The index of the last epoch when resuming training. Return: `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. N)r rHr)rrKr )rr rHrr'rrrget_inverse_sqrt_schedule%srL)namerr r*cCst|}t|}|tjks$|tjkr,||S|dkrBt|d|tjkrX|||dS|tjkrn|||dS|dkrt|d||||dS)a Unified API to get any scheduler from its name. Args: name (`str` or `SchedulerType`): The name of the scheduler to use. optimizer (`torch.optim.Optimizer`): The optimizer that will be used during training. num_warmup_steps (`int`, *optional*): The number of warmup steps to do. This is not required by all schedulers (hence the argument being optional), the function will raise an error if it's unset and the scheduler type requires it. num_training_steps (`int``, *optional*): The number of training steps to do. This is not required by all schedulers (hence the argument being optional), the function will raise an error if it's unset and the scheduler type requires it. Nz; requires `num_warmup_steps`, please provide that argument.r&z= requires `num_training_steps`, please provide that argument.r-)r TYPE_TO_SCHEDULER_FUNCTIONCONSTANTREDUCE_ON_PLATEAUrFCONSTANT_WITH_WARMUP INVERSE_SQRT)rMrr r*Z schedule_funcrrr get_schedulerOs    rSc s\eZdZdZdeejjee eefeee e dfd d Z e ded d dZZS)AdamWa5 Implements Adam algorithm with weight decay fix as introduced in [Decoupled Weight Decay Regularization](https://arxiv.org/abs/1711.05101). Parameters: params (`Iterable[nn.parameter.Parameter]`): Iterable of parameters to optimize or dictionaries defining parameter groups. lr (`float`, *optional*, defaults to 1e-3): The learning rate to use. betas (`Tuple[float,float]`, *optional*, defaults to (0.9, 0.999)): Adam's betas parameters (b1, b2). eps (`float`, *optional*, defaults to 1e-6): Adam's epsilon for numerical stability. weight_decay (`float`, *optional*, defaults to 0): Decoupled weight decay to apply. correct_bias (`bool`, *optional*, defaults to `True`): Whether or not to correct bias in Adam (for instance, in Bert TF repository they use `False`). no_deprecation_warning (`bool`, *optional*, defaults to `False`): A flag used to disable the deprecation warning (set to `True` to disable the warning). MbP?g?g+?ư>r+TF)paramsrCbetaseps weight_decay correct_biasno_deprecation_warningc s|stdttd|dkr0td|dd|dkrHdks^ntd|dd d|d krvdksntd|d d d|kstd |d|||||d }t||dS) NzThis implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning torch>=1.5.0r+zInvalid learning rate: z - should be >= 0.0rr!zInvalid beta parameter: z - should be in [0.0, 1.0)r zInvalid epsilon value: )rCrYrZr[r\)warningswarn FutureWarningrrFsuper__init__) selfrXrCrYrZr[r\r]rE __class__rrrcs zAdamW.__init__N)closurecCs~d}|dk r|}|jD]^}|dD]N}|jdkr8q&|j}|jrLtd|j|}t|dkrd|d<t||d<t||d<|d|d}}|d\} } |dd 7<|| j |d | d || j ||d | d | |d } |d} |dr.lr_lambdar)rnrbrc)rdrrr'ryrerrrcs     zAdafactorSchedule.__init__cs0|jfddjD}t|dkr,|j}|S)Ncs8g|]0}|ddjdk r|j|ddqS)rXrN)rorrr).0ryoptrr sz,AdafactorSchedule.get_lr..r)rrnrsZbase_lrs)rdZlrsrrrget_lrs  zAdafactorSchedule.get_lr)r+)r}r~rrrcrrrrrerrs rr+cCs t||S)aX Get a proxy schedule for [`~optimization.Adafactor`] Args: optimizer ([`~torch.optim.Optimizer`]): The optimizer for which to schedule the learning rate. initial_lr (`float`, *optional*, defaults to 0.0): Initial lr Return: [`~optimization.Adafactor`] proxy schedule object. )r)rrrrrget_adafactor_schedulesr)N)r)r)r)r0r)r r)rBr!r)Nr)NN)r+);rr2r_ functoolsrtypingrrrrrrtrZ torch.optimr Ztorch.optim.lr_schedulerr r Z trainer_utilsr utilsrZutils.versionsrZ get_loggerr}loggerrintrrr%r(r,r.r#r7r:r;r<rArGrKrLZLINEARZCOSINEZCOSINE_WITH_RESTARTSZ POLYNOMIALrOrQrRrPrNstrrSrTrrrrrrrs           #  "  .    +rr