U 9%et @sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZddlmZmZmZddlZddlmZddlmZmZmZmZmZee Z!erddl"Z"ej#$d dk oe%d d &d kZ'e'rbzPddl(Z(e)e(d re(j*+drdZ'n$e%d d &d krZ>ddZ?ddZ@dd ZAd!d"ZBd#d$ZCd%d&ZDd'd(ZEd)d*ZFd+d,ZGd-d.ZHd/d0ZId1d2ZJd3d4ZKd5d6ZLd7d8ZMd9d:ZNd;d<ZOePeQe9d=d>d?ZRePeQe9d=d@dAZSePeQe9d=dBdCZTePeQe9d=dDdEZUdFdGZVdHdIZWGdJdKdKe6ZXGdLdMdMe6ZYGdNdOdOe6ZZGdPdQdQe6Z[GdRdSdSe6Z\GdTdUdUe\Z]GdVdWdWe^Z_GdXdYdYe6Z`GdZd[d[e6ZaGd\d]d]e6ZbGd^d_d_e6Zce[eZe\e`eXeYeaebe]ecd` ZddadbZedS)cz+ Integrations with other Python libraries. N)asdict)Path) TYPE_CHECKINGDictOptional) __version__) flatten_dictis_datasets_availableis_pandas_availableis_torch_availableloggingcomet_ml COMET_MODEZDISABLEDconfigz comet.api_keyTz5comet_ml is installed but `COMET_API_KEY` is not set.Fneptunezneptune-clientzNeptune version z available.zNeptune-client version )ProgressCallbackTrainerCallback)PREFIX_CHECKPOINT_DIRBestRunIntervalStrategy) ParallelMode)ENV_VARS_TRUE_VALUESis_torch_tpu_availablecCs2tddtkr"tddStjddk S)NZWANDB_DISABLEDrzUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).Fwandb) osgetenvupperrloggerwarning importlibutil find_specr$r$j/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/integrations/integration_utils.pyis_wandb_availableNs r&cCstjddk S)Nclearmlr!r"r#r$r$r$r%is_clearml_availableYsr)cCstSN) _has_cometr$r$r$r%is_comet_available]sr,cCs tjddk ptjddk S)N tensorboard tensorboardXr(r$r$r$r%is_tensorboard_availableasr/cCstjddk S)Noptunar(r$r$r$r%is_optuna_availableesr1cCstjddk S)Nrayr(r$r$r$r%is_ray_availableisr3cCsts dStjddk S)NFzray.tune)r3r!r"r#r$r$r$r%is_ray_tune_availablemsr4cCstjddk S)Nsigoptr(r$r$r$r%is_sigopt_availablessr6cCs8tjddkrdStjddkr(dStjddk S)NZazuremlFz azureml.corezazureml.core.runr(r$r$r$r%is_azureml_availablews r7cCs(tdddkrdStjddk S)NZDISABLE_MLFLOW_INTEGRATIONFALSETRUEFmlflow)rrrr!r"r#r$r$r$r%is_mlflow_availablesr;cCsdtjdtjdfkS)Ndagshubr:r(r$r$r$r%is_dagshub_availablesr=cCstjddk S)NZ fairscaler(r$r$r$r%is_fairscale_availablesr>cCstSr*) _has_neptuner$r$r$r%is_neptune_availablesr@cCstjddk S)N codecarbonr(r$r$r$r%is_codecarbon_availablesrBcCstjddk S)Nflytekitr(r$r$r$r%is_flytekit_availablesrDcCsts dStjddk S)NFzflytekitplugins.deck)rDr!r"r#r$r$r$r% is_flyte_deck_standard_availablesrEcCsptr ddl}t||jr |jStr4t|tr4|StrHt|trH|Str\t|tr\|St d|j dS)NrzUnknown type for trial ) r1r0 isinstanceZTrialparamsr4dictr6r& RuntimeError __class__)trialr0r$r$r% hp_paramss    rL)n_trials directionreturnc s~ddl}jjdkrdfdd }|dd}|dd}t|trH|nd}|dk rXdn|}|jf||d|} | j||||d| s| j } t t | j | j | jS| j} d d | DSnt|D]} d_ttj} jjtjkrtd tjj| dd tt| }t|D]"\}}|d kr tj||q j ddt!dddkrƈ"}#|_qdSdS)Nrcsd}|r0t|D]}|trtj||}qd_jjdkrjj t j krXt d |tjjtjddj|dnj||dtdddkr}|_jS)NConly support DDP optuna HPO for ParallelMode.DISTRIBUTED currently.rsrcresume_from_checkpointrUrK objective)rlistdir startswithrpathjoinrWargs world_size parallel_moder DISTRIBUTEDrI_hp_search_setuptorch distributedbroadcast_object_listpickledumpstraingetattrevaluatecompute_objective)rKcheckpoint_dir checkpointsubdirmetricstrainerr$r% _objectives"    z(run_hp_search_optuna.._objectivetimeoutn_jobsrP)rN directions)rMrqrrcSs"g|]}tt|j|j|jqSr$)rstrnumbervaluesrG).0bestr$r$r% sz(run_hp_search_optuna..rQrR local_rankrTrW)N)$r0r\ process_indexpoprFlistZ create_studyoptimizeZ_is_multi_objective best_trialrrtruvaluerG best_trialsrangerWrdrer^rr_rIrarbrcloadsbytesritemssetattrrfrgrhri)rorMrNkwargsr0rprqrrrsZstudyrriargs_main_rankr\keyrrmr$rnr%run_hp_search_optunas:      rc sZddld%fdd }|jjs@ddlm}td|dd|_|t}d|_ d |krd d i|d <|j j dkr|d |d d <d |j j dkrdnd}t d|d|d  d d}||j _d|krddlm} | dgd|d<d|kr$|ddkr$d|_|dd kr$td|ddd|krddlm} m} m} m} t|d| rh|jshtdt|d| | | | fr|j jr|j jtjkrtdjt|djdj j!||dt"#fdd }t$d!rj%|_%j j&|f|'d|d"|}|j(d|dd#|j j)d$}t*|j+|j,d|j-|}|dk rV|.||S)&Nrcsz$ddlm}||r"|tWntk r8YnXd}|rjt|D]}|t rLtj ||}qLd|_ |j ||dt|dddkr|}|||_ |jjfd|j i|ddidS)Nr)NotebookProgressCallbackrVrWdoneT)Ztransformers.utils.notebookr pop_callback add_callbackrModuleNotFoundErrorrrXrYrrZr[rWrfrgrhriZ_tune_save_checkpointtunereport)rK local_trainerrjrrkrlrm)r2r$r%rps$    z%run_hp_search_ray.._objectiver)TrainerMemoryTrackerzMemory tracking for your Trainer is currently enabled. Automatically disabling the memory tracker since the memory tracker is not serializable.T)skip_memory_metricsZresources_per_trialcpurPZgpuz1 CPUz and 1 GPUrzgNo `resources_per_trial` arg was passed into `hyperparameter_search`. Setting it to a default value of z for each trial.Zprogress_reporter) CLIReporterrW)Zmetric_columnsZkeep_checkpoints_numzCurrently keeping ze checkpoints for each trial. Checkpoints are usually huge, consider setting `keep_checkpoints_num=1`.Z scheduler) ASHASchedulerHyperBandForBOHBMedianStoppingRulePopulationBasedTraininga/You are using PopulationBasedTraining but you haven't enabled checkpointing. This means your trials will train from scratch everytime they are exploiting new configurations. Consider enabling checkpointing by passing `keep_checkpoints_num=1` as an additional argument to `Trainer.hyperparameter_search`.aYou are using {cls} as a scheduler but you haven't enabled evaluation during training. This means your trials will not report intermediate results to Ray Tune, and can thus not be stopped early or used to exploit other trials parameters. If this is what you want, do not use {cls}. If you would like to use {cls}, make sure you pass `do_eval=True` and `evaluation_strategy='steps'` in the Trainer `args`.)cls)rcs^trTddl}tj|jd}tj d|}tj |}|t j |j <|j|||S)a" Wrapper around `tune.with_parameters` to ensure datasets_modules are loaded on each Actor. Without this, an ImportError will be thrown. See https://github.com/huggingface/transformers/issues/11565. Assumes that `_objective`, defined above, is a function. rNz __init__.pydatasets_modules)r Z datasets.loadrrZr[loadZinit_dynamic_modulesr!r"spec_from_file_locationmodule_from_specsysmodulesnameloader exec_module)r\rZdatasetsZdynamic_modules_pathspecr) trainabler$r% dynamic_modules_import_trainablePs    z;run_hp_search_ray..dynamic_modules_import_trainable __mixins__)rZ num_samples)metricmodescope)N)/r2Z_memory_trackerr trainer_utilsrrr rTensorBoardCallbackmodelr\Zn_gpuinfogetZ_n_gpuZray.tunerZuse_tune_checkpointsZray.tune.schedulersrrrrrFZdo_evalZevaluation_strategyrNOrIformattype__name__rZwith_parameters functoolswrapshasattrrrunhp_spaceZget_best_trialZ ray_scoperZtrial_idZ last_resultrr)rorMrNrrprZ _tb_writerZ resource_msgZgpus_per_trialrrrrrrZanalysisrbest_runr$)r2rr%run_hp_search_rays                rc Ksddl}|jjdkrtjddkrR|d|jdd|dd|dd gd |d }t d |j | D]}|d|_ |jjd kr|jjtjkrtd ||jtjjt|jdd|jddn|jd|jdt|dddkr |}|||_ |d|j W5QRXqrt| d}t!|j |j"dj#|j$} nddlm%} | } |&dd} | dk r| '| | (j)d|dd|dd gd |dd}t d |j |j*j+|j,kr| (|j -)} d|_ |jjd kr@|jjtjkrtd || tjjt|jdd|jddn|jd| dt|dddkrt|}|||_ d|j dg}| (|j .j)| j |d}t d| j d|j d| (|j /}qt| (|j 0/1d}t!|j |j#|j$} | St2|D]}d|_ tt|j}|jjtjkr>td tjj|ddt3t4|}t5|6D]"\}}|dkrht7|j||qh|jddt|dddkr |}|||_ q dSdS)Nrr5z8.0.0 huggingfacezhuggingface-tuneZofflinerWr~)rrWZstrategyrP)rr parametersrmparallel_bandwidthZbudgetz6created experiment: https://app.sigopt.com/experiment/zConly support DDP Sigopt HPO for ParallelMode.DISTRIBUTED currently.rRrTrV) Connectionproxies)rrrmrobservation_budgetprojectrr) suggestionrvz"[suggestion_id, observation_id]: [, ]rz)8r5r\r{r!metadataversionZ set_projectZcreate_experimentrrridlooprWr]r^rr_rIr`rrarbrcrdrerfrgrhriZ log_metricr}Z get_best_runsrrvr assignmentsrr|Z set_proxiesZ experimentscreateprogressZobservation_countrZ suggestionsZ observationsfetchZbest_assignmentsZ iterate_pagesrrrrrr)rorMrNrr5 experimentrrmrxrrconnrrrvZobsrrr\rrr$r$r%run_hp_search_sigoptus                  rc  s:ddlm}|stdddld}jjD]}t|tr.d}qFq.|sVtdgj _ dddd| d d}| d d}| d d} | d d} | d d d} | d d<| d d <| r| | d <fdd} |sj | || dn|}td|j|| |dtdddS)Nr)r&z8This function needs wandb installed: `pip install wandb`rFTr)run_idrWhyperparameterssweep_idrrentityrz eval/lossZgoalcsjr jn}|jj_|jidj}d_jdt |ddt dddkr } |_t |}|krtdd|d}ddk rވd krȈjdk}nd krވjdk}|sddkr|jd<jd<t|d <jS) N)rrZ_itemsrVrWzProvided metric zU not found. This might result in unexpected sweeps charts. The available metrics are FrZminimizeZmaximizer)rinitrstate trial_namerupdaterWrfvarsrgrhri rewrite_logsrr keysrrH)rrrmZformat_metricsZ best_scorerrNrrorr$r%rps2      z'run_hp_search_wandb.._objective)rrzwandb sweep id - )functioncountrrWr) integrationsr& ImportErrorrcallback_handler callbacksrF WandbCallbackrr\ report_tor|rZsweeprrZagentr) rorMrNrr&Zreporting_to_wandbcallbackrrrrZ sweep_configrpr$rr%run_hp_search_wandbs8              !rcCsg}trts|dtr*|dtr:|dtrJ|dtrZ|dtrj|dtrz|dtr|dt r|d |S) Nazure_mlrr<r:rr-rrAr') r7r;appendr,r=r@r/r&rBr))rr$r$r%$get_available_reporting_integrationss(          rcCs~i}d}t|}d}t|}|D]T\}}||rL||d||d<q$||rl||d||d<q$||d|<q$|S)Neval_test_eval/ztest/ztrain/)lenrrY)dZnew_d eval_prefixeval_prefix_len test_prefixtest_prefix_lenkvr$r$r%r3s  rc@s>eZdZdZd ddZdddZddZdd d Zd d ZdS)rz A [`TrainerCallback`] that sends the logs to [TensorBoard](https://www.tensorflow.org/tensorboard). Args: tb_writer (`SummaryWriter`, *optional*): The writer to use. Will instantiate one if not set. Nc Cst}|std|rvzddlm}||_Wq|tk rrzddlm}||_Wntk rld|_YnXYq|Xnd|_||_dS)NzuTensorBoardCallback requires tensorboard to be installed. Either update your PyTorch version or install tensorboardX.r) SummaryWriter)r/rIZtorch.utils.tensorboardr_SummaryWriterrr. tb_writer)selfrZhas_tensorboardrr$r$r%__init__Ls"    zTensorBoardCallback.__init__cCs&|p|j}|jdk r"|j|d|_dS)N)log_dir) logging_dirrr)rr\rr$r$r%_init_summary_writercs  z(TensorBoardCallback._init_summary_writerc Ks|js dSd}|jr2|j}|dk r2tj|j|}|jdkrH||||jdk r|j d| d|kr|d}t |dr|j dk r|j }|j d|dS)Nr\rr model_config) is_world_process_zerois_hyper_param_searchrrrZr[rrradd_textZto_json_stringrr) rr\rcontrolrrrrZmodel_config_jsonr$r$r%on_train_beginhs     z"TensorBoardCallback.on_train_beginc Ks|js dS|jdkr|||jdk rt|}|D]L\}}t|ttfrb|j|||j q8t d|dt |d|dq8|j dS)N)Trainer is attempting to log a value of " " of type for key "zn" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.)rrrrrrFintfloatZ add_scalar global_steprr rflushrr\rrlogsrrrr$r$r%on_log~s   zTensorBoardCallback.on_logcKs|jr|jd|_dSr*)rcloserr\rrrr$r$r% on_train_ends z TensorBoardCallback.on_train_end)N)N)N) r __module__ __qualname____doc__rrrrrr$r$r$r%rCs    rc@sFeZdZdZddZddZdddZdd d Zdd d Zd dZ dS)rz{ A [`TrainerCallback`] that logs metrics, media, model checkpoints to [Weight and Biases](https://www.wandb.com/). cCst}|std|r$ddl}||_d|_tddt dhkr|t dtddt d tdd d |_ ntdd |_ dS) NzFWandbCallback requires wandb to be installed. Run `pip install wandb`.rFZWANDB_LOG_MODELr8r9zSetting `WANDB_LOG_MODEL` as zn is deprecated and will be removed in version 5 of transformers. Use one of `'end'` or `'checkpoint'` instead.zSetting `WANDB_LOG_MODEL` from z to `end` insteadendfalse)r&rIr_wandb _initializedrrrrunionDeprecationWarningrr _log_modellower)rZ has_wandbrr$r$r%rszWandbCallback.__init__c KsB|jdkrdSd|_|jr>td|}t|drV|jdk rV|j}||}|j}i}|dk r|||d<|j |d<n |j dks|j |j ks|j |d<|jj dkr|jj fdt dd i||jjj|dd t|jd dr|jd |jjd d ddt dd} ts>| dkr>|jj|| td|jddS)u Setup the optional Weights & Biases (*wandb*) integration. One can subclass and override this method to customize the setup if needed. Find more information [here](https://docs.wandb.ai/guides/integrations/huggingface). You can also override the following environment variables: Environment: - **WANDB_LOG_MODEL** (`str`, *optional*, defaults to `"false"`): Whether to log model and checkpoints during training. Can be `"end"`, `"checkpoint"` or `"false"`. If set to `"end"`, the model will be uploaded at the end of training. If set to `"checkpoint"`, the checkpoint will be uploaded every `args.save_steps` . If set to `"false"`, the model will not be uploaded. Use along with [`~transformers.TrainingArguments.load_best_model_at_end`] to upload best model. Setting `WANDB_LOG_MODEL` as `bool` will be deprecated in version 5 of 🤗 Transformers. - **WANDB_WATCH** (`str`, *optional* defaults to `"false"`): Can be `"gradients"`, `"all"`, `"parameters"`, or `"false"`. Set to `"all"` to log gradients and parameters. - **WANDB_PROJECT** (`str`, *optional*, defaults to `"huggingface"`): Set this to a custom string to store results in a different project. - **WANDB_DISABLED** (`bool`, *optional*, defaults to `False`): Whether to disable wandb entirely. Set `WANDB_DISABLED=true` to disable. NTz`Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"rrgrouprZ WANDB_PROJECTr)Zallow_val_change define_metrictrain/global_step*)Z step_metricZ step_syncZ WANDB_WATCHr)allrZ gradientsd)logZlog_freq)rrrrrto_dictrrrrun_name output_dirrrrrrrgrrwatchmaxZ logging_steps) rr\rrr combined_dictrrZ init_argsZ _watch_modelr$r$r%setups>          zWandbCallback.setupNcKsJ|jdkrdS|j}|r.|jd|_d|_|jsF|j|||f|dSNF)rrfinishrr&r+)rr\rrrrZ hp_searchr$r$r%rs  zWandbCallback.on_train_beginc KsD|jdkrdS|jdkr@|jr@|jr@ddlm}||||d}t} || |j sxddt |jj  Dnd|j |jd|ji} td |jdks|j|jkrd |jjjnd |jjj} |jj| d | d } t| d D]6} | r| j| jdd}|| W5QRXq|jj| W5QRXdS)N)rrkr)Trainer)r\r tokenizercSs,i|]$\}}t|tjr|ds||qS_rFnumbersNumberrYrwrrr$r$r% s  z.WandbCallback.on_train_end..rztrain/total_flosszLogging model artifacts. ...zmodel-rrrrr!wb)r)rrrrror.tempfileTemporaryDirectoryZ save_modelload_best_model_at_endrHsummaryrmetric_for_best_model best_metric total_flosrrr&r'rrrArtifactrglobis_filenew_filewrite read_bytes log_artifact)rr\rrrr/rr.Z fake_trainertemp_dirrZ model_nameartifactffar$r$r%rs<       zWandbCallback.on_train_endcKsJ|jdkrdS|js"|||||jrFt|}|j|d|jidS)Nr )rrr+rrr$r )rr\rrrrrr$r$r%r"s zWandbCallback.on_logc Ks|jdkr|jr|jrddt|jjD}d|j}tj |j |}t d|d|jdkst|j|j krd|jjjnd|jjj}|jj|d|d} | ||jj| d|jgd dS) NrkcSs,i|]$\}}t|tjr|ds||qSr0r2r5r$r$r%r6-s  z)WandbCallback.on_save.. checkpoint- Logging checkpoint artifacts in z. ...rr7)aliases)rrrrHrr<rr rrZr[r'rrr&rrrr@add_dirrF) rr\rrrZcheckpoint_metadatackpt_dir artifact_pathZcheckpoint_namerHr$r$r%on_save+s    zWandbCallback.on_save)N)NN)NN rrrrrr+rrrrQr$r$r$r%rsC " rc@s<eZdZdZddZddZd ddZdd d Zd d ZdS) CometCallbackz^ A [`TrainerCallback`] that sends the logs to [Comet ML](https://www.comet.ml/site/). cCsts tdd|_d|_dS)NzLCometCallback requires comet-ml to be installed. Run `pip install comet-ml`.F)r+rIr _log_assetsrr$r$r%rEszCometCallback.__init__cCsd|_tdd}|dkr$d|_|jrtdd}d}dtd d i}|dkrztjf|}|d d t d n:|dkrtdd|d<tj f|}|d d t d|dk r|j |d d|j |dd dt|dr|j |jdd ddS)a Setup the optional Comet.ml integration. Environment: - **COMET_MODE** (`str`, *optional*, defaults to `ONLINE`): Whether to create an online, offline experiment or disable Comet logging. Can be `OFFLINE`, `ONLINE`, or `DISABLED`. - **COMET_PROJECT_NAME** (`str`, *optional*): Comet project name for experiments. - **COMET_OFFLINE_DIRECTORY** (`str`, *optional*): Folder to use for saving offline experiments when `COMET_MODE` is `OFFLINE`. - **COMET_LOG_ASSETS** (`str`, *optional*, defaults to `TRUE`): Whether or not to log training assets (tf event logs, checkpoints, etc), to Comet. Can be `TRUE`, or `FALSE`. For a number of configurable items in the environment, see [here](https://www.comet.ml/docs/python-sdk/advanced/#comet-configuration-variables). TZCOMET_LOG_ASSETSr8>1r9rZONLINEN project_nameZCOMET_PROJECT_NAMErz Created from transformersz)Automatic Comet.ml online logging enabledZOFFLINEZCOMET_OFFLINE_DIRECTORYz./Zoffline_directoryzLAutomatic Comet.ml offline logging enabled; use `comet upload` when finished) frameworkzargs/)prefixrYrzconfig/)rrrrrTrrZ ExperimentZ log_otherrrZOfflineExperimentZ_set_model_graph_log_parametersrr)rr\rrZ log_assetsZ comet_moderZexperiment_kwargsr$r$r%r+Ks,       zCometCallback.setupNcKs|js||||dSr*rr+rr\rrrrr$r$r%ruszCometCallback.on_train_begincKsF|js|||||jrBtj}|dk rB|j||j|jdddS)NrX)stepepochrY) rr+rrrget_global_experimentZ _log_metricsr r_)rr\rrrrrrr$r$r%rys  zCometCallback.on_logcKsT|jrP|jrPtj}|dk rP|jdkrHtd|j|j dd|j d| dS)NTz(Logging checkpoints. This may take time.) recursiveZ log_file_namer^) rrrrr`rTrrZlog_asset_folderr'r r)rr\rrrrr$r$r%rs    zCometCallback.on_train_end)N)NN) rrrrrr+rrrr$r$r$r%rS@s *  rSc@s,eZdZdZd ddZddZd ddZdS) AzureMLCallbackzh A [`TrainerCallback`] that sends the logs to [AzureML](https://pypi.org/project/azureml-sdk/). NcCststd||_dS)NzPAzureMLCallback requires azureml to be installed. Run `pip install azureml-sdk`.)r7rI azureml_run)rrcr$r$r%rszAzureMLCallback.__init__cKs*ddlm}|jdkr&|jr&||_dS)NrRun)Zazureml.core.runrercrZ get_context)rr\rrrrer$r$r% on_init_ends zAzureMLCallback.on_init_endcKsB|jr>|jr>|D](\}}t|ttfr|jj|||dqdS)N) description)rcrrrFr r r$r r$r$r%rs zAzureMLCallback.on_log)N)N)rrrrrrfrr$r$r$r%rbs rbc@sLeZdZdZddZddZdddZdd d Zd d Zd dZ ddZ dS)MLflowCallbackz A [`TrainerCallback`] that sends the logs to [MLflow](https://www.mlflow.org/). Can be disabled by setting environment variable `DISABLE_MLFLOW_INTEGRATION = TRUE`. cCsJtstdddl}|jjj|_|jjj|_d|_ d|_ d|_ ||_ dS)NzIMLflowCallback requires mlflow to be installed. Run `pip install mlflow`.rF) r;rIr:utilsZ validationZMAX_PARAM_VAL_LENGTH_MAX_PARAM_VAL_LENGTHZMAX_PARAMS_TAGS_PER_BATCH_MAX_PARAMS_TAGS_PER_BATCHr _auto_end_run_log_artifacts_ml_flow)rr:r$r$r%rs  zMLflowCallback.__init__c Cstddtk|_tddtk|_tdd|_tddtk|_tdd|_t d|jd |j d |jd |j|j r|j dks|js|jr|jr|j |j|j j|j |jd t d |j jjd|_|}t|dr$|jdk r$|j}||}|jr4t|n|}t|D]<\}}tt||jkrDt d|d|d||=qDt|}tdt||jD]$} |j t!|| | |jqtdd} | rt"#| } |j $| d|_%dS)at Setup the optional MLflow integration. Environment: - **HF_MLFLOW_LOG_ARTIFACTS** (`str`, *optional*): Whether to use MLflow `.log_artifact()` facility to log artifacts. This only makes sense if logging to a remote server, e.g. s3 or GCS. If set to `True` or *1*, will copy each saved checkpoint on each save in [`TrainingArguments`]'s `output_dir` to the local or remote artifact storage. Using it without a remote storage will just copy the files to your artifact location. - **MLFLOW_EXPERIMENT_NAME** (`str`, *optional*, defaults to `None`): Whether to use an MLflow experiment_name under which to launch the run. Default to `None` which will point to the `Default` experiment in MLflow. Otherwise, it is a case sensitive name of the experiment to be activated. If an experiment with this name does not exist, a new experiment with this name is created. - **MLFLOW_TAGS** (`str`, *optional*): A string dump of a dictionary of key/value pair to be added to the MLflow run as tags. Example: `os.environ['MLFLOW_TAGS']='{"release.candidate": "RC1", "release.version": "2.2.0"}'`. - **MLFLOW_NESTED_RUN** (`str`, *optional*): Whether to use MLflow nested runs. If set to `True` or *1*, will create a nested run inside the current run. - **MLFLOW_RUN_ID** (`str`, *optional*): Allow to reattach to an existing run which can be usefull when resuming training from a checkpoint. When `MLFLOW_RUN_ID` environment variable is set, `start_run` attempts to resume a run with the specified run ID and other parameters are ignored. - **MLFLOW_FLATTEN_PARAMS** (`str`, *optional*, defaults to `False`): Whether to flatten the parameters dictionary before logging. ZHF_MLFLOW_LOG_ARTIFACTSr8ZMLFLOW_NESTED_RUNZMLFLOW_EXPERIMENT_NAMENZMLFLOW_FLATTEN_PARAMSZ MLFLOW_RUN_IDzMLflow experiment_name=z , run_name=z , nested=z, tags=)r&nestedzMLflow run started with run_id=Trrz " for key "z" as a parameter. MLflow's log_param() only accepts values no longer than 250 characters so we dropped this attribute. You can use `MLFLOW_FLATTEN_PARAMS` environment variable to flatten the parameters and avoid this message.rZ MLFLOW_TAGS)&rrrrrmZ _nested_runZ_experiment_nameZ_flatten_params_run_idrdebugr&rrn active_runZset_experimentZ start_runrrrlr%rrr r}rrrtrjr rrkZ log_paramsrHjsonrZset_tagsr) rr\rrr*rrrZcombined_dict_itemsrZ mlflow_tagsr$r$r%r+sD"   "   zMLflowCallback.setupNcKs|js||||dSr*r\r]r$r$r%rszMLflowCallback.on_train_beginc Ks|js|||||jr|i}|D]B\}} t| ttfrF| ||<q&td| dt | d|dq&|j j ||j ddS)Nrrrzc" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.)rmr^) rr+rrrFr r rr rrnZ log_metricsr ) rr\rrrrrrmrrr$r$r%rs zMLflowCallback.on_logcKs*|jr&|jr&|jr&|jr&|jdSr*)rrrlrnrrend_runrr$r$r%rs zMLflowCallback.on_train_endcKsd|jr`|jr`|jr`d|j}tj|j|}t d|d|j j j |d|i|j j ddS)NrKrL. This may take time.Z model_path) artifactsZ python_model)rrrmr rrZr[r'rrrnZpyfuncZ log_modelZ PythonModelrr\rrrrOrPr$r$r%rQs  zMLflowCallback.on_savecCs4|jr0tt|jddr0|jdk r0|jdS)Nrr)rlcallablergrnrrrtrUr$r$r%__del__ s zMLflowCallback.__del__)N)N) rrrrrr+rrrrQryr$r$r$r%rhs F   rhcs4eZdZdZfddZfddZddZZS)DagsHubCallbackzh A [`TrainerCallback`] that logs to [DagsHub](https://dagshub.com/). Extends [`MLflowCallback`] cs.ttstdddlm}||_dS)NzLDagsHubCallback requires dagshub to be installed. Run `pip install dagshub`.r)Repo)superrr=rZdagshub.uploadr{)rr{rJr$r%r0s   zDagsHubCallback.__init__cstddtk|_tdp"d|_td|_|j|jtj d|jtj ddd td pldd |_ t d |_ |jd krt dtj||d S)z Setup the DagsHub's Logging integration. Environment: - **HF_DAGSHUB_LOG_ARTIFACTS** (`str`, *optional*): Whether to save the data and model artifacts for the experiment. Default to `False`. ZHF_DAGSHUB_LOG_ARTIFACTSr8ZHF_DAGSHUB_MODEL_NAMEmainZMLFLOW_TRACKING_URI.rBRANCH)ownerrbranchrvNzpDagsHubCallback requires the `MLFLOW_TRACKING_URI` environment variable to be set. Did you run `dagshub.init()`?)rrrr log_artifactsrremoter{splitsepreporrZrIr|r+)rr\rr}r$r%r+9s     zDagsHubCallback.setupcKsL|jrHt|ddr.t|jjtj|j d|j t |j |j dS)Ntrain_dataloaderz dataset.pt)rrgrasaverZdatasetrrZr[r'r directoryrtrNrr$r$r%rTs zDagsHubCallback.on_train_end)rrrrrr+r __classcell__r$r$r}r%rz+s rzcseZdZfddZZS)NeptuneMissingConfigurationcstddS)NaA ------ Unsupported ---- We were not able to create new runs. You provided a custom Neptune run to `NeptuneCallback` with the `run` argument. For the integration to work fully, provide your `api_token` and `project` by saving them as environment variables or passing them to the callback. )r|rrUr}r$r%r]sz$NeptuneMissingConfiguration.__init__)rrrrrr$r$r}r%r\src@s0eZdZdZdZdZdZdZdZdhZ dddd dd dd e e e e e e e e e e d d dZ ddZddZddZddZddZeddZeddZddZdd Zd!d"Zd#d$Ze e d%d&d'Zd(d)Zd9d*d+Zd,d-Zd.d/Zd0d1Zd:d2d3Z e!d4d5Z"d;e e#e e$fd6d7d8Z%dS)<NeptuneCallbackaTrainerCallback that sends the logs to [Neptune](https://app.neptune.ai). Args: api_token (`str`, *optional*): Neptune API token obtained upon registration. You can leave this argument out if you have saved your token to the `NEPTUNE_API_TOKEN` environment variable (strongly recommended). See full setup instructions in the [docs](https://docs.neptune.ai/setup/installation). project (`str`, *optional*): Name of an existing Neptune project, in the form "workspace-name/project-name". You can find and copy the name in Neptune from the project settings -> Properties. If None (default), the value of the `NEPTUNE_PROJECT` environment variable is used. name (`str`, *optional*): Custom name for the run. base_namespace (`str`, optional, defaults to "finetuning"): In the Neptune run, the root namespace that will contain all of the metadata logged by the callback. log_parameters (`bool`, *optional*, defaults to `True`): If True, logs all Trainer arguments and model parameters provided by the Trainer. log_checkpoints (`str`, *optional*): If "same", uploads checkpoints whenever they are saved by the Trainer. If "last", uploads only the most recently saved checkpoint. If "best", uploads the best checkpoint (among the ones saved by the Trainer). If `None`, does not upload checkpoints. run (`Run`, *optional*): Pass a Neptune run object if you want to continue logging to an existing run. Read more about resuming runs in the [docs](https://docs.neptune.ai/logging/to_existing_object). **neptune_run_kwargs (*optional*): Additional keyword arguments to be passed directly to the [`neptune.init_run()`](https://docs.neptune.ai/api/neptune#init_run) function when a new run is created. For instructions and examples, see the [Transformers integration guide](https://docs.neptune.ai/integrations/transformers) in the Neptune documentation. z%source_code/integrations/transformersZmodel_parametersrK trial_paramsZtrainer_parametersz train/epochNZ finetuningT) api_tokenrrbase_namespacerlog_parameterslog_checkpoints)rrrrrrc Ks`tstdzddlm} ddlm} Wn,tk rVddlm} ddlm} YnX| d|t t df| d|t t df| d|t t df| d|t | d || t df| d |t | d |t t df||_ ||_ ||_||_d|_d |_d|_d |_|||d ||_d|_|jdk |_d|_|jdkrPd|j|_d|_n d|_d |_dS)NzwNeptuneCallback requires the Neptune client library to be installed. To install the library, run `pip install neptune`.rrd) verify_typerrrrrrrF)rrr>rxlastz checkpoints/TZ checkpoints)r@ ValueErrorrreZneptune.internal.utilsrrZneptune.new.internal.utilsZ#neptune.new.metadata_containers.runrtrbool_base_namespace_pathr[_log_checkpoints _initial_run_run_is_monitoring_runrp_force_reset_monitoring_run_init_run_kwargs_volatile_checkpoints_dir_should_upload_checkpoint_recent_checkpoint_path_target_checkpoints_namespace*_should_clean_recently_uploaded_checkpoint) rrrrrrrrZneptune_run_kwargsrerr$r$r%rsD       zNeptuneCallback.__init__cCs|jr|j|`d|_dSr*)rstoprUr$r$r%_stop_run_if_existss z#NeptuneCallback._stop_run_if_existsc Ksz ddlm}ddlm}m}Wn0tk rPddlm}ddlm}m}YnX|z&|f|j ||_ |j d |_ Wn.||fk r}z t |W5d}~XYnXdS)Nr)init_run)NeptuneMissingApiTokenException"NeptuneMissingProjectNameExceptionsys/id)rrZneptune.exceptionsrrrZ neptune.newZneptune.new.exceptionsrrrrrpr)rZadditional_neptune_kwargsrrrer$r$r%_initialize_runs  zNeptuneCallback._initialize_runcCs(|j|_d|_|jd|_d|_dS)NTr)rrrrrprUr$r$r%_use_initial_runsz NeptuneCallback._use_initial_runcCs^|jdk r|nF|js$|jr$dS|jrL|jsL|jsL|j|jdd|_n|d|_dS)N)with_idTF)rrrrrrrprUr$r$r%_ensure_run_with_monitorings   z+NeptuneCallback._ensure_run_with_monitoringcCs:|jdk r|n"|js6|j|jdddddd|_dS)NF)rZcapture_stdoutZcapture_stderrZcapture_hardware_metricsZcapture_traceback)rrrrrprrUr$r$r%'_ensure_at_least_run_without_monitorings  z7NeptuneCallback._ensure_at_least_run_without_monitoringcCs|jdkr||jSr*)rrrUr$r$r%rs zNeptuneCallback.runcCs |j|jSr*)rrrUr$r$r%_metadata_namespacesz#NeptuneCallback._metadata_namespacecCst|jtj<dSr*)rrrintegration_version_keyrUr$r$r%_log_integration_versionsz(NeptuneCallback._log_integration_versioncCs||jtj<dSr*)Zto_sanitized_dictrrtrainer_parameters_key)rr\r$r$r%_log_trainer_parameterssz'NeptuneCallback._log_trainer_parameterscCs>ddlm}|r:t|dr:|jdk r:||j|jtj<dS)Nr)stringify_unsupportedr)Z neptune.utilsrrrr%rrmodel_parameters_key)rrrr$r$r%_log_model_parameters s  z%NeptuneCallback._log_model_parameterscCsF|rt|dr|j|jtj<|rBt|drB|jdk rB|j|jtj<dS)Nrr)rrrrtrial_name_keyrtrial_params_key)rrr$r$r%"_log_hyper_param_search_parameterssz2NeptuneCallback._log_hyper_param_search_parameters)source_directoryrkc Cstj||}}|jdk rtj|j|}z8|ddtjj}tj||}t|||}Wn2t k r}zt d |W5d}~XYnX|j |j||jr|jdk r|j |j|j||_dS)Nz..rzoNeptuneCallback was unable to made a copy of checkpoint due to I/O exception: '{}'.Could fail trying to upload.)rrZr[rreplacelstriprshutilcopytreeIOErrorrr rrrZ upload_filesrrZ delete_files) rrrk target_path relative_pathZconsistent_checkpoint_pathZ cpkt_pathZ copy_pathrr$r$r%_log_model_checkpoints$  z%NeptuneCallback._log_model_checkpointcKsDd|_|jr(|js|jdk r(tj|_|jdkr@|js@tddS)NrxzWTo save the best model checkpoint, the load_best_model_at_end argument must be enabled.) rrZoverwrite_output_dirZsave_total_limitr9r:rr;rrr$r$r%rf1s  zNeptuneCallback.on_init_endcKsN|js dS|d|_||jr:|||||jrJ||dS)NT) rrrrr[rrrrr]r$r$r%r9s  zNeptuneCallback.on_train_begincKs |dSr*)rrr$r$r%rHszNeptuneCallback.on_train_endcCs&|jdk rtj|jdd|dS)NT) ignore_errors)rrrmtreerrUr$r$r%ryKs zNeptuneCallback.__del__cKs |jr||jd|jdS)NrK)rrr'r rr$r$r%rQQszNeptuneCallback.on_savec Ks\|jdkrX|j}|ds$d|}||}|jr:tjntj}|jdkpT|||j|_ dS)Nrxr) rr=rYrZgreater_is_betternpZgreaterlessr>r) rr\rrrmrZbest_metric_nameZ metric_valueoperatorr$r$r% on_evaluateUs    zNeptuneCallback.on_evaluatecCs.|jjD]}t||r|jSqtddS)Nz6The trainer doesn't have a NeptuneCallback configured.)rrrFr Exception)rrorr$r$r%get_runas   zNeptuneCallback.get_run)rcKsf|js dS|dk rbt|D]B\}}t|ttfr|tjkrJ||j|<q|j|j ||j dqdS)N)r^) rrrrFr r r flat_metricsrr$r )rr\rrrrrrr$r$r%ris  zNeptuneCallback.on_log)N)N)N)&rrrrrrrrrrrrtrrrrrrrpropertyrrrrrrrrfrrryrQr classmethodrrr rr$r$r$r%rgsZ 7     rc@s2eZdZdZddZddZd ddZd d ZdS) CodeCarbonCallbackzI A [`TrainerCallback`] that tracks the CO2 emission of training. cCs&tstdddl}||_d|_dS)NzWCodeCarbonCallback requires `codecarbon` to be installed. Run `pip install codecarbon`.r)rBrIrA _codecarbontracker)rrAr$r$r%r{szCodeCarbonCallback.__init__cKs&|jdkr"|jr"|jj|jd|_dS)N)r')ris_local_process_zerorZEmissionsTrackerr'rr$r$r%rfszCodeCarbonCallback.on_init_endNcKs|jr|jr|jdSr*)rrstartr]r$r$r%rs z!CodeCarbonCallback.on_train_begincKs|jr|jr|jdSr*)rrrrr$r$r%rs zCodeCarbonCallback.on_train_end)N)rrrrrrfrrr$r$r$r%rvs   rc@sFeZdZdZddZddZdddZdd d Zdd d Zd dZ dS)ClearMLCallbacka A [`TrainerCallback`] that sends the logs to [ClearML](https://clear.ml/). Environment: - **CLEARML_PROJECT** (`str`, *optional*, defaults to `HuggingFace Transformers`): ClearML project name. - **CLEARML_TASK** (`str`, *optional*, defaults to `Trainer`): ClearML task name. - **CLEARML_LOG_MODEL** (`bool`, *optional*, defaults to `False`): Whether to log models as artifacts during training. cCsLtrddl}||_ntdd|_d|_tddt dhk|_ dS)NrzNClearMLCallback requires 'clearml' to be installed. Run `pip install clearml`.FZCLEARML_LOG_MODELr8r9) r)r'_clearmlrIr _clearml_taskrrrrrr)rr'r$r$r%rszClearMLCallback.__init__cKs|jdkrdS|jrdS|jrtd|jdkr|jjr^|jj|_d|_tdn>|jjjt ddt ddddd dd |_d|_td |j |d t |d r|j dk r|j |j ddS)Nz"Automatic ClearML logging enabled.Tz)External ClearML Task has been connected.ZCLEARML_PROJECTzHuggingFace TransformersZ CLEARML_TASKr.F)r-Zpytorch)rWZ task_nameZauto_connect_frameworksZ output_uriz"ClearML Task has been initialized.ZArgsrzModel Configuration)rrrrrrTask current_taskrrrconnectrr)rr\rrr/rr$r$r%r+s,        zClearMLCallback.setupNcKs8|jdkrdS|jrd|_|js4|j||||f|dSr,)rrrr+)rr\rrrr/rr$r$r%rs  zClearMLCallback.on_train_beginc Ks(|jdkrdS|jr$|jr$|jdSr*)rrrr) rr\rrrr/rmrrr$r$r%rs  zClearMLCallback.on_train_endc Ks6|jdkrdS|js(|j||||f||jr2d}t|} d} t| } ddddddg} |D]\} }t|ttfr| | kr|j j | |d nv| |r|j j | | dd ||jd nH| | r|j j | | dd ||jd n|j j | d ||jd q`td|dt|d| dq`dS)NrrZ train_runtimeZtrain_samples_per_secondZtrain_steps_per_secondZ train_lossr?r_reval)titleZseriesr iterationtestrfrrrzn" as a scalar. This invocation of ClearML logger's report_scalar() is incorrect so we dropped this attribute.)rrr+rrrrFr r r get_loggerZreport_single_valuerYZ report_scalarr rr r)rr\rrrr/rrrrrrZsingle_value_scalarsrrr$r$r%rsV        zClearMLCallback.on_logcKsX|jrT|jrT|jrTd|j}tj|j|}t d|d|jj ||jdddS)NrKrLruF)rZauto_delete_file) rrrr rrZr[r'rrZupdate_output_modelrwr$r$r%rQs  zClearMLCallback.on_save)NN)NNNN)NNNrRr$r$r$r%rs     *rcs:eZdZdZd eedfdd ZddZdd ZZS) FlyteCallbacka^A [`TrainerCallback`] that sends the logs to [Flyte](https://flyte.org/). NOTE: This callback only works within a Flyte task. Args: save_log_history (`bool`, *optional*, defaults to `True`): When set to True, the training logs are saved as a Flyte Deck. sync_checkpoints (`bool`, *optional*, defaults to `True`): When set to True, checkpoints are synced with Flyte and can be used to resume training in the case of an interruption. Example: ```python # Note: This example skips over some setup steps for brevity. from flytekit import current_context, task @task def train_hf_transformer(): cp = current_context().checkpoint trainer = Trainer(..., callbacks=[FlyteCallback()]) output = trainer.train(resume_from_checkpoint=cp.restore()) ``` T)save_log_historysync_checkpointscsXttstdtr$ts2tdd}ddlm }|j |_ ||_ ||_ dS)NzLFlyteCallback requires flytekit to be installed. Run `pip install flytekit`.zSyncing log history requires both flytekitplugins-deck-standard and pandas to be installed. Run `pip install flytekitplugins-deck-standard pandas` to enable this feature.Fr)current_context)r|rrDrrEr rr rCrrkcprr)rrrrr}r$r%r%s    zFlyteCallback.__init__cKsJ|jrF|jrFd|j}tj|j|}td|d|j |dS)NrKzSyncing checkpoint in z to Flyte. This may take time.) rrr rrZr[r'rrrrrwr$r$r%rQ7s   zFlyteCallback.on_savec KsH|jrDddl}ddlm}ddlm}||j}|d||dS)Nr)Deck) TableRendererz Log History) rZpandasrCrZflytekitplugins.deck.rendererrZ DataFrameZ log_historyZto_html) rr\rrrpdrrZlog_history_dfr$r$r%r?s    zFlyteCallback.on_train_end)TT) rrrrrrrQrrr$r$r}r%r sr) rrr:rr-rrAr'r<ZflytecCs>|D]*}|tkrt|ddtdqdd|DS)Nz is not supported, only rz are supported.cSsg|] }t|qSr$)INTEGRATION_TO_CALLBACK)rw integrationr$r$r%ry^sz7get_reporting_integration_callbacks..)rrr[r)rrr$r$r%#get_reporting_integration_callbacksWs r)frrimportlib.metadatar!importlib.utilrsr3rrdrrr9 dataclassesrpathlibrtypingrrrnumpyrrrrrir r r r r rrrrar"r#rrr+rrrZ get_configr rrr?rZ_neptune_versionrZPackageNotFoundErrorZtrainer_callbackrrrrrrZ training_argsrrrr&r)r,r/r1r3r4r6r7r;r=r>r@rBrDrErLr rtrrrrrrrrrSrbrhrzrrrrrrrrr$r$r$r%s    $        9 bEV(M1 w@