+ QV-j8vRt^RIt^RIt^RIt^RIt^RIt^RIt^RIt^RI t ^RI t ^RI t ^RI t ^RI t ^RIt^RIHt^RIHt^RIHt^RIHtHtHt^RIt^RIt]P:!R4R8Xd ]!R4^R IH t H!t!^R IH"t#^R I$H%t%H&t&H't'H(t(H)t)H*t*]*PV!],4t-])!4'd^RI.t.R t/]P`PGR 4t1Rt2]PFPg]14]PFPg]/48t4^RI5t5]5PlPoR4eRt8MRt8]P~PR4RJ;'g]P~PR4RJtA]'d:]A'd2]P`PGR4tB]-PR]B R24^RIHDtD^RIEHFtFHGtG^RIHHItIHJtJHKtK^RILHMtM^RI$HNtNHOtORtPRtQRtRRtSRtTR tUR!tVR"tWR#tXR$tYR%tZR&t[R't\R(t]R)t^R*t_R+t`R,taR-tbR.R/ltcR0R1ltdR2R3lteR4tfR5tgR6R7lth!R8R9]G4tiR:R;ltj!R<R=]k]4tl!R>R?]G4tm!R@RA]G4tn!RBRC]G4to!RDRE]G4tp!RFRG]G4tq!RHRI]q4tr!RJRK]s4tt!RLRM]G4tu!RNRO]G4tv!RPRQ]G4tw!RRRS]G4tx!RTRU]G4ty!RVRW]G4tz!RXRY]G4t{RZ]pR ]oR[]qR]uR\]iR]]nR^]mR_]vR`]wRa]rRb]xRc]yRd]zRe]{/t|Rft}R# ]P`Pr]:];]<]=]>3d Rt1Rt2Rt4Rt8ELi;i ]P`Prd[]P`PGR4tB]-PR]B R24EL ]P`PrdRtAELi;ii;i)gz+ Integrations with other Python libraries. N)fields)Enum)Path) TYPE_CHECKINGAnyLiteral WANDB_MODEofflinez$[INFO] Running in WANDB offline mode)PreTrainedModelTrainingArguments) __version__)PushToHubMixin flatten_dictis_datasets_availableis_pandas_availableis_torch_availableloggingz3.43.2comet_mlTz comet.api_keyFneptunezneptune-clientzNeptune version z available.zNeptune-client version ) modelcard)ProgressCallbackTrainerCallback)PREFIX_CHECKPOINT_DIRBestRunIntervalStrategy) ParallelMode)ENV_VARS_TRUE_VALUESis_torch_xla_availablech\PPR4e^RIp\ VR4#R#)wandbNrunF) importlibutil find_specrhasattr)rs |/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/integrations/integration_utils.pyis_wandb_availabler&hs,~~(4ue$$cD\PPR4RJ#)trackioNr!r"r#r'r%is_trackio_availabler,s >> # #I .d ::r'cD\PPR4RJ#)clearmlNr*r+r'r%is_clearml_availabler0wr-r'c\RJdR#\RJd'\PR\\ \ 4R#\ RJd\PR4R#R#)Fzcomet_ml version %s is installed, but version %s or higher is required. Please update comet_ml to the latest version to enable Comet logging with pip install 'comet-ml>=%s'.a#comet_ml is installed but the Comet API Key is not configured. Please set the `COMET_API_KEY` environment variable to enable Comet logging. Check out the documentation for other ways of configuring it: https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#set-the-api-keyT)_is_comet_installed_is_comet_recent_enoughloggerwarning_comet_version_MIN_COMET_VERSION_is_comet_configuredr+r'r%is_comet_availabler9{sWe#%' t     u$ h  r'c\PPR4RJ;'g"\PPR4RJ#) tensorboardN tensorboardXr*r+r'r%is_tensorboard_availabler=s: >> # #M 2$ > v v)..BZBZ[iBjrvBvvr'cD\PPR4RJ#)optunaNr*r+r'r%is_optuna_availabler@s >> # #H -T 99r'cD\PPR4RJ#)rayNr*r+r'r%is_ray_availablerCs >> # #E *$ 66r'ch\4'gR#\PPR4RJ#)Fzray.tuneN)rCr!r"r#r+r'r%is_ray_tune_availablerEs(    >> # #J /t ;;r'c\PPR4fR#\PPR4fR#\PPR4RJ#)azuremlNFz azureml.corezazureml.core.runr*r+r'r%is_azureml_availablerHsM~~ *2~~/7 >> # #$6 7t CCr'c\P!RR4P4R8XdR#\PP R4RJ#)DISABLE_MLFLOW_INTEGRATIONFALSETRUEFmlflowN)osgetenvupperr!r"r#r+r'r%is_mlflow_availablerQs< yy-w7==?6I >> # #H -T 99r'cR\PPR4\PPR439#)NdagshubrMr*r+r'r%is_dagshub_availablerTs1  00;Y^^=U=UV^=_` ``r'c\#N) _has_neptuner+r'r%is_neptune_availablerXs r'cD\PPR4RJ#) codecarbonNr*r+r'r%is_codecarbon_availabler[s >> # #L 1 ==r'cD\PPR4RJ#)flytekitNr*r+r'r%is_flytekit_availabler^s >> # #J /t ;;r'ch\4'gR#\PPR4RJ#)Fzflytekitplugins.deckN)r^r!r"r#r+r'r% is_flyte_deck_standard_availabler`s) " " >> # #$: ;4 GGr'cD\PPR4RJ#)dvcliveNr*r+r'r%is_dvclive_availablercr-r'cD\PPR4RJ#)swanlabNr*r+r'r%is_swanlab_availablerfr-r'c\P!RR4P4R8XdR#\P!R4RJ#)DISABLE_KUBEFLOW_INTEGRATIONrKrLFKUBEFLOW_TRAINER_SERVER_URLN)rNrOrPr+r'r%is_kubeflow_availablerjs7 yy/9??AVK 992 34 ??r'c\\4'd6^RIp\WPP4'd VP #\ 4'd\V\4'dV#\4'd\V\4'dV#\RVP 24h)NzUnknown type for trial ) r@r? isinstancetrial BaseTrialparamsrEdictr& RuntimeError __class__)rnr?s& r% hp_paramsrts{ e\\33 4 4<<  eT " "L eT " "L 00AB CCr'c<V^8dQhR\R\R\/#n_trials directionreturnintstrr)formats"r% __annotate__rs!>>C>C>g>r'c zaaa^RIo^RIHoSPP^8XEdKRV3RlVVV3RlllpVP RR4pVP R^4pVP RR4pVP R R4p\ V\4'dTMRp V eRMTpSP!RR VR V /VBp V PWAWVWxR 7V P4'gAV Pp \\V P4V PV P 4#V P"p V U u.uF7p \\V P4V P$V P 4NK9 up #\'V4FpRSnR.pSPP*\,P.8wd \1R 4h\2P4P7V^R7SP9RV^,R7\;SRR4eKSP=4pSP?V4SnK R#uup i)rlN)release_memoryc2<V^8dQhRSP/#rwrn)Trial)r~r?s"r%r*run_hp_search_optuna..__annotate__s % %fll %r'c<RpV'dZ\P!V4F?pVP\4'gK \PP W4pKA RS nS PP^8dS PP\P8wd \R4hS PV4SPPVP VP"4pV.p\$P&P)V^R7S P+W R7MS P+W R7\-S RR4f'S P/4pS P1V4S nS!S P2S P44wS nS nS P6P94S P #)NConly support DDP optuna HPO for ParallelMode.DISTRIBUTED currently.srcresume_from_checkpointrn objective)rNlistdir startswithrpathjoinrargs world_size parallel_moder DISTRIBUTEDrrhp_spacern FixedTrialrpnumbertorch distributedbroadcast_object_listtraingetattrevaluatecompute_objective model_wrappedmodel acceleratorclear) rncheckpoint_dir checkpointsubdir fixed_trialtrial_main_rank_listmetricsr?rtrainers && r% _objective(run_hp_search_optuna.._objectivesaJ jj8F(()>??%'WW\\.%I 9!%G ||&&*<<--1I1II&'lmm  '$ll55ellELLQ (3}$!!778LRS7T Z M Z Mw T2:!**,$+$=$=g$F!4B'BWBWY`YfYf3g 0G !7=    % % '$$ $r'timeoutn_jobsgc_after_trialFcatchry directions)rxrrrrrrrrrVr+) r?accelerate.utils.memoryrr process_indexpoprmlist create_studyoptimize_is_multi_objective best_trialrr}rvaluerp best_trialsvaluesrangerrrrrrrrrrrrr)rrxrykwargsrrrrrrstudyrrbestirrr?rsf&&, @@r%run_hp_search_optunars6||!!Q& % %8**Y-Ha($4e< 7B'",Y"="=Y4 &2D ##YiYJYRXY  7Zh  ((**))J3z001:3C3CZEVEVW W++KT_`T_DGC ,dkk4;;GT_` `xA $G $(6 ||))\-E-EE"#hii    3 34Ha 3 P MM=QRS=TM Uw T2:!**,$+$=$=g$F!!as4=H8c<V^8dQhR\R\R\/#rvr{)r~s"r%rr,s&MMMM7Mr'c aa^RIoRV3RllpVPP'g*^RIHp\ P R4V!RR7VnVP\4pRVn RV9dnR ^/VR&VPP^8d ^VR,R &R VPP^8dR MR ,p\ PRV R24VR,PR ^4pWPnRV9d^RIHp V !R.R7VR&RV9d^RIHp Hp Hp Hp \+VR,WW34'd{VPP,'d*VPP.\0P28Xd6\5RP7\9VR,4P:R74hSP<P?W@R7o\@PB!S4V3Rl4p\ESR4'dSPFVn#SP<PH!V3RVPKR4RV/VBp\LPN!RR4pVPQRVR,VR 7p\SVPTVPVR,VPXV4pVeVP[V4V#)!a Environment: - **RAY_SCOPE** (`str`, *optional*, defaults to `"last"`): The scope to use when doing hyperparameter search with Ray. By default, `"last"` will be used. Ray will then use the last checkpoint of all trials, compare those, and select the best one. However, other options are also available. See the Ray documentation (https://docs.ray.io/en/latest/tune/api_docs/analysis.html#ray.tune.ExperimentAnalysis.get_best_trial) for more options Nc$V^8dQhR\/#r)rq)r~s"r%r'run_hp_search_ray..__annotate__7s$@$@$$@r'c<^RIHpVPV4'dVP\4RTnSPP4pT'dtRTnTP4;_uu_4p\\T4P\ R244P4pTPYPR7RRR4MTPTR7\!TRR4fTP#4pTP%T4TnTP'RTP RR/4\(P*!4;_uu_4pTP-TR 7SPP.P1T4pSPP3YcR 7RRR4R#R# \ dELi;i +'giL;i +'giR#;i) rl)NotebookProgressCallbackNr*r)rndoneT)r)r)transformers.utils.notebookr pop_callback add_callbackrModuleNotFoundErrorrtuneget_checkpoint as_directorynextrglobras_posixrrrrupdatetempfileTemporaryDirectory_tune_save_checkpoint Checkpointfrom_directoryreport) rn local_trainerrrrcheckpoint_pathrtemp_checkpoint_dirrBs && r%r%run_hp_search_ray.._objective7s  L))*BCC**+;<#' XX,,.  '2M #((**n"&tN';'@'@DYCZZ[A\']"^"g"g"i##?#X+*   e  , =+t 4 <#,,.G&3&E&Eg&NM # NNK)@)@&$O P,,..2E33CV3W XX00??@ST ?/. =+#   +*/..s0F4F4=A GAG4 GG G  G* )TrainerMemoryTrackerzMemory tracking for your Trainer is currently enabled. Automatically disabling the memory tracker since the memory tracker is not serializable.T)skip_memory_metricsresources_per_trialcpugpuz1 CPUz and 1 GPUzgNo `resources_per_trial` arg was passed into `hyperparameter_search`. Setting it to a default value of z for each trial.progress_reporter) CLIReporterr)metric_columns scheduler) ASHASchedulerHyperBandForBOHBMedianStoppingRulePopulationBasedTrainingaYou are using {cls} as a scheduler but you haven't enabled evaluation during training. This means your trials will not report intermediate results to Ray Tune, and can thus not be stopped early or used to exploit other trials parameters. If this is what you want, do not use {cls}. If you would like to use {cls}, make sure you pass `do_eval=True` and `eval_strategy='steps'` in the Trainer `args`.)cls)rcX<\4'Ed\PP\P PR44\PPR48d^RIp\PPVPP4R4p\PPRV4p\PPV4pV\P VP"&VP$P'V4S!V/VB#)a Wrapper around `tune.with_parameters` to ensure datasets_modules are loaded on each Actor. Without this, an ImportError will be thrown. See https://github.com/huggingface/transformers/issues/11565. Assumes that `_objective`, defined above, is a function. datasetsz4.0.0Nz __init__.pydatasets_modules)r packagingversionparser!metadata datasets.loadrNrrloadinit_dynamic_modulesr"spec_from_file_locationmodule_from_specsysmodulesnameloader exec_module)rrrdynamic_modules_pathspecr trainables*, r% dynamic_modules_import_trainable;run_hp_search_ray..dynamic_modules_import_trainables ! " "y'8'8'>'>    & &z 2(    # #G ,(- !#%77<< 0R0R0TVc#d >>99:LNbcD(~~>>tD %5CKK " KK # #$4 5$)&))r' __mixins__config num_samples RAY_SCOPElast:NN)metricmodescope).ray.tune_memory_trackerr trainer_utilsrr4r5rTensorBoardCallbackrrn_gpuinfoget_n_gpurray.tune.schedulersrrrrrmdo_eval eval_strategyrNOrrr~type__name__rwith_parameters functoolswrapsr$rr rrNrOget_best_trialrtrial_id last_resultrr)rrxryrrr _tb_writer resource_msggpus_per_trialrrrrrranalysis ray_scoperbest_runrBrs&&&, @@r%run_hp_search_rayr',s$@$@L  " " 6 6 68 < #74"P%%&9:JGMF*). $% <<   !34F( )% 0',,2D2Dq2H,bQ   / 1 1266ua@N(LL&((&1+&O"#ftt  ; -EU!o  <<'''7<<+E+EIYI\I\+\" #)&T&2E-F-O-O&"P  (((KI__Y* *,y,''6?6J6J(3xx||(%  H  +v.I(( )B-W`(aJz**J,B,B;,OQ[QbQbdlmHZ( Or'c<V^8dQhR\R\R\/#rvr{)r~s"r%rrs+HkHk3Hk3HkWHkr'c aaa a a\4'g \R4h^RIoRpSPPFp\ V\ 4'gKRpM V'gSP\ 44R.SPn RRRRRR/o VPRR4pVPR R4pVPR R4pVPR R4p VPR R 4o SPR4p SV R ,R&S V R ,R &V'dWR &V VV VV3Rlp V'gSPWV R7pMB^RI oV 'dSPPV 4SPP!V4\"P%RV 24SP'WkVR7\)S R,S R,S R,V4#)z8This function needs wandb installed: `pip install wandb`NFTrrun_idrhyperparameterssweep_idprojectrentityr z eval/lossgoalcF<S P'd S PMS P4pVPSPnVP P R/RS/4S P pRSnSPR\V4R,R7\SRR4fbSP4pSPV4Sn\V4pSV9d*\PRS RVP!4 24RpSR ,e<SR 8XdSPSR,8pMSR 8XdSPSR,8pV'g SR ,f-VP"SR &SPSR&\%V4SR &SP#) assignmentsr N_itemsrrzProvided metric zU not found. This might result in unexpected sweeps charts. The available metrics are Fr*minimizemaximizer+)r initrstate trial_namerrrrvarsrrr rewrite_logsr4r5keysidrq) r rrformat_metrics best_scorerryr rrs r%r'run_hp_search_wandb.._objectivesq 999eii%**,#&88   ="h?@  Tfh9O P 7K . 6&&(G ' 9 9' BG )'2N^+&vh/$$2$7$7$9#:< h  +J&$..K1HH j($..K1HH H-5#&66Jx &-&7&7J{ #,0LJ( )   r')r-r.zwandb sweep id - )functioncount)r& ImportErrorrcallback_handler callbacksrm WandbCallbackrr report_torrsweep wandb.envenv set_entity set_projectr4ragentr)rrxryrreporting_to_wandbcallbackr,r-rr. sweep_configrrr rsf&f, @@@r%run_hp_search_wandbrOs   TUU,,66 h . .!%  7 ]_-%YGLLD+t5FMJzz*d+HjjD)G ::fd #D ZZ$ 'F ZZ+ .F##D)L%.L6"%+L6" #V!!B ;;|V;L  II  ( g& KK#H:./ KKXK> :h'K)@*M^B_ai jjr'c.p\4'd"\4'gVPR4\4'dVPR4\ 4'dVPR4\ 4'dVPR4\4'dVPR4\ 4'dVPR4\4'dVPR4\4'dVPR4\4'dVPR4\4'dVPR 4\4'dVPR 4\4'dVPR 4\4'dVPR 4V#) azure_mlrrSrbrMrr;rrZr/rer)kubeflow)rHrQappendr9rTrcrXr=r&r[r0rfr,rj) integrationss r%$get_available_reporting_integrationsrUsAL&9&;&;J'J'I&I&H%I&!!M*G$  L)I&I&I&J' r'c/pRp\V4pRp\V4pVP4F\wrgVPV4'dWqRWcR,&K+VPV4'dWqRWeR,&KQWqRV,&K^ V#)eval_test_eval/Nztest/ztrain/)lenitemsr)dnew_d eval_prefixeval_prefix_len test_prefixtest_prefix_lenkvs& r%r9r9&s EK+&OK+&O  << $ $34'A.// 0 \\+ & &34'A.// 0"#(Q,   Lr'c$V^8dQhR\/#rwrzr})r~s"r%rr6sKKKr'c^RIp^RIHpVP4PR4p\P P RVR,VP4,4#)z Same default as PyTorch N)datetimez %b%d_%H-%M-%Sruns_)socketrhnowstrftimerNrr gethostname)rkrh current_times r%default_logdirrp6sH!<<>**?;L 77<< s 2V5G5G5I I JJr'cJa]tRtRtoRtR RltRtRtR RltRt R t Vt R#) riAa A [`TrainerCallback`] that sends the logs to [TensorBoard](https://www.tensorflow.org/tensorboard). Args: tb_writer (`SummaryWriter`, *optional*): The writer to use. Will instantiate one if not set. Environment: - **TENSORBOARD_LOGGING_DIR** (`str`, *optional*, defaults to `None`): The logging dir to log the results. Default value is os.path.join(args.output_dir, default_logdir()) NcD\4'g \R4h^RIHpW nWn\P!RR4Vn VPe1\PPVP4Vn R#R# \d ^RIHpL{i;i)zuTensorBoardCallback requires tensorboard to be installed. Either update your PyTorch version or install tensorboardX.) SummaryWriterTENSORBOARD_LOGGING_DIRN) r=rrtorch.utils.tensorboardrsrAr<_SummaryWriter tb_writerrNrO logging_dirr expanduser)selfrwrss&& r%__init__TensorBoardCallback.__init__Ms')))  3 =,"99%>E    '!ww11$2B2BCD  (  3 2 3sB BBchVPe$VPVPR7VnR#R#)N)log_dirrvrxrwrzrs&&r%_init_summary_writer(TensorBoardCallback._init_summary_writer^s.    *!009I9I0JDN +r'c VP'gR#VP'dIVPpVe9\PP VP \4V4VnVPf8\PP VP \44VnVPfVPV4VPeVPPRVP44RV9dfVR,p\VR4'dIVPe9VPP4pVPPRV4R#R#R#R#R#)Nrrr model_config)is_world_process_zerois_hyper_param_searchr7rNrr output_dirrprxrwradd_textto_json_stringr$r)rzrr6controlrr7rmodel_config_jsons&&&&, r%on_train_begin"TensorBoardCallback.on_train_beginbs***   & & &))J%#%77<<AQS]#^    #!ww||DOO^=MND  >> !  % %d + >> % NN # #FD,?,?,A B& w5(++ 0H(- (C(C(E%NN++N> !  % %d + >> %%D a#u..NN--aE4E4EF3''NN++A%2C2CDNN3ja A3?EE % NN " &r'c pVP'd$VPP4RVnR#R#rV)rwcloserzrr6rrs&&&&,r% on_train_end TensorBoardCallback.on_train_ends' >>> NN "!DN r'rrV) r __module__ __qualname____firstlineno____doc__r{rrrr__static_attributes____classdictcell__ __classdict__s@r%rrAs+ D"KO0#.""r'rc0V^8dQhR\R\/#)rwrr)rr})r~s"r%rrs!!3!C!r'cp\V R2R4;_uu_4p\V\4'd\WR7M^\ 4'dO\V\ P P\34'd\VR4'd \WR7RRR4R# +'giR#;i)z/model_architecture.txtzw+)file base_modelN) openrmr printrrnnModuler r$)rrfs&& r%save_model_architecture_to_filersu  34d ; ;q e_ - - %  ! ! uuxx? @ @WUT`EaEa % < ; ; ;s BB$$ B5 cha]tRtRtoRtRtRtRt]V3RlRl4t ] V3RlR l4t R t Vt R #) WandbLogModeliz)Enum of possible log model values in W&B.rendfalsec <V^8dQhRS[/#rebool)r~rs"r%rWandbLogModel.__annotate__sEEDEr'c HV\P\P39#)zYCheck if the value corresponds to a state where the `WANDB_LOG_MODEL` setting is enabled.)r CHECKPOINTENDrzs&r% is_enabledWandbLogModel.is_enableds  00-2C2CDDDr'c$<V^8dQhRS[RR/#)rwrrzr)r)r~rs"r%rrs##c#o#r'c\V\4'g\R\V4 24h\P RV R24\ P#)z>Expecting to have a string `WANDB_LOG_MODEL` setting, but got z6Received unrecognized `WANDB_LOG_MODEL` setting value=z ; so disabling `WANDB_LOG_MODEL`)rmr} TypeErrorrr4r5rrK)rrs&&r% _missing_WandbLogModel._missing_sQ%%%\]abg]h\ijk kDUGKk l """r'r+N)rrrrrrrrKpropertyr classmethodrrrrs@r%rrs?3J C E EE##r'rcfa]tRtRtoRtRtRtR RltRV3RlRlltRR lt R t R t R t Vt R#)rDizs A [`TrainerCallback`] that logs metrics, media, model checkpoints to [Weight and Biases](https://www.wandb.com/). c\4pV'g \R4h^RIpW nRVn\ \ P!RR44VnR#)zFWandbCallback requires wandb to be installed. Run `pip install wandb`.NFWANDB_LOG_MODELr) r&rrr_wandb _initializedrrNrO _log_model)rz has_wandbrs& r%r{WandbCallback.__init__sA&( gh h !' 2CW(MNr'c VPfR#RVn^RIHpVP'Ed/VP 4Cp\ VR4'd[VPeM\VP\4'd VPMVPP 4p/VCVCp\ VR4'd!VPeVPpRV/VCpVPp /p V e)WR&VP;'g VPV R&MUVPeHVPV R&VPVP8XdVPPRR R 7VPPf4VPP !R(R \"P$!R R 4/V BVPPP'T;'g/RR7\)VPRR4'd:VPP+R4VPP+RRRR7\"P$!RR4p \-4'g9V R)9d2VPP/W;\1^dVP24R7VPPP5RR7VP74VPPR&VP@PB'Ed\DPF!4;_uu_4p VPeVPVP8Xd$RVPPPH 2M"RVPPPJ 2p VPPMT RR\ VR4'dVPP 4MRRVPPPOR4RR/R 7p\QW<4\SV 4PUR4FhpVPW4'gKVPYVPJR!R"7;_uu_4pVP[VP]44RRR4Kj VPPP_VR#.R$7R%VPPP` R&2p\b;PdR'V 2, un2RRR4R#R#R# \8d\:P=R4ELKTd\:P?R4ELii;i +'giEK=;i +'giR#;i)*aL Setup the optional Weights & Biases (*wandb*) integration. One can subclass and override this method to customize the setup if needed. Find more information [here](https://docs.wandb.ai/guides/integrations/huggingface). You can also override the following environment variables: Environment: - **WANDB_LOG_MODEL** (`str`, *optional*, defaults to `"false"`): Whether to log model and checkpoints during training. Can be `"end"`, `"checkpoint"` or `"false"`. If set to `"end"`, the model will be uploaded at the end of training. If set to `"checkpoint"`, the checkpoint will be uploaded every `args.save_steps` . If set to `"false"`, the model will not be uploaded. Use along with [`~transformers.TrainingArguments.load_best_model_at_end`] to upload best model. - **WANDB_WATCH** (`str`, *optional* defaults to `"false"`): Can be `"gradients"`, `"all"`, `"parameters"`, or `"false"`. Set to `"all"` to log gradients and parameters. - **WANDB_PROJECT** (`str`, *optional*, defaults to `"huggingface"`): Set this to a custom string to store results in a different project. NT) ConfigErrorr peft_configrgroupzThe `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.F)repeatr- WANDB_PROJECT huggingfaceallow_val_change define_metrictrain/global_stepr) step_metric step_sync WANDB_WATCHr)loglog_freqtransformers_trainer)codemodel/num_parameterszZCould not log the number of model parameters in Weights & Biases due to an AttributeError.zbA ConfigError was raised whilst setting the number of model parameters in Weights & Biases config.model-rrnum_parameters initial_modelrrrwbr raliasesz[Visualize in Weights & Biases]() r+)all parameters gradients)3rrwandb.sdk.lib.config_utilrrto_dictr$rrmrqrr7run_namertermwarnr r5rNrOrrrrwatchmax logging_steps_labelrAttributeErrorr4rr5rrrrr;rArtifactrrrris_filenew_filewrite read_bytes log_artifacturlrAUTOGENERATED_TRAINER_COMMENT)rzrr6rrWandbConfigError combined_dictrrr7 init_args _watch_modeltemp_dir model_namemodel_artifactrfabadge_markdowns&&&&, r%setupWandbCallback.setupsr( ;;    N  & & &.t||~.Muh''ELL,D/9%,,/M/Mu||SXS_S_SgSgSi A< A= A um,,1B1B1N#// !. M} M ))JI%$.&!%)]]%E%Edoo '"*$(MM &!==DOO3KK((C$) {{&   IIo}E KK   % %m&9&9rD % Qt{{OT:: ))*=> ))#;NZ^)_99]G " ? =B=Q=Q=S ""#9:)))0022h!MM1T]]doo5U!!3!3 45%dkkoo&:&:%;< &*[[%9%9'$*geU]F^F^ELL,@,@,Bdh,dkk.@.@.D.DE[.\+T"&:&N4ED!(^00599;;!/!8!8d!8!K!Kr " 8"L!K6KKOO00,0X--1[[__-@-@,AD# ;;NCS?TT;=32*o 'Z"  p$ x 6"L!K!K)322sJ'T)DU1 %U11 U A8U1 U:U?UU U.' U11 V Nc VPfR#VPpV'd)VPP4RVnRVnVP'gVP !WV3/VBR#R#NF)rrfinishrrr)rzrr6rrr hp_searchs&&&&&, r%rWandbCallback.on_train_begin:s` ;;  //  KK   %D  DM   JJtE 4V 4!r'c <V^8dQhRS[/#rwrr )r~rs"r%rWandbCallback.__annotate__Es*P*P!2*Pr'c bVPfR#VPP'EdVP'EdVP'Ed^RIHp\P!V4pRVn RVn V!WVR.R7p \P!4;_uu_4p V PV 4VP'g}\VPP 4P#4U U u/uFBwr\%V \&P(4'gK'V P+R4'dK@WbKD up p MKRVP, 2VP.RVP0RVPP2P5R4/p RV R &\6P9R 4VP:eVP:VP<8Xd$R VPP>P@ 2M"R VPP>PB 2p\EWJ4VPPGVR V R 7p\IV 4PKR4FhpVPM4'gKVPOVPBRR7;_uu_4pVPQVPS44RRR4Kj VPP>PUVR .R7RRR4R#R#R#R#uup p i +'giK;i +'giR#;i)NTrainerfakerrprocessing_class eval_datasetrjrYztrain/total_flossrT final_modelzLogging model artifacts. ...rrrrrrr)+rrrrrrrcopydeepcopy deepspeeddeepspeed_pluginrr save_modelload_best_model_at_endrqsummaryr[rmnumbersNumberrmetric_for_best_model best_metric total_flosrrr4rrrr r;rrrrrrrrrr)rzrr6rrrrr args_for_fake fake_trainerrrbrcrrartifactrrs&&&&&&, r%rWandbCallback.on_train_endEs ;;   ?? % % %$*;*;*;@[@[@[ ) MM$/M&*M #-1M *""BRbhaiL,,..(''1 666%))<)<$=$C$C$E$EDA%a8ABcAR$E  : :;O>O+U-=-=. 0B0B0F0FG]0^+/' :; -$//1QT[[__//01!$++//"6"6!78 0@;;//ZgX`/ah,,S1Ayy{{%..qvvD.AARHHQ\\^4BA2 ,,X ,O=/.A\*; %2BAA9/..sJ'AL;$L $L =L D2L:%L L ?4LL L L L. c .ROpVPfR#VP'gVPWV4VP'dVP 4F/wrW9gK WPP P V&K1 VP 4UU u/uFwrW9gK WbK p pp \V 4p VPP/V CRVP/C4R#R#uup pi) train_runtimeNrr,train_samples_per_secondtrain_steps_per_second train_lossr&) rrrrr[r r!r9rr rzrr6rrrrsingle_value_scalarsrbrcnon_scalar_logss &&&&&&, r%rWandbCallback.on_logqs  ;;     JJtE *  & & & ,12KKOO++A.%15 ^ @]tqt O^*?;O KKOOWW0CUEVEVW X '_s ! C30C3c VVP\P8XEdVP'EdVP'Ed\ VP P4P4UUu/uFBwrV\V\P4'gK'VPR4'dK@WVbKD pppVP PPR4VR&RVP 2p\ P"P%VP&V4p \(P+RV R24VP,eVP,VP&8Xd$RVP P.P0 2M"RVP P.P2 2p VP P5V RVR7p V P7V 4VP P9V R \;VP<^4 2R VP 2.R 7R#R#R#R#uuppi) rjr checkpoint- Logging checkpoint artifacts in z. ...Nrrrepoch_checkpoint_global_step_r)rrrrrrqrr!r[rmr"r#rrrrrNrrrr4rrr r;rradd_dirrroundepoch) rzrr6rrrbrccheckpoint_metadatackpt_dir artifact_pathcheckpoint_namer)s &&&&, r%on_saveWandbCallback.on_saves ??m66 64;L;L;LQVQlQlQl!!4!45;;=#=DAa09:c9J= # ;?++:L:L:P:PQg:h  6 7$U%6%6$78HGGLL(CM KK:8*EJ KMM)T]]doo-M++,-dkkoo2234  {{++wYl+mH   ] + KK $ $VE%++q,A+B#CG^_d_p_p^qEr"s % %Rm;L 6#s7$H% H%9H%c VPfR#VP'gVP!W3/VBVP'd)\ V4pVPP V4R#R#rV)rrrrr9rrzrr6rrrs&&&&&,r% on_predictWandbCallback.on_predictsW ;;     JJt -f -  & & &"7+G KKOOG $ 'r')rrrrVNNrrrrrr{rrrrrArErrrs@r%rDrDs? OqUf 5*P*PXY*.%%r'rDca]tRtRtoRtRtRtRt]V3RlRl4t Rt R t V3R lR lt V3R lR lt RRltRV3RlRlltRRltRtRtRtRtVtR#)TrackioCallbackiz5 A [`TrainerCallback`] that logs metrics to Trackio. z(https://huggingface.co/spaces/{space_id}z0.21.1zVisualize in Trackioc&<V^8dQhRS[RS[/#)rwr-rzrf)r~rs"r%rTrackioCallback.__annotate__s ) )s )s )r'c ZVP4P4PRR4p\P!RRV4p\P!RRV4PR4pV'gRpVR,P R4p\ P!^4pV RV 2#) z\Build a Hub-safe name for a static Space from the Trackio project and a short random suffix./-z [^a-z0-9._-]z-+z-_.ztrackio-project:NQNz-static-)striplowerreplaceresubrstripsecrets token_hex)r-sbasesuffixs& r%_space_repo_name_from_project-TrackioCallback._space_repo_name_from_projects MMO ! ! # + +C 5 FF?C + FF5#q ! ' ' .!Av}}S!""1%xx((r'cz\4'g \R4h^RIpWnRVnRVnRVnR#)zLTrackioCallback requires trackio to be installed. Run `pip install trackio`.NF)r,rrr)_trackior _space_id_static_space_id)rzr)s& r%r{TrackioCallback.__init__s6#%%mn n !%),0r'c VP'Ed/VP4Cp\VR4'd[VPeM\ VP\ 4'd VPMVPP4p/VCVCp\VR4'd!VP eVP pRV/VCpVPPVPVPVPRVPVPR7VPPPP!4VnVPPP%VRR7VP'4VPPR&RVnR# \(d\*P-R 4L+i;i) z Setup the optional Trackio integration. To customize the setup you can also set `project`, `trackio_space_id`, `trackio_bucket_id`, `trackio_static_space_id`, and `hub_private_repo` in [`TrainingArguments`]. rNrallow)r-rspace_idresumeprivate bucket_idTrrzQCould not log the number of model parameters in Trackio due to an AttributeError.)rrr$rrmrqrr_r5r-rtrackio_space_idhub_private_repotrackio_bucket_id context_varscurrent_space_idrr`rrrr4rr)rzrr6rrrrrs&&&&, r%rTrackioCallback.setupsr  & & &.t||~.Muh''ELL,D/9%,,/M/Mu||SXS_S_SgSgSi A< A= A um,,1B1B1N#// !. M} M MM   ]]..--00  "]]77HHLLNDN MM ' '  ' M q?D?S?S?U $$%;<!" q op qs2'F"" GGc<V^8dQhRR/#)rwrzNr+)r~rs"r%rrLsBB4Br'c tVPfVPfR#\VRR4;'g.p\RV4R4pV'dqVP R^4^,pVP P VR7p\P;'gRpWV9dVPVR4\nVP;'g VPpVPP VR7pRV 2p VP P VR7p VU u.uF4p \V \4'dV PR 4'dK2V NK6 p p R V 3Fp W9gK V PV 4K Wn\P;'gRpW,\nR#uup i) a, Adds a link from a model's model card to the Trackio Space (preferring the `self._static_space_id` if not None). Also adds two tags to the model to indicate that it was trained with Trackio: - "trackio" - "trackio:" (this is used to embed the Space in the model's "Training Metrics" tab) N model_tagsc3"TF8p\V\4'gKVPR4'gK4VxK: R#5i)trackio:https://N)rmr}r).0ts& r% =TrackioCallback._point_model_card_at_space..s)c4a:a+=!,,OaBb4sAA A:) space_urlr)reztrackio:rsr))r`rarrsplitBADGE_MARKDOWNr~rrrS SPACE_URLrmr}rrSrq)rzrprevold_tagold_url old_commentcmtpreferred_space_id preferred_urlnew_tag new_commentrukeptextracs&& r%_point_model_card_at_space*TrackioCallback._point_model_card_at_spacesr >> !d&;&;&C ulD177Rc4ceij mmC+A.G--44w4GK99??RC!:=++kSU:V 7!22DDdnn--7I-J ]O,))00=0I a4a 1c(:(:q||L^?_4a)E  E"*   3 3 9 9r2=/ / bs "/F5F5c <V^8dQhRS[/#rr)r~rs"r%rrLs//"3/r'c ^VPRJdR#\PPVPP 4\PPVP 48d&\PRVP R24R#VP;'g0VP;'gVPVP4pVPPVPVPVVPR7Vn VPV4R#)zp Freezes the Gradio Space after training is complete, if `trackio_static_space_id` is set to a `str` or `None`. FNzAn older version of Trackio is installed; the post-training static snapshot Space (`trackio.freeze`) will not be created. Upgrade with `pip install trackio>=zO` to enable it, or set `trackio_static_space_id=False` to silence this warning.)rer- new_space_idrg)trackio_static_space_idrrrr_r MIN_TRACKIO_VERSION_FOR_FREEZEr4r5rar\r-freezer`rjr)rzrrstatic_space_ids&&& r% _freeze_spaceTrackioCallback._freeze_spaces  ' '5 0     " "4==#<#< = @Q@Q@W@W  / /A   NN))-)L)L(MNKK    ! ! u uT%A%A u uTEgEghlhthtEu !% 4 4^^LL()) !5!  ''.r'Nc VVP'gVP!WV3/VBR#R#rVrrrzrr6rrrs&&&&&,r%rTrackioCallback.on_train_begin)&   JJtE 4V 4!r'c <V^8dQhRS[/#rr)r~rs"r%rrL-saa!2ar'c 4VP'dVP'gR#VPP4VP'dVP W4R#R# \ d$p\PRT 24Rp?R#Rp?ii;i)Nz:Trackio could not freeze the Gradio Space after training: ) rrr_r r`r Exceptionr4r5)rzrr6rrrres&&&&&&, r%rTrackioCallback.on_train_end-s{***$2C2C2C   >>> a""4/  a![\][^_`` asA)) B4BBc L.ROpVP'gVPWV4VP'ddVP4UU u/uFwrW9gK WbK p pp \ V 4p VP P /V CRVP/C4R#R#uup pi)r,rNr-)rrrr[r9r_rrr1s &&&&&&, r%rTrackioCallback.on_log7s    JJtE *  & & &04 ^ @]tqt O^*?;O MM  YY2EuGXGXY Z '^s  B B c R#rVr+rs&&&&,r%rATrackioCallback.on_saveGsr'c VPfR#VP'gVP!W3/VBVP'd)\ V4pVPP V4R#R#rV)r_rrrr9rrDs&&&&&,r%rETrackioCallback.on_predictJsY ==    JJt -f -  & & &"7+G MM  g & 'r'c VP'dVPfR#VPPPP 4;pfR#VP 'gVP RJdR#VP;'g0VP ;'gVPVP4pVPPVRVVPVPRR7VnVPV4R#)NFstaticT)r-sdkrergrhforce)rr_rlcurrent_projectrr`rrar\r-syncrjrkr)rzrr6rrrrrs&&&&&, r% on_push_beginTrackioCallback.on_push_beginSs***dmm.C #}}99IIMMO OO X  >>>T99UB   ! ! u uT%A%A u uTEgEghlhthtEu !% 2 2#$)),, !3!  ''.r')rr`rar_rVrG)rrrrrr|rr{ staticmethodr\r{rrrrrrrArErrrrs@r%rJrJs;I%-" D  ) )1#!JBB@//65aa[ '//r'rJcPa]tRtRtoRtRtRtR RltR RltRt R t R t Vt R#) CometCallbackijzW A [`TrainerCallback`] that sends the logs to [Comet ML](https://www.comet.com/site/). c\RJg \RJd\R\ R\ R24hRVnRVnRVnR#)Fz!CometCallback requires comet-ml>=z- to be installed. Run `pip install comet-ml>=z`.N)r2r3rrr7r _log_assets _experimentrs&r%r{CometCallback.__init__os` % '+Be+K34F3GGtvHuIIKL " r'c 0RVn\P!RR4P4pVR9dRVnVP 'Ed\P!R4pRpRpVe;VP 4pVR9dTpM V'd\PRV4R#VP'dVe\PRV4Rp^RI pVP!VPR 7p VP!WvV R 7VnVPPVR R 7R VP!4/p \#VR4'd-VP$eVP$P!4p WR&\#VR4'dVP&eVP&p WR&VPP)V R RRR7VP'd9\+VRR4p \+VRR4pVPP-WR7R#R#R#)a| Setup the optional Comet integration. Environment: - **COMET_MODE** (`str`, *optional*, default to `get_or_create`): Control whether to create and log to a new Comet experiment or append to an existing experiment. It accepts the following values: * `get_or_create`: Decides automatically depending if `COMET_EXPERIMENT_KEY` is set and whether an Experiment with that key already exists or not. * `create`: Always create a new Comet Experiment. * `get`: Always try to append to an Existing Comet Experiment. Requires `COMET_EXPERIMENT_KEY` to be set. - **COMET_START_ONLINE** (`bool`, *optional*): Whether to create an online or offline Experiment. - **COMET_PROJECT_NAME** (`str`, *optional*): Comet project name for experiments. - **COMET_LOG_ASSETS** (`str`, *optional*, defaults to `TRUE`): Whether or not to log training assets (checkpoints, etc), to Comet. Can be `TRUE`, or `FALSE`. For a number of configurable items in the environment, see [here](https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options). TCOMET_LOG_ASSETSrK COMET_MODENcreatez:Invalid COMET_MODE env value %r, Comet logging is disabledzjHyperparameter Search is enabled, forcing the creation of new experiments, COMET_MODE value %r is ignored)r)onliner experiment_config transformers) frameworkrrrmanual)rsourceflatten_nestedr7 trial_params)optimization_idr>1rL)r get_or_creater)rrNrOrPrrrRr4r5rrExperimentConfigrstartr!__internal_api__set_model_graph__rr$rr __internal_api__log_parameters__rlog_optimization)rzrr6r log_assetscomet_old_moder rrrrprrroptimization_paramss&&&& r%rCometCallback.setupxs2!YY17;AAC  &#D   & & &YY|4NDF)!/!5!5!7!%GG)D#NN#_aop***#NNE&   ( 9 9t}} M '~~VZklD     > >uP^ > _dlln-Fuh''ELL,D$||335 #/x um,,1B1B1N#// (3}%    = =.RV > ***")%t"D&-e^T&J#  11/1r +W 'r'Nc RVP'gVPWV4R#R#rVrrs&&&&&,r%rCometCallback.on_train_begin!   JJtE *!r'c VP'gVPWV4VP'dPVPe@\ V4pVPP WrP VPRR7R#R#R#)Nrstepr<rrrrrr9__internal_api__log_metrics__rr<)rzrr6rrrrrewritten_logss&&&&&&, r%rCometCallback.on_logss   JJtE *  & & &+!-d!3  >>"):):%++Yg?, 'r'c VP'dVP'dVPeYVPRJdI\P R4VPP VPRRVPR7VP'd$VPP4RVnR#R#R#R#)NTz(Logging checkpoints. This may take time.) recursive log_file_namerF) rrrrr4rlog_asset_folderrrrcleanrs&&&&,r%rCometCallback.on_train_ends    !a]tRtRtoRtRRltRtRRltRtVt R#) AzureMLCallbackiz` A [`TrainerCallback`] that sends the logs to [AzureML](https://pypi.org/project/azureml-sdk/). NcH\4'g \R4hWnR#)zPAzureMLCallback requires azureml to be installed. Run `pip install azureml-sdk`.N)rHrr azureml_run)rzrs&&r%r{AzureMLCallback.__init__s#%%qr r&r'c ^RIHpVPf,VP'dVP 4VnR#R#R#)rlRunN)azureml.core.runrrr get_context)rzrr6rrrs&&&&, r% on_init_endAzureMLCallback.on_init_ends4(    #(C(C(C"0D )D #r'c VP'dkVP'dWVP4F@wrg\V\\ 34'gK#VPP WgVR7KB R#R#R#)) descriptionN)rrr[rmr|rrrs&&&&&, r%rAzureMLCallback.on_logs]     ; ; ; a#u..$$((1(=%!< r')rrV) rrrrrr{rrrrrs@r%rrs' 1 >>r'rcVa]tRtRtoRtRtRtR RltR RltRt R t R t R t Vt R#) MLflowCallbackiz A [`TrainerCallback`] that sends the logs to [MLflow](https://www.mlflow.org/). Can be disabled by setting environment variable `DISABLE_MLFLOW_INTEGRATION = TRUE`. c\4'g \R4h^RIpVPPP VnVPPPVnRVn RVn RVn Wn R#)zIMLflowCallback requires mlflow to be installed. Run `pip install mlflow`.NF) rQrrrMutils validationMAX_PARAM_VAL_LENGTH_MAX_PARAM_VAL_LENGTHMAX_PARAMS_TAGS_PER_BATCH_MAX_PARAMS_TAGS_PER_BATCHr _auto_end_run_log_artifacts_ml_flow)rzrMs& r%r{MLflowCallback.__init__ sg"$$jk k%+\\%<%<%Q%Q"*0,,*A*A*[*['!"# r'c  \P!RR4P4\9Vn\P!RR4P4\9Vn\P!RR4Vn\P!RR4Vn\P!RR4P4\9Vn\P!RR4Vn \P!R R4Vn \PPVPP4\PPR 48Vn\"P%R VP R VP& R VP RVP 24VP('EdVPP+4'gqVP 'dIVPP-VP 4\"P%RVP 24MF\"P%R4M0\"P%RVPP/4 24VPP14e%VP 'gVP'dVP'd&VPP3VP4VPP5VP&VP R7\"P%RVPP14P6P8 24RVnVP=4p\?VR4'd/VP@e!VP@P=4p/VCVCpVP'd \CV4MTp\EVPG44FHwrg\I\KV44VPL8gK*\"PORV RV R24WFKJ \EVPG44pVP'dpVPPQ4'dP\SVP4p V \IV48d+\"P%R\IV4 RV R24VRV p\U^\IV4VPV4Fp VP 'd;VPPY\[WWPV,4RR7KOVPPY\[WWPV,44K \P!RR4p V 'd2\\P^!V 4p VPPaV 4RVn1R#)a Setup the optional MLflow integration. Environment: - **HF_MLFLOW_LOG_ARTIFACTS** (`str`, *optional*): Whether to use MLflow `.log_artifact()` facility to log artifacts. This only makes sense if logging to a remote server, e.g. s3 or GCS. If set to `True` or *1*, will copy each saved checkpoint on each save in [`TrainingArguments`]'s `output_dir` to the local or remote artifact storage. Using it without a remote storage will just copy the files to your artifact location. - **MLFLOW_TRACKING_URI** (`str`, *optional*): Whether to store runs at a specific path or remote server. Unset by default, which skips setting the tracking URI entirely. - **MLFLOW_EXPERIMENT_NAME** (`str`, *optional*, defaults to `None`): Whether to use an MLflow experiment_name under which to launch the run. Default to `None` which will point to the `Default` experiment in MLflow. Otherwise, it is a case sensitive name of the experiment to be activated. If an experiment with this name does not exist, a new experiment with this name is created. - **MLFLOW_TAGS** (`str`, *optional*): A string dump of a dictionary of key/value pair to be added to the MLflow run as tags. Example: `os.environ['MLFLOW_TAGS']='{"release.candidate": "RC1", "release.version": "2.2.0"}'`. - **MLFLOW_NESTED_RUN** (`str`, *optional*): Whether to use MLflow nested runs. If set to `True` or *1*, will create a nested run inside the current run. - **MLFLOW_RUN_ID** (`str`, *optional*): Allow to reattach to an existing run which can be useful when resuming training from a checkpoint. When `MLFLOW_RUN_ID` environment variable is set, `start_run` attempts to resume a run with the specified run ID and other parameters are ignored. - **MLFLOW_FLATTEN_PARAMS** (`str`, *optional*, defaults to `False`): Whether to flatten the parameters dictionary before logging. - **MLFLOW_MAX_LOG_PARAMS** (`int`, *optional*): Set the maximum number of parameters to log in the run. HF_MLFLOW_LOG_ARTIFACTSrKMLFLOW_NESTED_RUNMLFLOW_TRACKING_URINMLFLOW_EXPERIMENT_NAMEMLFLOW_FLATTEN_PARAMS MLFLOW_RUN_IDMLFLOW_MAX_LOG_PARAMSz2.8.0zMLflow experiment_name=z , run_name=z , nested=z, tracking_uri=zMLflow tracking URI is set to zdEnvironment variable `MLFLOW_TRACKING_URI` is not provided and therefore will not be explicitly set.)rnestedzMLflow run started with run_id=Trrz " for key "z" as a parameter. MLflow's log_param() only accepts values no longer than 250 characters so we dropped this attribute. You can use `MLFLOW_FLATTEN_PARAMS` environment variable to flatten the parameters and avoid this message.z.Reducing the number of parameters to log from z to .F) synchronous MLFLOW_TAGS)2rNrOrPrr _nested_run _tracking_uri_experiment_name_flatten_params_run_id_max_log_paramsrrrrr _async_logr4debugrris_tracking_uri_setset_tracking_uriget_tracking_uri active_runset_experiment start_runrr*rrr$rrrr[rZr}rr5isdigitr|rr log_paramsrqjsonloadsset_tagsr) rzrr6rrrrrcombined_dict_itemsmax_log_paramsr mlflow_tagss &&&& r%rMLflowCallback.setups{@!ii(A7KQQSWkk99%8'BHHJNbbYY'\>\>^=_`a}}'')1T5E5E5E(((MM001F1FG ''tGWGW'X >t}}?W?W?Y?^?^?e?e>fgh%)" LLNMuh''ELL,D$||335 A< A= A ;?;O;O;OL7UbM#M$7$7$9: s5z?T%?%??NNCE7+VZU[\// &+ ;#'}':':'<"= ###(<(<(D(D(F(F!$T%9%9!:!C(;$<<LLHM`IaHbbfgufvvwx+>o~*N'1c"568W8WX???MM,,0Q9X9X5XYZhm-MM,,T2E!NmNmJm2n-op Y))M48K"jj5  &&{3 r'Nc RVP'gVPWV4R#R#rVrrs&&&&&,r%rMLflowCallback.on_train_beginrr'c VP'gVPWV4VP'EdL/pVP4Fwr\ V \ \ 34'dWV&K'\ V \P4'd*V P4^8XdV P4Wx&Kp\PRV R\V 4 RV R24K VP4UU u/uFwr\P!RRV4V bK p pp VP 'd*VP"P%WP&RR7R #VP"P%WP&R 7R #R #uup pi) rrrzc" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.z[^0-9A-Za-z_\-\.\ :/]rjF)rrr)rrN)rrrr[rmr|rrTensornumelitemr4r5rrTrUr r log_metricsr) rzrr6rrrrrrbrcsanitized_metricss &&&&&&, r%rMLflowCallback.on_logsB   JJtE *  & & &G a#u..!"AJ5<<00QWWY!^!"GJNNCA3jQUVWQXPYYcdecfgoo %ZaYfYfYh iYhQUQR(@#q!I1!LYh  i ))2CJ[J[in)o ))2CJ[J[)\' '!js9$Fc VP'dgVP'dSVP'd?VPP 4'dVPP 4R#R#R#R#R#rV)rrrrrend_runrs&&&&,r%rMLflowCallback.on_train_endsX    !&>&@&@ %%''A!"= r'c VP'dVP'dVP'dRVP 2p\P P VPV4p\PRV R24VPPPVRV/VPPP4R7R#R#R#R#)r6r7z. This may take time. model_path) artifacts python_modelN)rrrrrNrrrr4rrpyfunc log_model PythonModelrzrr6rrr>r?s&&&&, r%rAMLflowCallback.on_saves    ! [!WARNING] > Neptune integration is deprecated and will be removed in a future version of Transformers. We recommend using > other supported experiment tracking integrations. Args: api_token (`str`, *optional*): Neptune API token obtained upon registration. You can leave this argument out if you have saved your token to the `NEPTUNE_API_TOKEN` environment variable (strongly recommended). See full setup instructions in the [docs](https://docs.neptune.ai/setup/installation). project (`str`, *optional*): Name of an existing Neptune project, in the form "workspace-name/project-name". You can find and copy the name in Neptune from the project settings -> Properties. If None (default), the value of the `NEPTUNE_PROJECT` environment variable is used. name (`str`, *optional*): Custom name for the run. base_namespace (`str`, *optional*, defaults to "finetuning"): In the Neptune run, the root namespace that will contain all of the metadata logged by the callback. log_parameters (`bool`, *optional*, defaults to `True`): If True, logs all Trainer arguments and model parameters provided by the Trainer. log_checkpoints (`str`, *optional*): If "same", uploads checkpoints whenever they are saved by the Trainer. If "last", uploads only the most recently saved checkpoint. If "best", uploads the best checkpoint (among the ones saved by the Trainer). If `None`, does not upload checkpoints. run (`Run`, *optional*): Pass a Neptune run object if you want to continue logging to an existing run. Read more about resuming runs in the [docs](https://docs.neptune.ai/logging/to_existing_object). **neptune_run_kwargs (*optional*): Additional keyword arguments to be passed directly to the [`neptune.init_run()`](https://docs.neptune.ai/api/neptune#init_run) function when a new run is created. For instructions and examples, see the [Transformers integration guide](https://docs.neptune.ai/integrations/transformers) in the Neptune documentation. z%source_code/integrations/transformersmodel_parametersrnrtrainer_parametersz train/epoch api_tokenNr-rbase_namespace finetuningr log_parametersTlog_checkpointsc v<V^8dQhRS[R,RS[R,RS[R,RS[RS[RS[R,/#)rwr\Nr-rr]r_r`)r}r)r~rs"r%rNeptuneCallback.__annotate__sa:D:D::Dt :D Dj :D  :D:Dt:Dr'c 0\P!R\4\4'g \ R4h^RIHp ^RIHp V !RV\\R434V !RV\\R434V !RV\\R434V !RV\4V !R WY\R434V !R V\4V !R V\\R434W@nW`nWpnWPnRVnR VnRVnR VnRVRVRV/VCVnRVnVP"RJVnRVnVP"R9dR VP" 2VnRVnR#RVnR VnR# \d^RI Hp ^RI Hp ELPi;i)zThe NeptuneCallback is deprecated and will be removed in a future version of Transformers. We recommend using other supported experiment tracking integrations.zwNeptuneCallback requires the Neptune client library to be installed. To install the library, run `pip install neptune`.r) verify_typer\Nr-rr]r r_r`Fz checkpoints/T checkpoints>rr)warningswarn FutureWarningrX ValueErrorrrneptune.internal.utilsrdrAneptune.new.internal.utils#neptune.new.metadata_containers.runr}rr_base_namespace_path_log_parameters_log_checkpoints _initial_run_run_is_monitoring_runr _force_reset_monitoring_run_init_run_kwargs_volatile_checkpoints_dir_should_upload_checkpoint_recent_checkpoint_path_target_checkpoints_namespace*_should_clean_recently_uploaded_checkpoint) rzr\r-rr]r r_r`neptune_run_kwargsrrds &$$$$$$$, r%r{NeptuneCallback.__init__s   F  $%%E   @ # : KS$t*,=>Iwd4j(9:FD3T "34$nc:E3d4j 12$nd;%d4j8IJ$2!- /(+ "' +0(!,iGVUY p]o p)-&)-)>)>d)J&'+$  $4 43?@U@U?V1WD .>BD ;1>D .>CD ;A @ > ? ? @s E::FFctVP'd&VPP4V=RVnR#R#rV)rqstoprs&r%_stop_run_if_exists#NeptuneCallback._stop_run_if_existsXs* 999 IINN  DI r'c v^RIHp^RIHpHpVP4VP4pVPVP4V!R/VBVn VPR,P4VnR# \ d^RIHp^RIHpHpLi;i YC3dp\4ThRp?ii;i)rl)init_run)NeptuneMissingApiTokenException"NeptuneMissingProjectNameExceptionsys/idNr+)rrneptune.exceptionsrrrA neptune.newneptune.new.exceptionsr~rrrtrqfetchr rU)rzadditional_neptune_kwargsrrr run_paramsrs&, r%_initialize_runNeptuneCallback._initialize_run^s s ( n   " 72779J   d33 4 .:.DI99X.446DL s , r r s3T 7-/Q 6 7s)BAB BB B8( B33B8cVPVnRVnVPR,P4VnRVnR#)TrN)rprqrrrr rs&r%_use_initial_run NeptuneCallback._use_initial_runps8%% "&yy*002  r'cvVPeVP4R#VP'gVP'dR#VP'dJVP'g8VP'g&VP VP R7RVnR#VP 4RVnR#)N)with_idTF)rprrsrrrqrr rs&r%_ensure_run_with_monitoring+NeptuneCallback._ensure_run_with_monitoringvs    (  ! ! #3338O8O8Oyyy!8!8!8AaAaAa$$T\\$:*.'$$&380r'cVPeVP4R#VP'g*VPVPRRRRR7RVnR#R#)NF)rcapture_stdoutcapture_stderrcapture_hardware_metricscapture_traceback)rprrqrr rrrs&r%'_ensure_at_least_run_without_monitoring7NeptuneCallback._ensure_at_least_run_without_monitoringsX    (  ! ! #999$$ LL#(#(-2&+ %+0'r'cVVPfVP4VP#rV)rqrrs&r%r NeptuneCallback.runs" 99   8 8 :yyr'c<VPVP,#rV)r rmrs&r%_metadata_namespace#NeptuneCallback._metadata_namespacesxx1122r'cH\VP\P&R#rV)rr rYintegration_version_keyrs&r%_log_integration_version(NeptuneCallback._log_integration_versionsV!VPP 44VP \ P&R#R#R#R#)rl)stringify_unsupportedrN) neptune.utilsrr$rrrrYmodel_parameters_key)rzrrs&& r%_log_model_parameters%NeptuneCallback._log_model_parameterssT7 WUH--%,,2JMb $$&ND $ $_%I%I J3K-5r'c2V'd:\VR4'd(VPVP\P&V'dN\VR4'd:VP e*VP VP\P &R#R#R#R#)r7rN)r$r7rrYtrial_name_keyrtrial_params_key)rzr6s&&r%"_log_hyper_param_search_parameters2NeptuneCallback._log_hyper_param_search_parameterssn WUL11GLGWGWD $ $_%C%C D WUN338J8J8VINI[I[D $ $_%E%E F9W35r'c&<V^8dQhRS[RS[/#)rwsource_directoryrrf)r~rs"r%rrbs55c5s5r'c \PPW4;r4VPe\PPVPV4pVP RR4P \PP 4p\PPWV4p\P!WG4TpVPVP,PV4VP'dEVP e7VPVP,P#VP 4W@nR# \d$p\PRT R24Rp?LRp?ii;i)Nz..rzONeptuneCallback was unable to made a copy of checkpoint due to I/O exception: 'z'. Could fail trying to upload.)rNrrrurSlstriprIshutilcopytreeOSErrorr4r5rrx upload_filesryrw delete_files) rzrr target_path relative_pathconsistent_checkpoint_path cpkt_path copy_pathrs &&& r%_log_model_checkpoint%NeptuneCallback._log_model_checkpoints*&(ggll3C&PP  ) ) 5)+d6T6TV`)a & )11$;BB277;;O GGLL)CO  98    !C!CDQQR]^  : : :t?[?[?g  $ $T%G%G H U UVZVrVr s'4$ efgehi33 sA0E FE==Fc RVnVP'd3VPe%\P!4P VnVPR8Xd VP 'g \R4hR#R#)NrzWTo save the best model checkpoint, the load_best_model_at_end argument must be enabled.)rurosave_total_limitrrrr rirs&&&&,r%rNeptuneCallback.on_init_endsf)-&  T%:%:%F-5-H-H-J-O-OD *  F *43N3N3Nvw w4O *r'c .VP'gR#VP4RVnVP4VP'd#VP V4VP V4VP'dVPV4R#R#)NT) rrrsrrnrrrrrs&&&&&,r%rNeptuneCallback.on_train_beginsx***  ((*+/( %%'      ( ( .  & &u -  & & &  3 3E : 'r'c &VP4R#rV)r~rs&&&&,r%rNeptuneCallback.on_train_ends   "r'cVPe#\P!VPRR7VP4R#)NT) ignore_errors)rurrmtreer~rs&r%r5NeptuneCallback.__del__s.  ) ) 5 MM$88 M   "r'c VP'd,VPVPRVP 24R#R#)r6N)rvrrrrs&&&&,r%rANeptuneCallback.on_saves5  ) ) )  & &t+eFWFWEX8Y Z *r'c \VPR8XdVPpVPR4'gRV 2pVPV4pVP'd\ P M\ PpVPRJ;'gV!WrP4Vn R#R#)rrWN) ror$rrgreater_is_betternpgreaterlessr%rv) rzrr6rrrbest_metric_name metric_valueoperators &&&&&, r% on_evaluateNeptuneCallback.on_evaluates  F *#99 #..w77%*+;*<#= ";;'78L%)%;%;%;rzzH-2->->$-F-s-s(S_ararJsD * +r'cVPPF"p\W 4'gKVPu# \ R4h)z6The trainer doesn't have a NeptuneCallback configured.)rBrCrmr r)rrrMs&& r%get_runNeptuneCallback.get_runs;00::H(((||#;PQQr'cD<V^8dQhRS[S[S[3,R,/#)rwrN)rqr}r)r~rs"r%rrbs% Z Zc5j1AD1H Zr'c ^VP'gR#Ve\V4P4Fuwrg\V\\ 34'gK#V\ P9dWpPV&KHVPV,PWrPR7Kw R#R#)Nr) rr9r[rmr|rrY flat_metricsrrr)rzrr6rrrrrs&&&&&, r%rNeptuneCallback.on_logs***   +D1779 ec5\22;;;9>006006::5GXGX:Y : r')rmrsrtrprrrornrwrqr ryrvrxrurV)$rrrrrrrrrrrr{r~rrrrrr rrrrrrrrrr5rArrrrrrrs@r%rYrYs7@F-N%1!?L:D!%:D# :D  :D + :D :D $:D'+:D:Dx 7$! 9 0 33Dd\550x ;## [ tRR Z Z Zr'rYc@a]tRtRtoRtRtRtR RltRtRt Vt R#) CodeCarbonCallbackizA A [`TrainerCallback`] that tracks the CO2 emission of training. c\4'g \R4h\PP'd \R4h^RIpWnRVnR#)zWCodeCarbonCallback requires `codecarbon` to be installed. Run `pip install codecarbon`.aZCodeCarbonCallback requires `codecarbon` package, which is not compatible with AMD ROCm (https://github.com/mlco2/codecarbon/pull/490). When using the Trainer, please specify the `report_to` argument (https://huggingface.co/docs/transformers/v4.39.3/en/main_classes/trainer#transformers.TrainingArguments.report_to) to disable CodeCarbonCallback.N)r[rrrrhiprZ _codecarbontracker)rzrZs& r%r{CodeCarbonCallback.__init__sR&((i ]]   m  % r'c VPfBVP'd.VPPVPR7VnR#R#R#)N)r)ris_local_process_zerorEmissionsTrackerrrs&&&&,r%rCodeCarbonCallback.on_init_end"s? << E$?$?$?++<<<XDL%@ r'Nc VP'd1VP'dVPP4R#R#R#rV)rrrrs&&&&&,r%r!CodeCarbonCallback.on_train_begin's- <<  ! ! #Mop p!!"$r'c  VPfR#VP'dR#\;P^, un\;P^, un\P^8XdRMR\ \P4,\nVP'Ed\PR4VPEf0\PflVPPP4'd+VPPP4'd R\n M R\n VPPP4'dVPPP4'dVPPP4Vn \P !R\P"'gRMR4P%4\&P(!R049Vn\PR 4MVPPP-\P !R R 4\P !R R 4RRRR/RR7Vn \P !RR4P%4\&P(!R049VnR\n\PR4RVn\P.\P ,pVR,\P0,pVPPP4'd6VP3W4VPP5VR\6RR7MuVPP9VRRR7'gVPP;W4M5VP3V\P.\P ,4\=VRR4EeVR,\P>,p\P@PC\P4p \P\P8wdV R\PD,, p VPPP4'dVPP5VR\6RR7VPPG\PH\P ,VPJPM4V R7R#VPP9VRRR7'g_VPJPOVPPQ\PH\P ,44Vn%R#VPPG\PH\P ,VPJPM4V R7R#R#R#)Nrrjz"Automatic ClearML logging enabled.FTCLEARML_LOG_MODELrKrLz)External ClearML Task has been connected.CLEARML_PROJECTzHuggingFace Transformers CLEARML_TASKrr;pytorch) project_name task_nameauto_connect_frameworks output_uriz"ClearML Task has been initialized.rN)rr value_typer)defaultcastr )r config_dictrzIf True, ignore Transformers hyperparameters overrides done in the UI/backend when running remotely. Otherwise, the overrides will be applied when running remotelyzIf True, ignore Transformers model configuration overrides done in the UI/backend when running remotely. Otherwise, the overrides will be applied when running remotely))rrr_train_run_counter_model_connect_counterr} log_suffixrr4rr_should_close_on_train_endTaskrunning_locally current_taskrNrO_task_created_in_callbackrPrunionrr5_hparams_section_ignore_hparams_overrides_copy_training_args_as_hparams set_parameterr get_parameterconnectr_ignoge_model_config_overrides_model_config_descriptionr~_model_config_description_noteset_configuration_object_model_config_sectionrr from_dict get_configuration_object_as_dict) rzrr6rrrsuffixed_hparams_sectionignore_hparams_config_sectionignore_model_config_section configuration_object_descriptions &&&&&, r%rClearMLCallback.setup]s ==      **a/*..!3.!449BsSIkIkEl?l "  & & & KK< =!!)"==E==--==??4==CUCUCbCbCdCdEJBEIB==%%5577DMMyRW0X#' *A*D& ')ii0CV&L&R&R&TXlXrXrY'DOAEO=KK DE$(!'6'G'G/JdJd'd $,Ds,J_MvMv,v )}}!!113333DS""006#r 1''556S]ahl5mm""**4J33/::_=W=WWuh-9,s2_5c5cc,4C3\3\3c3c#::40#99_=_=__4o>l>l8ll4==%%5577&&448"#'v 5&&??,BB_E_E__$)LL$8$8$:$D@ ++99:U_cjn9oo#(<<#9#9**KK+AAOD^D^^$EL &&??,BB_E_E__$)LL$8$8$:$D@?:i 'r'c VPfR#.VnVP'dRVnVP'gVP!WWE3/VBR#R#r )rrrrrrzrr6rrrrs&&&&&&,r%rClearMLCallback.on_train_beginsO == "$  & & & %D    JJtE Fv F!r'c \P'd(VPP4^\nR#R#)rlN)rr rrr rs&&&&,r%rClearMLCallback.on_train_ends-  5 5 5    $ $ &12O . 6r'c dVPfR#VP'gVP!WWE3/VBVP'EdRp\ V4p Rp \ V 4p .R Op VP 4EFwr\ V\\34'EdgW9dCVPP4PV \P,VR7KkV PV4'dQVPP4PR\P,WRVVP R7KV PV 4'dRVPP4PR\P,WRVVP R7EK:VPP4PR\P,V VVP R7EK\"P%RV R \'V4 R V R 24EK R#R#) NrWrX)rreval)titleseriesr iterationtestrrrrzn" as a scalar. This invocation of ClearML logger's report_scalar() is incorrect so we dropped this attribute.)r,r.r/r0r&r<)rrrrrZr[rmr|rr get_loggerreport_single_valuerr r report_scalarrr4r5r)rzrr6rrrrrr^r_r`rar2rbrcs&&&&&&&, r%rClearMLCallback.on_logs ==    JJtE Fv F  & & &!K!+.O!K!+.O$  a#u..0**557KK!"_%?%?!?qLk22**557EE"(?+E+E"E#$%5#6"#&+&7&7 F k22**557EE"(?+E+E"E#$%5#6"#&+&7&7 F**557EE")O,F,F"F#$"#&+&7&7 FNN3ja A3?EE9% 'r'c VP'EdVP'EdVP'EdRVP 2p\P P VPV4pV\P,p\PRV R24VPPVPVR7pVPVPVR7VPVVVPRR7VP P#V4VP$'dVP$\'VP 48d`VPP(P*P-VP ^,RRRR7TP R ,TnKR #R #R #R #R # \.dBp \P1RTP ^,P2 R T 24R p ? R #R p ? ii;i) r6zLogging checkpoint artifact `z`. This may take some time.)taskrF) weights_pathtarget_filenamer.auto_delete_fileT)delete_weights_filerraise_on_errorszCould not remove checkpoint `z5` after going over the `save_total_limit`. Error is: N:r NN)rrrrrNrrrrr r4rr OutputModelrupdate_weights_packagerrSrrZrModelremoverr5r) rzrr6rrr>r?r output_modelrs &&&&, r%rAClearMLCallback.on_saves ???t111e6Q6Q6Q$U%6%6$78HGGLL(CMo888D KK7v=XY Z==44$:L:LSW4XL  d&8&8t D  / /* (++!& 0   # # * *< 8'''D,A,ACH_H_D`,` MM''--44//2,0"(, 5+/*A*A"*E'-a'7R1?,!NN78O8OPQ8R8W8W7XYNOPNQR sAG H  6HH c\V4Uu/uFZpVP'gKVPPR4'dK:VP\ WP4bK\ ppVP P PPV4P4UUu/uFwrV\V4VbK pppVPPPWrR7R#uupiuuppi)_token)prefixN)rr5rendswithrr utilities proxy_objectflatten_dictionaryr[r}r _argumentscopy_from_dict)rz training_argsrCfieldas_dictrbrc flat_dicts&&& r%r.ClearMLCallback._copy_training_args_as_hparams!s . .zz ;"'**"5"5h"? ;EJJ zz: :.  ,0==+B+B+O+O+b+bcj+k+q+q+st+s41SVQY+s t %%44Y4N  usC2C2 $C20C7)rrrrrrG)NNN)rrrrrr rrrrrrr r rr r{rrrrrArrrrs@r%rr0s J%1 ?%I" G 9#  %!% %aFG3 3jF<OOr'rcRaa]tRtRtoRtRV3RlV3RllltRtRtRtVt V;t #) FlyteCallbacki+aA [`TrainerCallback`] that sends the logs to [Flyte](https://flyte.org/). NOTE: This callback only works within a Flyte task. Args: save_log_history (`bool`, *optional*, defaults to `True`): When set to True, the training logs are saved as a Flyte Deck. sync_checkpoints (`bool`, *optional*, defaults to `True`): When set to True, checkpoints are synced with Flyte and can be used to resume training in the case of an interruption. Example: ```python # Note: This example skips over some setup steps for brevity. from flytekit import current_context, task @task def train_hf_transformer(): cp = current_context().checkpoint trainer = Trainer(..., callbacks=[FlyteCallback()]) output = trainer.train(resume_from_checkpoint=cp.restore()) ``` c&<V^8dQhRS[RS[/#)rwsave_log_historysync_checkpointsr)r~rs"r%rFlyteCallback.__annotate__Fs1111r'c<\SV`4\4'g \R4h\ 4'd\ 4'g\ PR4Rp^RIH pV!4PVn Wn W n R#)zLFlyteCallback requires flytekit to be installed. Run `pip install flytekit`.zSyncing log history requires both flytekitplugins-deck-standard and pandas to be installed. Run `pip install flytekitplugins-deck-standard pandas` to enable this feature.F)current_contextN)r;r{r^rAr`rr4r5r]rVrcprRrS)rzrRrSrVrss&&& r%r{FlyteCallback.__init__Fsn $&&lm m/119L9N9N NNa  % ,!#.. 0 0r'c 0VP'dVP'dpRVP 2p\PP VP V4p\PRV R24VPPV4R#R#R#)r6zSyncing checkpoint in z to Flyte. This may take time.N) rSrrrNrrrr4rrWrNr1s&&&&, r%rAFlyteCallback.on_saveXst  U%@%@%@$U%6%6$78HGGLL(CM KK0 :XY Z GGLL ' &A r'c VP'dK^RIp^RIHp^RIHpVP VP4pV!RV!4PV44R#R#)rlN)Deck) TableRendererz Log History) rRpandasr]r\flytekitplugins.deck.rendererr] DataFrame log_historyto_html) rzrr6rrpdr\r]log_history_dfs &&&&, r%rFlyteCallback.on_train_end`sG   % C\\%*;*;D(4/(r'c \4'g \R4h^RIHpRVnRVn\ W4'dWnMVe\RVP R24hW nVPfZ\P!RR4pVP4\9d RVnR#VP4R 8Xd R VnR#R#R#) zLDVCLiveCallback requires dvclive to be installed. Run `pip install dvclive`.LiveFNz Found class z for live, expected dvclive.LiveHF_DVCLIVE_LOG_MODELrKTr)rcrrrbrmrrirmrsrrNrOrPrrR)rzrir/rrm log_model_envs&&&, r%r{DVCLiveCallback.__init__zs $%%mn n ! d ! !I  dnn-==]^_ _# ?? "II&Y>Y>Y II " "4?? 3?Z(9 #r'c VP'dVP'd^RIHpVPRJdV!VVP R4VP R4R.R7pVP 'dRMRp\PPVPV4pVPV4VPPWRRR 7VPP4R #R #R #) rlrTrrrrrr)rrrN)rrtransformers.trainerrrrr rNrrrrrirr) rzrr6rrrr(rrs &&&&, r%rDVCLiveCallback.on_train_ends    !r'c ,RVnVP'EdT\PR4/VP 4Cp\ VR4'd[VP eM\VP \4'd VP MVP P 4p/VCVCp\ VR4'd!VPeVPpRV/VCpVPp/p Ve$VPeVP RV 2V R&M&VPeVPV R&MVeWR&\P!RR4V R &\P!R R4p V eWR &\P!R R4p V eWR &MVP'dRV R &VPP!4fVPP"!R/V BRVPP R&VPP P%V4VPP P%RVP'4/4\)V4P*R8Xg\)V4P*R8XdaVP-4wrVPP P%RV /4VPP P%RV /4TP0et\P3R4RTPP!4P4P6P8 R2p\:;P<RT 2, unR#R#R# \.d\PR4Li;i)a Setup the optional SwanLab (*swanlab*) integration. One can subclass and override this method to customize the setup if needed. Find more information [here](https://docs.swanlab.cn/guide_cloud/integration/integration-huggingface-transformers.html). You can also override the following environment variables. Find more information about environment variables [here](https://docs.swanlab.cn/en/api/environment-variable.html#environment-variables) Environment: - **SWANLAB_API_KEY** (`str`, *optional*, defaults to `None`): Cloud API Key. During login, this environment variable is checked first. If it doesn't exist, the system checks if the user is already logged in. If not, the login process is initiated. - If a string is passed to the login interface, this environment variable is ignored. - If the user is already logged in, this environment variable takes precedence over locally stored login information. - **SWANLAB_PROJECT** (`str`, *optional*, defaults to `None`): Set this to a custom string to store results in a different project. If not specified, the name of the current running directory is used. - **SWANLAB_LOG_DIR** (`str`, *optional*, defaults to `swanlog`): This environment variable specifies the storage path for log files when running in local mode. By default, logs are saved in a folder named swanlog under the working directory. - **SWANLAB_MODE** (`Literal["local", "cloud", "disabled"]`, *optional*, defaults to `cloud`): SwanLab's parsing mode, which involves callbacks registered by the operator. Currently, there are three modes: local, cloud, and disabled. Note: Case-sensitive. Find more information [here](https://docs.swanlab.cn/en/api/py-init.html#swanlab-init) - **SWANLAB_LOG_MODEL** (`str`, *optional*, defaults to `None`): SwanLab does not currently support the save mode functionality.This feature will be available in a future release - **SWANLAB_WEB_HOST** (`str`, *optional*, defaults to `None`): Web address for the SwanLab cloud environment for private version (its free) - **SWANLAB_API_HOST** (`str`, *optional*, defaults to `None`): API address for the SwanLab cloud environment for private version (its free) - **SWANLAB_RUN_ID** (`str`, *optional*, defaults to `None`): Experiment ID to resume a previous run. Use with `SWANLAB_RESUME` to continue an existing experiment. - **SWANLAB_RESUME** (`str`, *optional*, defaults to `None`): Resume mode (`"must"`, `"allow"`, `"never"`). Defaults to `"allow"` when `resume_from_checkpoint` is used. TzYAutomatic SwanLab logging enabled, to disable set os.environ["SWANLAB_MODE"] = "disabled"rNrrOexperiment_nameSWANLAB_PROJECTr-SWANLAB_RUN_IDr;SWANLAB_RESUMErfrdu🤗transformers FRAMEWORKmodel_num_parameters PeftModelPeftMixedModelpeft_model_trainable_paramspeft_model_all_paramzQCould not log the number of model parameters in SwanLab due to an AttributeError.sSwanLab does not currently support the save mode functionality. This feature will be available in a future release.z[Visualize in SwanLab](rrr+)rrr4rrr$rrmrqrr7rrNrOrrrr5rrrrget_nb_trainable_parametersrrr5publiccloudexperiment_urlrr)rzrr6rrrrrr7rr*rftrainable_params all_paramrs&&&&, r%rSwanLabCallback.setupsb!  & & & KKs t.t||~.Muh''ELL,D/9%,,/M/Mu||SXS_S_SgSgSi A< A= A um,,1B1B1N#// !. M} M ))JI%$--*C26--*/N +,*/3}} +,'/9+,#%99->#EIi YY/6F!"($YY/6F!&,(#,,,&- (#}}$$&. ""1CDMM  - MM ' ' 6 q $$++-CUEYEYE[,\];'';6$u+:N:NRb:b272S2S2U/$MM((//1NP`0abMM((//1G0ST *J ))-)>)>)@)G)G)M)M)\)\(]]^` 77R?O;PP7+g '^" q op qs"C M00 NNNc VVP'gVP!WV3/VBR#R#rVrrs&&&&&,r%rSwanLabCallback.on_train_beginT rr'c VPe@VP'd,VP'd\P R4R#R#R#R#Nrrrrr4r5r&s&&&&&&,r%rSwanLabCallback.on_train_endX A ?? &4+<+<+<A\A\A\ NNF B]+< &r'c .ROpVP'gVPWV4VP'dVP4F9wrW9gK VPP RV 2V /VP R7K; VP4UU u/uFwrW9gK WbK p pp \V 4p VPP /V CRVP /CVP R7R#R#uup pi)r,z single_value/rrNr-)rrrr[rrrr9r1s &&&&&&, r%rSwanLabCallback.on_log_ s    JJtE *  & & & ,MM%%qc':A&>UEVEV%W%15 ^ @]tqt O^*?;O MM  YY2EuGXGXY`e`q`q  r '_s  C9*C9c VPe@VP'd,VP'd\P R4R#R#R#R#rrrs&&&&,r%rASwanLabCallback.on_saver rr'c VP'gVP!W3/VBVP'd)\V4pVPP V4R#R#rV)rrrr9rrrDs&&&&&,r%rESwanLabCallback.on_predicty sL   JJt -f -  & & &"7+G MM  g & 'r')rrrrVrGrHrs@r%rrs6?qQf5s&''r'rcpa]tRtRtoRtRtRtRtRtRt Rt R t R t RR lt R tRRltRtRtRtVtR #)KubeflowCallbacki ag A [`TrainerCallback`] that reports training progress to [Kubeflow Trainer](https://github.com/kubeflow/trainer). This callback is automatically registered when training inside a Kubeflow TrainJob with the `TrainJobRuntimeStatus` feature gate enabled. The Kubeflow controller injects the required environment variables into the training pod. **Environment Variables (injected by controller):** - `KUBEFLOW_TRAINER_SERVER_URL`: HTTPS endpoint for status updates - `KUBEFLOW_TRAINER_SERVER_CA_CERT`: Path to CA certificate for TLS verification - `KUBEFLOW_TRAINER_SERVER_TOKEN`: Path to service account token for authentication **Reported Information:** - Progress percentage (0-100%) - Estimated time remaining (seconds) - Training metrics (loss, learning_rate, etc.) **Features:** - Automatic throttling (max 1 update per 5 seconds) to avoid overwhelming the controller - Token caching (5 minutes) to minimize file I/O - Only rank 0 reports progress in distributed training - Silent failures - network issues won't interrupt training Can be disabled by setting environment variable `DISABLE_KUBEFLOW_INTEGRATION=TRUE`. g@gr@riKUBEFLOW_TRAINER_SERVER_CA_CERTKUBEFLOW_TRAINER_SERVER_TOKENc\4'g \R4hRVn/VnRVnRVnRVnRVnRVnRVn \PR4R#)zKubeflowCallback requires KUBEFLOW_TRAINER_SERVER_URL environment variable to be set. This is automatically set when running inside a Kubeflow TrainJob with TrainJobRuntimeStatus enabled.FNgz[Kubeflow] Callback initialized) rjrrr_metrics _start_time_last_update_time _cached_token_token_read_time _ssl_context_ssl_context_initializedr4r rs&r%r{KubeflowCallback.__init__ sl$&&x  " !$! # (-% 67r'c r^RIpVP'd VP#\PP VP 4pV'dVPVR7VnRVnVP# \d-p\PRT RT 24RTnRp?LERp?ii;i)z,Get cached SSL context for TLS verification.N)cafilez5[Kubeflow] Failed to create SSL context with CA file z: T) sslrrrNenvironr _ENV_CA_CERTcreate_default_contextrr4r5)rzrca_filers& r%_get_ssl_context!KubeflowCallback._get_ssl_context s  ( ( ($$ $**..!2!23  )$'$>$>g$>$N!)-%    )!VW^V__abcadef$(!! )sA?? B6 "B11B6c ^RIpVP4pVP'd.W P, VP8d VP#\ P PVP4pV'd&\ PPV4'g\PRV 24R#\V4;_uu_4pVP4P4VnW nVPuuRRR4# +'giR#;i \ d$p\PRT 24Rp?R#Rp?ii;i)z!Get cached service account token.Nz![Kubeflow] Token file not found: z&[Kubeflow] Failed to read token file: )time monotonicrr_TOKEN_CACHE_DURATIONrNrr_ENV_TOKEN_PATHrexistsr4r rreadrQr)rzrrl token_pathrrs& r% _get_tokenKubeflowCallback._get_token snn    3)>)>#>$B\B\"\%% %ZZ^^D$8$89  !;!; LLVP'4UUu.uFwrR\)V4R\)V4/NK uppV R &VP*!R V /4P-R 4pR R V 2RR/pVP.P1V VVRR7pVP.P3V^VP54R7;_uu_4pVP6^8HuuRRR4#uuppi +'giR#;i \8d$p\:P=RT 24Rp?R#Rp?ii;i)z4Send progress update to Kubeflow Trainer controller.N)rhtimezoneFlastUpdatedTimeprogressPercentageestimatedRemainingSecondsrrr trainerStatuszutf-8 AuthorizationzBearer z Content-Typezapplication/jsonPOST)dataheadersmethod)rcontextz$[Kubeflow] Failed to update status: )rrurllib.requestrhrrNrr_ENV_SERVER_URLrr_MIN_UPDATE_INTERVALrrlutc isoformatrminr|r[r}dumpsencoderequestRequesturlopenrstatusrr4r )rzprogress_percentestimated_time_remainingrrrrurllibrhrrrltokentrainer_statusrbrcrrreqresprs&&&&& r%_update_statusKubeflowCallback._update_status s/! **..!5!56C.."Cc$:$::d>W>WW%( "OO%E/hll1K1U1U1WXN+7:1c#GW>X7Y34'3>A!SIaEb>c:;[b[h[h[j,k[jSWSTfc!fgs1v-N[j,ky)::?@GGPD&'%(9>K]^G..((4QW(XC''Q@U@U@W'XX\`{{c)YX -l YXX  LL?sC D sg0H H!!HH$A+HH##G&BH G, H&H, G= 7H=H H. H))H.c (VP'gR#^RIpVP4Vn/VnRVn\ P RVP 24TP^VP'dRVP/MRRR7R#)NTz'[Kubeflow] Training started, max_steps= total_steps)rrr) rrrrrr4r  max_stepsr)rzrr6rrrs&&&&, r%rKubeflowCallback.on_train_begin sy*** 99;   >u>OPQ 8=]EOO4T  r'c VP'dVP'dVfR#VP4F1wrg\V\\ 34'gK#WpP V&K3 R#rV)rrr[rmr|rr)rzrr6rrrr}rs&&&&&, r%rKubeflowCallback.on_log sO   (C(C(Ct| **,JC%#u..%* c"'r'c VP'dVP'gR#VP'dVP^8:dR#^RIp\ VP VP, ^d,4p\ V^c4pRpVP'dsVP ^8dbVP4VP, pWP , p VPVP , p \ W,4p/VPCRVP RVP/Cp VPe\VP^4V R&VPVVV R7R#)N current_stepr current_epoch)rrr) rrrrr|rrrrr<r;r) rzrr6rrrprogress eta_secondselapsedavg_time_per_stepremaining_stepsrs &&&&, r% on_step_endKubeflowCallback.on_step_end! s.   (C(C(C %//Q"6 ))EOO;sBCx$     1 1A 5iikD$4$44G '*;*; ; #oo0A0AAO/ABK mm E-- 5??  ;; "',U[[!'I" I 5 5-7 BYB^B^B`8a7bbqr ! EN NI[ #K 0 0I NN NsB)~rrrimportlib.metadatar!importlib.utilrr"rNrTrWrrrrf dataclassesrenumrpathlibrtypingrrrnumpyrpackaging.versionrrOrrr r r rrr rrrrrr0rr4rr7rr6r2rr3rr get_configr8PackageNotFoundErrorrArirrKeyErrorr"r#rW_neptune_versionrrtrainer_callbackrrrrrrrJrrrr&r,r0r9r=r@rCrErHrQrTrXr[r^r`rcrfrjrtrr'rOrUr9rprrr}rrDrJrrrr8rrUrYrrrPrgrrrrr+r'r%rs%    ..99\i' 011%   H %!''// ;N'//55nEIZIZI`I`asItt!!/2>#$NNY't3mmy~~7O7OP`7aim7m \!$--55i@ &'7&8 DE@LL(@;;4w:7< D: a><H ;;@ D$>BM`HkV>  KS"/S"l!#C#,m%Om%`@/o@/FOD>o>.r$_r$j.In.Ib ) WZoWZt  DxOoxOv<IO<I~fofRk'ok'\L L `  n &  ]$   ]   $OmP //j)Ucemn!N#  !    2 2! !(1199:JK  KK12B1C;O P!!66 ! L ! !sC>A7L6L 0M)M?MN80NN4/N83N44N8