Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / trainer / base.py: 63%
556 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2025-2026 Bytedance Ltd. and/or its affiliates
2# Copyright 2026 Huawei Technologies Co., Ltd
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
16"""
17Base Trainer class for distributed training.
19This module provides the BaseTrainer class which serves as the foundation
20for all trainer implementations. Subclasses can override specific methods
21to customize training behavior.
23Features:
24 - Callback system for extensible training hooks
25 - Distributed training support
26 - Gradient accumulation
27 - Checkpointing
28"""
30from __future__ import annotations
32import json
33import logging
34import os
35from abc import ABC
36from collections import defaultdict
37from collections.abc import Callable
38from contextlib import nullcontext
39from functools import partial
40from typing import TYPE_CHECKING, Any, Dict, List, Optional
42import torch
43import torch.distributed as dist
44from torch.distributed.checkpoint.stateful import Stateful
45from torch.optim.lr_scheduler import LRScheduler
46from torch.utils.data import Dataset
47from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase
48from transformers.modeling_outputs import ModelOutput
50from hyper_parallel import DTensor, HSDPModule, SkipDTensorDispatch
51from hyper_parallel.compile.compiler import GraphCompiler
52from hyper_parallel.core.tensor_parallel import loss_parallel
53from hyper_parallel.core.utils import clip_grad_norm_
54from hyper_parallel.trainer.config import (
55 OptimizerSwapConfig,
56 TrainerConfig,
57 normalize_distributed_setup_overrides,
58 save_configs,
59)
60from hyper_parallel.data.dataset_logging import enable_dataset_logging
61from hyper_parallel.data.batching import build_dataloader
62from hyper_parallel.data.parallel import DataLoaderParallelContext, create_dataloader_parallel_context
63from hyper_parallel.trainer.runtime.distributed import (
64 all_reduce,
65 get_global_rank_safe,
66 get_local_rank_safe,
67 get_world_size_safe,
68)
69from hyper_parallel.trainer.runtime.distributed import (
70 create_distributed_setup_from_config,
71 destroy_process_group,
72 initialize_distributed,
73)
74from hyper_parallel.trainer.runtime.logging import setup_logging
75from hyper_parallel.trainer.runtime.loss_aggregation import count_loss_token
76from hyper_parallel.trainer.runtime.metrics import mean_global_loss
77from hyper_parallel.components.functional.aux_loss import aux_loss_scale_context, bind_aux_loss_scale
78from hyper_parallel.components.modules.moe_utils import MoEMonitorCallback
79from hyper_parallel.models._transformers.loss_parallel import causal_lm_loss_parallel
80from hyper_parallel.models._transformers.model_builder import _validate_optimize_dtype
81from hyper_parallel.models.external_state import get_model_external_state
82from hyper_parallel.components.losses.model_output import ModelOutputLoss
83from hyper_parallel.core.optimizer import (
84 ChainedOptimizer,
85 SwapOptimizerConfig,
86 swap_optimizer,
87)
88from hyper_parallel.components.optim.mixed_precision_optimizer import (
89 Float16OptimizerWithFloat16Params,
90 MixedPrecisionOptimizer,
91)
92from hyper_parallel.trainer.runtime import fsdp as fsdp_runtime
93from hyper_parallel.trainer.runtime import model_integration as model_integration_runtime
94from hyper_parallel.trainer.runtime.data_iterator import HyperIter
95from hyper_parallel.trainer.runtime.logging import enable_third_party_logging
96from hyper_parallel.trainer.runtime.memory import empty_cache, print_device_mem_info
97from hyper_parallel.trainer.runtime.memory_profiler import memory_profiler
98from hyper_parallel.trainer.runtime.random import enable_high_precision_for_bf16, set_seed
99from hyper_parallel.trainer.runtime.device import ( # pylint: disable=syntax-error
100 get_device_type,
101 get_torch_device,
102 synchronize,
103)
105from hyper_parallel.trainer.callbacks import (
106 EnvironMeterCallback,
107 EvaluateCallback,
108 GarbageCollectionCallback,
109 LoggingCallback,
110 ProfilerCallback,
111 StressDetectCallback,
112 TqdmCallback,
113 CheckpointerCallback,
114 TrainerState,
115)
117logger = logging.getLogger(__name__)
119if TYPE_CHECKING:
120 from hyper_parallel.data.text.chat_template import ChatTemplate
121 from hyper_parallel.distributed_data import BalancingAlgorithm, CostModel
122 from hyper_parallel.distributed_data.schema import SampleMetadata
125def _to_swap_optimizer_config(config: OptimizerSwapConfig) -> SwapOptimizerConfig:
126 """Translate the trainer-facing swap section into the core swap config.
128 Args:
129 config: Resolved ``optimizer.swap`` configuration section.
131 Returns:
132 The backend-neutral swap configuration consumed by ``swap_optimizer``.
133 """
134 return SwapOptimizerConfig(
135 swap_times=config.swap_times,
136 state_keys=config.state_keys,
137 min_numel=config.min_numel,
138 # ``None`` means "leave it to the backend default", not "off": the core
139 # config declares ``packed_swap: bool = True``, so pass that default
140 # through rather than letting ``None`` read as a hard False.
141 packed_swap=True if config.packed_swap is None else config.packed_swap,
142 )
145def _attach_optimizer_swap(optimizer: Any, config: OptimizerSwapConfig) -> Any:
146 """Attach optimizer-state swap to every Adam/AdamW leaf of ``optimizer``.
148 Every YAML optimizer target returns a ``ChainedOptimizer`` of leaf
149 optimizers -- also for a single AdamW -- and the fp32 main-parameter wrapper
150 keeps the same leaf containers. The swap runtime wraps one concrete
151 Adam/AdamW and delegates everything else, so it is attached to the leaves
152 and the outer object keeps dispatching.
154 Args:
155 optimizer: Optimizer built by the configured target, optionally already
156 wrapped for fp32 main parameters.
157 config: Resolved ``optimizer.swap`` configuration section.
159 Returns:
160 The optimizer carrying swap-wrapped leaves. A leaf the swap runtime does
161 not support fails here, before the first training step.
162 """
163 swap_config = _to_swap_optimizer_config(config)
164 chained = optimizer.optimizer if isinstance(optimizer, MixedPrecisionOptimizer) else optimizer
165 if not isinstance(chained, ChainedOptimizer):
166 return swap_optimizer(optimizer, swap_config)
168 names = list(chained.optimizers_dict)
169 leaves = [swap_optimizer(chained.optimizers_dict[name], swap_config) for name in names]
170 # In place: the fp32 main-parameter wrapper aliases both containers.
171 chained.chained_optimizers[:] = leaves
172 for name, leaf in zip(names, leaves):
173 chained.optimizers_dict[name] = leaf
174 return optimizer
177class BaseTrainer(Stateful, ABC):
178 """
179 Base trainer class for distributed model training.
181 This class provides the core training infrastructure including:
182 - Distributed initialization and parallelism setup
183 - Model, optimizer, and scheduler initialization
184 - Training step execution with gradient accumulation
185 - Checkpointing and fault tolerance
186 - Metrics logging
188 Subclasses can override the following methods to customize behavior:
189 - `_build_distributed_setup()`: Connect AutoModel DistributedSetup/MeshContext
190 - `_build_model()`: Connect AutoModel from_pretrained/parallelize
191 - `_build_model_assets()`: Build tokenizer/processor assets
192 - `_build_data_transform()`: Build modality-specific sample transforms
193 - `forward_backward_step()`: Customize forward/backward logic
194 - `train_step()`: Customize training step execution
195 - `train()`: Train the model
197 Callback Hooks:
198 The trainer calls callback methods at various stages:
199 - evaluate_callback: evaluation callback
200 - trace_callback: tracing callback (meter, wandb, tqdm, profile)
201 - checkpoint_callback: checkpointing callback
202 """
204 # Core configs
205 config: TrainerConfig
206 device: torch.device
207 # Attached at runtime by the model-integration pipeline (the Omni dataset builder sets
208 # ``data_transform``, the integration adapter attaches ``get_batch`` and
209 # ``num_micro_batches``): bare annotations, which assign nothing, so runtime behaviour is
210 # unchanged -- they exist so the type checker sees the contract this class reads.
211 data_transform: Any
212 get_batch: Any
213 num_micro_batches: int
215 # AutoModel distributed setup.
216 # These objects have one owner: ``../build_options.py`` and
217 # ``../components/distributed/mesh.py``. Trainer code consumes them but must
218 # not create a second DeviceMesh or a VeOmni ParallelState.
219 dist_info: Any
220 distributed_setup: Any
221 mesh_context: Any
222 device_mesh: Any
223 moe_mesh: Any
225 # Data
226 train_dataset: Dataset
227 data_transform: Any
228 collate_fn: Any
229 get_batch: Any
230 dataloader_context: DataLoaderParallelContext | None = None
231 train_dataloader: Any = None
232 num_micro_batches: int
234 # Model
235 model: PreTrainedModel = None
236 model_parts: Optional[List[torch.nn.Module]] = None
237 hsdp_model_parts: List[torch.nn.Module] = []
238 fsdp_runtime_diagnostics: Optional[fsdp_runtime.FSDPRuntimeDiagnostics] = None
239 model_integration: Optional[model_integration_runtime.ModelIntegrationSession] = None
240 model_config: PretrainedConfig = PretrainedConfig()
241 tokenizer: PreTrainedTokenizerBase = None
242 processor: Any = None
243 chat_template: Optional[ChatTemplate] = None
244 model_assets: List[Any] = []
246 # Training components
247 optimizer = None
248 lr_scheduler: LRScheduler = None
249 loss_fn: Optional[torch.nn.Module] = None
251 # Training context
252 model_fwd_context: Any
253 model_bwd_context: Any
255 # # Runtime metrics, controlled by trace_callback
256 # environ_meter: helper.EnvironMeter # see in trace_callback.EnvironMeterCallback
257 # step_env_metrics: Dict[str, Any] # mfu, flops, tokens, etc
258 # step_train_metrics: Dict[str, Any] # loss, grad_norm, lr, etc
260 # # Checkpointer
261 # checkpointer: CheckpointerBase # see in checkpoint_callback.CheckpointerCallback
263 # Callback system
264 state: TrainerState
266 # Default seed when training.seed is omitted.
267 default_seed: int = 42
269 def __init__(self, config: TrainerConfig) -> None:
270 """
271 Initialize the trainer.
273 Args:
274 args: Global Arguments
275 Should have attributes: model, data, train
276 model: ModelArguments
277 data: DataArguments
278 train: TrainingArguments
279 """
281 self.config: TrainerConfig = config
283 # General setup owns distributed initialization as its first stage.
284 # ``_build_distributed_setup`` remains a reserved backend hook; the
285 # trainer only defines its contract here.
286 self._setup()
287 memory_profiler.reset(
288 self.config.memory,
289 global_rank=self.global_rank,
290 tp_rank=self.mesh.tp_rank,
291 dp_rank=self.mesh.dp_rank,
292 )
294 # Reserved atomic model-build interface.
295 #
296 # Owner:
297 # - ``../_transformers/auto_model.py::HyperAutoModel*.from_pretrained``
298 # - ``../_transformers/model_builder.py`` for materialization,
299 # checkpoint loading, PEFT/quantization, and parallelize.
300 #
301 # Contract:
302 # - consume the ``DistributedSetup`` created above;
303 # - set ``model`` and the resolved checkpoint ``model_config``;
304 # - return a materialized, weight-loaded, already-parallelized model;
305 # - never call a second trainer-side ``build_parallelize_model``.
306 self._build_model()
307 self._build_loss()
309 # Build trainer-owned data components after the model finalizes parameters and sharding.
310 self._build_model_assets()
311 self._build_data_transform()
312 self._build_dataset()
313 self._build_collate_fn()
314 self._build_dataloader()
315 self._compute_train_iters()
317 self._build_optimizer()
318 self.model_integration.attach_optimizer(self.optimizer)
319 self._build_lr_scheduler()
320 self._build_training_context()
321 self._init_callbacks()
323 def _setup(self):
324 """Initialize logging, distributed state, and the local device."""
325 # log args
326 setup_logging()
328 # Initialize distributed process group
329 backend = getattr(self.config.training, "backend", "nccl")
330 if backend == "hccl" and self.config.training.enable_full_determinism:
331 # HCCL reads this when creating communicators, before set_seed runs below.
332 os.environ["HCCL_DETERMINISTIC"] = "true"
333 initialize_distributed(backend=backend)
335 self.local_rank = get_local_rank_safe()
336 self.global_rank = get_global_rank_safe()
337 self.world_size = get_world_size_safe()
338 if self.config.debug.check_dataset:
339 enable_dataset_logging(self.config.debug.check_dataset)
341 if self.config.training.seed is None:
342 self.config.training.seed = self.default_seed
344 if self.global_rank == 0:
345 logger.info(
346 "Trainer config:\n%s",
347 json.dumps(self.config.to_dict(), indent=2),
348 )
350 # init distributed environment
351 device_str = f"{get_device_type()}:{self.local_rank}"
352 get_torch_device().set_device(device_str)
353 self.device = torch.device(device_str)
355 logger.info("Process rank: %s, world size: %s", self.global_rank, self.world_size)
357 # Register distributed_setup; the Trainer-side desugar fills its
358 # normalized plan_overrides/module_replacements (05 §15.2.6) so the
359 # AutoModels build pipeline never sees raw YAML PlanOverride DTOs.
360 self.distributed_setup = create_distributed_setup_from_config(self.config)
361 normalize_distributed_setup_overrides(self.distributed_setup, self.config)
362 self.mesh = self.distributed_setup.mesh_context
363 self.device_mesh = self.mesh.device_mesh
364 self.dp_cp_mesh = self.mesh.dp_cp_mesh
366 # Set random seed
367 set_seed(self.config.training.seed, self.config.training.enable_full_determinism)
369 # Enable high precision for bf16
370 enable_high_precision_for_bf16()
372 # Enable third party logging
373 if self.local_rank == 0:
374 enable_third_party_logging()
376 # Save arguments
377 if self.global_rank == 0:
378 save_configs(self.config, self.config.checkpoint.checkpoint_dir)
380 def _build_model(self) -> None:
381 """Build the model and derive Trainer-owned runtime state."""
382 self.peft_config = self.config.peft
383 self.model = self.config.model.build(
384 distributed_setup=self.distributed_setup,
385 peft_config=self.peft_config,
386 activation_checkpoint=self.config.activation_checkpoint.mode,
387 activation_checkpoint_selection=self.config.activation_checkpoint.selection,
388 swap_inputs=getattr(self.config.activation_checkpoint, "swap_inputs", False),
389 allow_cache_entry_mutation=getattr(
390 self.config.activation_checkpoint, "allow_cache_entry_mutation", False),
391 activation_swap=self.config.activation_swap,
392 compile_config=self.config.compile,
393 # The final dtype is applied inside the atomic build (05 stage-5
394 # item 5); the Trainer no longer patches it afterwards.
395 model_init_dtype=self.config.model_init_dtype,
396 )
397 _validate_optimize_dtype(
398 self.model,
399 fp32_main_params=self.config.optimizer.fp32_main_params,
400 )
401 self.model_config = self.model.config
402 if self.global_rank == 0:
403 logger.info(
404 "Model config:\n%s",
405 json.dumps(self.model.config.to_dict(), indent=2),
406 )
407 if self.config.accelerator.loss_parallel:
408 self.model.loss_function = causal_lm_loss_parallel
409 model_parts = getattr(self.model, "parts", None)
410 self.model_parts = list(model_parts) if model_parts is not None else [self.model]
411 self.hsdp_model_parts = [
412 model_part
413 for model_part in self.model_parts
414 if isinstance(model_part, HSDPModule)
415 ]
416 self.model_integration = model_integration_runtime.build_model_integration_session(
417 self.model,
418 self.config,
419 self.global_rank,
420 self.mesh,
421 )
422 self.fsdp_runtime_diagnostics = (
423 fsdp_runtime.build_fsdp_runtime_diagnostics(self.hsdp_model_parts)
424 if getattr(self.config.debug, "check_fsdp_runtime", False)
425 else None
426 )
428 def _build_loss(self) -> None:
429 """Build the configured loss module or use the model-output default."""
430 loss_fn = (
431 self.config.loss_fn.build()
432 if self.config.loss_fn is not None
433 else ModelOutputLoss()
434 )
435 if not isinstance(loss_fn, torch.nn.Module):
436 raise ValueError("config.loss_fn must build a torch.nn.Module")
437 self.loss_fn = loss_fn
438 # Model-integrated objectives may need to intercept the model before its
439 # ordinary forward materializes an otherwise avoidable output tensor.
440 loss_model_binding_hook = getattr(self.loss_fn, "bind_model", None)
441 if callable(loss_model_binding_hook):
442 loss_model_binding_hook(self.model, self.distributed_setup)
444 def _build_model_assets(self) -> None:
445 """Require a concrete Trainer to build modality-specific model assets."""
446 raise NotImplementedError("Concrete Trainer must implement _build_model_assets")
448 def _build_data_transform(self) -> None:
449 """Require a concrete Trainer to build its sample transform."""
450 raise NotImplementedError("Concrete Trainer must implement _build_data_transform")
452 def _build_dataset(self) -> None:
453 """Build and assign one Dataset or a train-validation-test tuple."""
454 data_config = getattr(self.config.dataset, "data_config", {})
455 self.dataloader_context = create_dataloader_parallel_context(
456 self.mesh,
457 data_index_cache=bool(data_config.get("data_index_cache", False)),
458 shared_storage=not bool(data_config.get("no_shared_storage", False)),
459 tp_source_rank=None if self.config.dataloader.load_balance is not None else 0,
460 )
461 dataset_result = self.config.dataset.build(
462 transform=self.data_transform,
463 tokenizer=self.tokenizer,
464 mesh_context=self.mesh,
465 training_config=self.config.training,
466 sampler_type=self.config.dataloader.sampler_type,
467 dataloader_context=self.dataloader_context,
468 )
470 if isinstance(dataset_result, tuple) and len(dataset_result) == 3:
471 self.train_dataset, self.valid_dataset, self.test_dataset = dataset_result
472 return
474 self.train_dataset = dataset_result
475 self.valid_dataset = self.test_dataset = None
477 def _build_collate_fn(self) -> None:
478 """Require a concrete Trainer to build its micro-batch collator."""
479 raise NotImplementedError("Concrete Trainer must implement _build_collate_fn")
481 def _build_dataloader(
482 self,
483 *,
484 metadata_fn: Callable[[Any], SampleMetadata] | None = None,
485 cost_model: CostModel | None = None,
486 balancing_algorithm: BalancingAlgorithm | None = None,
487 ) -> None:
488 """Build and assign train, validation, and test dataloaders."""
489 split_names = ("train", "valid", "test")
490 datasets = tuple(getattr(self, f"{split_name}_dataset", None) for split_name in split_names)
491 dataloaders, batch_samplers = build_dataloader(
492 self.config.dataloader,
493 datasets=datasets,
494 collate_fn=self.collate_fn,
495 mesh_context=self.mesh,
496 training_config=self.config.training,
497 data_config=getattr(self.config.dataset, "data_config", {}),
498 max_seq_len=getattr(self.data_transform, "max_seq_len", None),
499 default_seed=self.default_seed,
500 metadata_fn=metadata_fn,
501 model_config=self.model_config,
502 cost_model=cost_model,
503 balancing_algorithm=balancing_algorithm,
504 )
505 for split_name, dataloader, batch_sampler in zip(split_names, dataloaders, batch_samplers):
506 setattr(self, f"{split_name}_dataloader", dataloader)
507 setattr(self, f"{split_name}_batch_sampler", batch_sampler)
509 def attach_model_integration_data_pipeline(self) -> None:
510 """Attach the final data pipeline through one shared Trainer path."""
511 self.model_integration.attach_data_pipeline(
512 runtime_adapter=getattr(self.get_batch, "runtime_input_adapter", None),
513 get_batch=self.get_batch,
514 model_assets=tuple(self.model_assets),
515 )
517 def _compute_train_iters(self) -> None:
518 """Resolve the run limit and the optimizer steps available per Dataset epoch."""
519 training_config = self.config.training
520 if training_config.train_iters is not None and training_config.train_samples is not None:
521 raise ValueError("configure only one of training.train_iters and training.train_samples")
523 if training_config.train_iters is None and training_config.train_samples is None:
524 raise ValueError("configure one of training.train_iters or training.train_samples")
526 train_iters = training_config.train_iters
527 if training_config.train_samples is not None:
528 train_iters = training_config.train_samples // training_config.global_batch_size
530 if train_iters <= 0:
531 raise ValueError("training configuration must produce at least one train iteration")
533 try:
534 micro_batches_per_epoch = len(self.train_dataloader)
535 except TypeError:
536 micro_batches_per_epoch = 0
538 if micro_batches_per_epoch:
539 train_steps = micro_batches_per_epoch // self.num_micro_batches
540 if train_steps <= 0:
541 raise ValueError("train DataLoader must provide one complete optimizer step per epoch")
542 else:
543 # Streaming and dynamic DataLoaders define an epoch only when their
544 # source iterator is exhausted.
545 train_steps = train_iters
547 self.train_iters = train_iters
548 self.train_steps = train_steps
549 self.train_epochs = (self.train_iters + self.train_steps - 1) // self.train_steps
551 def _build_optimizer(self) -> None:
552 """Build the configured optimizer with the runtime model context."""
553 config: TrainerConfig = self.config
554 external = get_model_external_state(self.model)
555 if external is not None:
556 def wrap_dense(dense_raw: Any) -> Any:
557 """Wrap dense parameters in FP32 main and optional state swap.
559 Args:
560 dense_raw: Constructed dense optimizer.
561 """
562 dense = (
563 Float16OptimizerWithFloat16Params(dense_raw, self.model)
564 if config.optimizer.fp32_main_params else dense_raw
565 )
566 if config.optimizer.swap.enabled:
567 dense = _attach_optimizer_swap(dense, config.optimizer.swap)
568 return dense
570 self.optimizer = external.build_optimizer(config.optimizer, wrap_dense=wrap_dense)
571 return
572 optimizer = config.optimizer.target.build(model=self.model).get_optimizer()
573 self.optimizer = (
574 Float16OptimizerWithFloat16Params(optimizer, self.model)
575 if config.optimizer.fp32_main_params
576 else optimizer
577 )
578 # Swap is attached after the fp32 main-parameter wrap so leaf optimizers
579 # already reference their final parameters, and the lr scheduler built
580 # right after reads the wrapped ``self.optimizer.param_groups``.
581 if config.optimizer.swap.enabled:
582 self.optimizer = _attach_optimizer_swap(self.optimizer, config.optimizer.swap)
584 def _build_lr_scheduler(self):
585 config: TrainerConfig = self.config
586 scheduler_iters = config.training.lr_scheduler_iters or self.train_iters
587 if scheduler_iters <= 0:
588 raise ValueError("training.lr_scheduler_iters must be positive when configured")
589 lr_scheduler = config.lr_scheduler.build(
590 optimizer=self.optimizer,
591 train_iters=scheduler_iters,
592 )
593 self.lr_scheduler = lr_scheduler.get_lr_scheduler()
595 def _build_training_context(self):
596 """Build training context for distributed training."""
597 if self.config.accelerator.loss_parallel:
598 tp_mesh = self.device_mesh["tp"]
599 self.model_fwd_context = partial(loss_parallel, mesh=tp_mesh)
600 else:
601 self.model_fwd_context = nullcontext()
602 self.model_bwd_context = nullcontext()
603 self.graph_compiler: Optional[GraphCompiler] = None
605 def _build_graph_compiler(self) -> None:
606 """Build the optional joint-graph compiler for this trainer runtime."""
607 if not self.config.compile.selects_graph_compiler():
608 return
609 self.graph_compiler = GraphCompiler(
610 model=self.model,
611 train_fn=self._graph_train_fn,
612 trainer_config=self.config,
613 device=self.device,
614 mesh_context=self.mesh,
615 source_shard_info=getattr(self.model, "_source_shard_info", None),
616 )
618 def _graph_train_fn(
619 self,
620 model: torch.nn.Module,
621 model_inputs: Dict[str, Any],
622 labels: Optional[torch.Tensor],
623 current_token_counts: Dict[str, torch.Tensor],
624 ) -> tuple[torch.Tensor, Dict[str, torch.Tensor]]:
625 """Capture forward/loss with explicit runtime token counts."""
626 model_fwd_context = (
627 self.model_fwd_context()
628 if callable(self.model_fwd_context)
629 else self.model_fwd_context
630 )
631 with model_fwd_context:
632 outputs = model(**model_inputs, use_cache=False)
633 # Counts must be graph inputs: packed lengths and ignored-label counts
634 # can vary even when tensor shapes happen to match the first batch.
635 loss, loss_dict = self.postforward(
636 outputs, labels,
637 current_token_counts=current_token_counts,
638 )
639 return loss, loss_dict
641 def _init_callbacks(self):
642 """Initialize callbacks."""
643 dp_cp_mesh = self.mesh.dp_cp_mesh
644 self.moe_monitor = MoEMonitorCallback(
645 self.model,
646 dp_group=dp_cp_mesh.get_group() if dp_cp_mesh is not None else None,
647 tp_group=self.device_mesh["tp"].get_group() if self.mesh.sequence_parallel else None,
648 )
649 self.state = TrainerState()
650 # Raw AUX is not normalized with the backward loss; average its observations.
651 self.environ_meter_callback = EnvironMeterCallback(self, microbatch_mean_metrics=("router_aux_loss",))
652 self.tqdm_callback = TqdmCallback(self)
653 self.logging_callback = LoggingCallback(self)
654 self.evaluate_callback = EvaluateCallback(self)
655 self.garbage_collection_callback = GarbageCollectionCallback(self)
656 self.profiler_callback = ProfilerCallback(self)
657 self._callbacks = [
658 self.environ_meter_callback,
659 self.logging_callback,
660 self.tqdm_callback,
661 self.evaluate_callback,
662 self.garbage_collection_callback,
663 self.profiler_callback,
664 ]
665 if self.config.stress_detect.op_stress_detect.detection_interval > 0:
666 self.stress_detect_callback = StressDetectCallback(self)
667 self._callbacks.append(self.stress_detect_callback)
668 # Registered to save, to restore, or both --- the two are independent, so
669 # a run that only loads an existing checkpoint still needs the callback.
670 if (
671 self.config.checkpoint.save_ckpt
672 or self.config.checkpoint.save_hf_weights
673 or self.config.checkpoint.restore_from is not None
674 ):
675 self._callbacks.append(CheckpointerCallback(self))
676 else:
677 logger.info(
678 "Checkpointing is inactive (save_ckpt=false, save_hf_weights=false, "
679 "restore_from=None); "
680 "no checkpoint callback registered."
681 )
683 def on_train_begin(self) -> None:
684 """Run all registered callbacks at the start of training."""
685 for callback in self._callbacks:
686 callback.on_train_begin(self.state)
688 def on_train_end(self) -> None:
689 """Run all registered callbacks at the end of training."""
690 for callback in self._callbacks:
691 callback.on_train_end(self.state)
693 def on_epoch_begin(self) -> None:
694 """Run all registered callbacks at the start of an epoch."""
695 for callback in self._callbacks:
696 callback.on_epoch_begin(self.state)
698 def on_epoch_end(self) -> None:
699 """Run all registered callbacks at the end of an epoch."""
700 for callback in self._callbacks:
701 callback.on_epoch_end(self.state)
703 def on_step_begin(self, **kwargs: Any) -> None:
704 """Run all registered callbacks at the start of a training step."""
705 for callback in self._callbacks:
706 callback.on_step_begin(self.state, **kwargs)
708 def on_micro_step_begin(self, micro_batch: Dict[str, Any], **kwargs: Any) -> None:
709 """Run all registered callbacks before one forward-backward micro step.
711 Args:
712 micro_batch: Prepared inputs for the current micro step.
713 **kwargs: Additional callback context.
714 """
715 for callback in self._callbacks:
716 callback.on_micro_step_begin(self.state, micro_batch, **kwargs)
718 def on_micro_step_end(self, loss_dict: Dict[str, torch.Tensor], **kwargs: Any) -> None:
719 """Run all registered callbacks after one forward-backward micro step.
721 Args:
722 loss_dict: Named losses and metrics returned by the completed micro step.
723 **kwargs: Additional callback context.
724 """
725 for callback in self._callbacks:
726 callback.on_micro_step_end(self.state, loss_dict=loss_dict, **kwargs)
728 def on_step_end(
729 self,
730 loss: Optional[float] = None,
731 loss_dict: Optional[Dict[str, Any]] = None,
732 grad_norm: Optional[float] = None,
733 ) -> None:
734 """Run all registered callbacks at the end of a training step.
736 Args:
737 loss: Aggregated loss for the optimizer step.
738 loss_dict: Named loss values for the optimizer step.
739 grad_norm: Gradient norm measured before the optimizer update.
740 """
741 for callback in self._callbacks:
742 callback.on_step_end(self.state, loss=loss, loss_dict=loss_dict, grad_norm=grad_norm)
744 def preforward(self, micro_batch: List[Dict[str, Any]]) -> Dict[str, Any]:
745 """Preprocess micro batches before forward pass.
747 Tensors are moved to ``self.device`` non-blockingly. Nested dicts
748 (e.g. ``multimodal_metadata`` emitted by ``PackingCollator``) are
749 recursed so inner tensor values land on the device too; Python ints
750 / lists / etc. pass through unchanged.
752 Args:
753 micro_batch: Batch fields to move to the training device.
755 Returns:
756 Batch fields with tensors moved to the training device.
757 """
759 def _to_device(v: Any) -> Any:
760 if isinstance(v, torch.Tensor):
761 return v.to(self.device, non_blocking=True)
762 if isinstance(v, dict):
763 return {k: _to_device(vv) for k, vv in v.items()}
764 return v
766 # chunk_mbs_config = getattr(self.config.training, "chunk_mbs_config", None)
767 # self._chunk_mbs_ranges = build_chunk_mbs_ranges(micro_batch, chunk_mbs_config)
768 micro_batch = {k: _to_device(v) for k, v in micro_batch.items()} # type: ignore
769 # if getattr(self, "LOG_SAMPLE", True):
770 # self.LOG_SAMPLE = False
771 return micro_batch
773 def postforward(
774 self, outputs: ModelOutput, labels: Optional[torch.Tensor],
775 *,
776 current_token_counts: Dict[str, torch.Tensor],
777 ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
778 """Postprocess model outputs after forward pass.
780 Args:
781 outputs: Model outputs consumed by the configured loss.
782 labels: Labels associated with the current micro-batch.
783 current_token_counts: Per-loss token counts for the current micro-batch.
785 Returns:
786 Backward loss and named globally aggregated loss values.
787 """
788 local_loss = self.loss_fn(model_output=outputs, labels=labels)
789 # Names are static graph metadata; diagnostics added below are not losses.
790 self.loss_names = tuple(local_loss) if isinstance(local_loss, dict) else ("foundation_loss",)
791 # Every foundation-family objective receives the same external token weight.
792 # Bind each branch before backward can reach its independently injected losses.
793 foundation_losses = (
794 (value for key, value in local_loss.items() if key.split("_loss", maxsplit=1)[0] == "foundation")
795 if isinstance(local_loss, dict) else (local_loss,)
796 )
797 for foundation_loss in foundation_losses:
798 if foundation_loss is not None:
799 bind_aux_loss_scale(foundation_loss)
800 loss_dict: Dict[str, torch.Tensor] = mean_global_loss(
801 local_loss,
802 current_token_counts,
803 device_mesh=self.mesh,
804 )
805 loss = torch.stack(list(loss_dict.values())).sum()
806 # Model-declared extra metrics join the logged mapping only after
807 # the backward loss is fixed: log-only, never part of the loss.
808 extra_metrics = getattr(outputs, "extra_metrics", None)
809 if extra_metrics is None:
810 model_ref = getattr(self, "model", None)
811 extra_metrics = getattr(model_ref, "pending_extra_metrics", None)
812 if extra_metrics is not None:
813 model_ref.pending_extra_metrics = None
814 if extra_metrics:
815 for name, value in extra_metrics.items():
816 loss_dict[name] = value.detach().float()
817 # Log-only outputs share the eager/graph path without entering backward loss.
818 moe_monitor = getattr(self, "moe_monitor", None)
819 if moe_monitor is not None:
820 aux_loss = moe_monitor.last_mean_aux_loss
821 if aux_loss is not None:
822 loss_dict["router_aux_loss"] = aux_loss
823 return loss, loss_dict
825 def forward_backward_step(
826 self,
827 micro_batch: dict[str, torch.Tensor],
828 loss_inputs: Optional[dict[str, torch.Tensor]] = None,
829 ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
830 """Run forward and backward computation for one micro batch.
832 Args:
833 micro_batch: Inputs forwarded to the model.
834 loss_inputs: Loss-only labels and masks from the get-batch runtime. If
835 omitted, the legacy single-dictionary path is preserved.
837 Returns:
838 Backward loss and the named globally aggregated loss mapping.
839 """
840 for name, count in self.current_token_counts.items():
841 self.step_token_counts[name] = self.step_token_counts.get(name, 0) + count.detach()
842 channel_loss_callback = getattr(self, "channel_loss_callback", None)
843 micro_step_context = (
844 channel_loss_callback.micro_step_context(self.state, micro_batch)
845 if channel_loss_callback is not None
846 else nullcontext()
847 )
848 with micro_step_context, aux_loss_scale_context():
849 micro_batch = self.preforward(micro_batch)
850 # TextTrainer passes loss-only fields separately. Keep the legacy
851 # BaseTrainer/VLM one-dictionary call contract working as well.
852 if loss_inputs is None:
853 loss_inputs = {
854 name: micro_batch[name]
855 for name in ("labels", "shift_labels", "loss_mask")
856 if name in micro_batch
857 }
858 else:
859 loss_inputs = self.preforward(loss_inputs)
860 labels = loss_inputs.get("labels", micro_batch.get("labels"))
862 # A model-integrated loss can translate the public batch fields into
863 # model-family protocol arguments before the model forward starts.
864 prepare_model_inputs = getattr(
865 self.loss_fn,
866 "prepare_model_inputs",
867 None,
868 )
869 if callable(prepare_model_inputs):
870 micro_batch = prepare_model_inputs(micro_batch, loss_inputs)
871 if channel_loss_callback is not None:
872 channel_loss_callback.strip_model_inputs(micro_batch)
873 self.model_integration.record_batch(micro_batch, loss_inputs)
875 if self.graph_compiler is not None:
876 runtime_adapter = getattr(getattr(self, "get_batch", None), "runtime_input_adapter", None)
877 if runtime_adapter is not None:
878 micro_batch = runtime_adapter.prepare_graph_inputs(micro_batch)
879 if self.config.training.empty_cache_before_backward:
880 logger.warning(
881 "training.empty_cache_before_backward is ignored in graph compiler mode "
882 "because forward and backward execute in one joint graph"
883 )
884 # The traced ``loss_dict`` values flow back as extra graph
885 # outputs (emitted after the gradients), so the logging
886 # metrics keep per-key parity with the eager path.
887 loss, loss_dict = self.graph_compiler.forward_backward(
888 model_inputs=micro_batch,
889 labels=labels,
890 current_token_counts=self.current_token_counts,
891 )
892 else:
893 model_fwd_context = (
894 self.model_fwd_context()
895 if callable(self.model_fwd_context)
896 else self.model_fwd_context
897 )
898 with model_fwd_context:
899 outputs: ModelOutput = self.model(**micro_batch, use_cache=False)
901 # with use_parallel_state("base"):
902 loss, loss_dict = self.postforward(
903 outputs, labels,
904 current_token_counts=self.current_token_counts,
905 )
906 prefetch_plan = getattr(self.train_dataloader, "prefetch_plan", None)
907 if callable(prefetch_plan):
908 prefetch_plan() # pylint: disable=not-callable # Guarded dynamic hook.
909 # The loss graph owns everything required for backward. Releasing
910 # the model output here avoids retaining large vocabulary logits
911 # until the whole backward pass finishes.
912 del outputs
913 if self.config.training.empty_cache_before_backward:
914 empty_cache()
916 # Backward pass
917 with self.model_bwd_context:
918 loss.backward()
919 prefetch = getattr(self.train_dataloader, "prefetch", None)
920 if callable(prefetch):
921 prefetch() # pylint: disable=not-callable # Guarded dynamic hook.
923 del micro_batch
924 return loss, loss_dict
926 def model_reshard(self, micro_step: int, num_micro_steps: int) -> None:
927 """Reshard model after backward pass; policy lives in ``runtime/fsdp.py``.
929 Args:
930 micro_step: Zero-based micro-step index.
931 num_micro_steps: Number of micro-steps in the optimizer step.
932 """
933 fsdp_runtime.model_reshard(self.hsdp_model_parts, self.config.fsdp_config, micro_step, num_micro_steps)
935 def _configure_fsdp_gradient_sync(self, micro_step: int, num_micro_steps: int):
936 """Configure FSDP gradient synchronization; policy lives in ``runtime/fsdp.py``."""
937 fsdp_runtime.configure_fsdp_gradient_sync(
938 self.hsdp_model_parts,
939 self.config.fsdp_config,
940 self.mesh.dp_replicate_size,
941 micro_step,
942 num_micro_steps,
943 )
945 def configure_fsdp_gradient_sync(self, micro_step: int, num_micro_steps: int) -> None:
946 """Configure FSDP gradient synchronization for an external training loop.
948 Args:
949 micro_step: Zero-based micro-step index.
950 num_micro_steps: Number of micro-steps in the optimizer step.
951 """
952 self._configure_fsdp_gradient_sync(micro_step, num_micro_steps)
954 def begin_fsdp_runtime_diagnostics(self, micro_step: int) -> None:
955 """Begin observing one FSDP forward/backward micro-step when enabled.
957 Args:
958 micro_step: Microbatch index.
959 """
960 if self.fsdp_runtime_diagnostics is not None:
961 self.fsdp_runtime_diagnostics.begin_micro_step(
962 self.state.global_step,
963 micro_step,
964 )
966 def validate_fsdp_runtime_before_optimizer(self) -> None:
967 """Fail before optimizer updates when FSDP prefetch state is unsafe."""
968 if self.fsdp_runtime_diagnostics is not None:
969 self.model_integration.record_fsdp_trace(
970 self.fsdp_runtime_diagnostics.trace_evidence()
971 )
972 self.fsdp_runtime_diagnostics.validate_before_optimizer_step()
974 def begin_loss_step(self) -> None:
975 """Reset scalar counters before reading the next optimizer step."""
976 self.step_token_counts = {}
978 def _loss_denominator(self) -> float:
979 """Return the optimizer-step valid-token count across DP/CP."""
980 family = self.loss_names[0].split("_loss", maxsplit=1)[0]
981 count = self.step_token_counts[f"{family}_tokens"].detach().clone()
982 mesh = self.mesh.dp_cp_mesh
983 if mesh is not None and mesh.size() > 1:
984 count = all_reduce(count.to(self.device), op="sum", group=mesh.get_group())
985 else:
986 count = count.item()
987 return max(count, 1)
989 @torch.no_grad()
990 def loss_normalize(self, loss: float, loss_dict: Dict[str, float], *, denominator: float | None = None) -> float:
991 """Normalize Trainer-owned gradients and loss metrics before clipping.
993 External providers expose their owned parameters through ``parameters``
994 and normalize those gradients in ``prepare_optimizer_step`` instead.
995 """
996 if denominator is None:
997 denominator = self._loss_denominator()
998 external = get_model_external_state(self.model)
999 excluded = {id(param) for param in external.parameters.values()} if external is not None else set()
1000 with SkipDTensorDispatch():
1001 for parameter in self.model.parameters():
1002 if id(parameter) in excluded:
1003 continue
1004 grad = getattr(parameter, "main_grad", None)
1005 if grad is None:
1006 grad = parameter.grad
1007 if grad is not None:
1008 if isinstance(grad, DTensor):
1009 grad = grad.to_local()
1010 grad.div_(denominator)
1011 for name in self.loss_names:
1012 loss_dict[name] /= denominator
1013 return loss / denominator
1015 def prepare_optimizer_step(self, loss: float, loss_dict: Dict[str, float]) -> tuple[float, Any]:
1016 """Normalize the step objective before clipping and optimizer preparation."""
1017 denominator = self._loss_denominator()
1018 loss = self.loss_normalize(loss, loss_dict, denominator=denominator)
1019 return loss, self.clip_grad_norm(gradient_divisor=denominator)
1021 def clip_grad_norm(self, *, gradient_divisor: float = 1.0) -> Any:
1022 """Run pre-optimizer diagnostics and clipping in their required order."""
1023 self.model_integration.after_backward_before_clip()
1024 self.validate_fsdp_runtime_before_optimizer()
1025 max_grad_norm = self.config.training.max_grad_norm
1026 external = get_model_external_state(self.model)
1027 if external is not None:
1028 preparation = external.prepare_optimizer_step(max_norm=max_grad_norm, gradient_divisor=gradient_divisor)
1029 self.model_integration.after_clip(preparation.post_clip_norm)
1030 return preparation.global_norm
1031 grad_norm: Any = 0.0
1032 post_clip_norm = None
1033 compiler = getattr(self, "graph_compiler", None)
1034 clip_kwargs = {"parameter_layouts": compiler.parameter_layouts} if compiler is not None else {}
1035 if max_grad_norm > 0:
1036 grad_norm = clip_grad_norm_(self.model, max_grad_norm, **clip_kwargs)
1037 if self.model_integration.runtime_enabled:
1038 grad_norm_value = (
1039 float(grad_norm.item())
1040 if isinstance(grad_norm, torch.Tensor)
1041 else float(grad_norm)
1042 )
1043 clip_coefficient = min(
1044 float(max_grad_norm) / (grad_norm_value + 1.0e-6),
1045 1.0,
1046 )
1047 post_clip_norm = grad_norm_value * clip_coefficient
1048 elif self.model_integration.runtime_enabled:
1049 grad_norm = clip_grad_norm_(self.model, float("inf"), **clip_kwargs)
1050 post_clip_norm = grad_norm
1051 self.model_integration.after_clip(post_clip_norm)
1052 return grad_norm
1054 def step_optimizers_and_schedulers(self) -> None:
1055 """Step every optimizer and scheduler, then expose the final state."""
1056 optimizers = self.optimizer if isinstance(self.optimizer, list) else [self.optimizer]
1057 for optimizer in optimizers:
1058 # Keep zeros_like dispatched so lazily created optimizer states retain DTensor metadata.
1059 with SkipDTensorDispatch(no_skip={torch.zeros_like}):
1060 optimizer.step()
1061 optimizer.zero_grad()
1062 moe_monitor = getattr(self, "moe_monitor", None)
1063 if moe_monitor is not None:
1064 moe_monitor.on_step_end()
1065 self.model_integration.after_optimizer()
1067 schedulers = (
1068 self.lr_scheduler
1069 if isinstance(self.lr_scheduler, list)
1070 else ([self.lr_scheduler] if self.lr_scheduler is not None else [])
1071 )
1072 for scheduler in schedulers:
1073 scheduler.step()
1075 def _end_model_integration_step(self, metrics: Dict[str, Any]) -> None:
1076 """Persist structured step metrics and global input identity."""
1077 if not self.model_integration.runtime_enabled:
1078 return
1079 resolved_metrics = dict(metrics)
1080 optimizers = self.optimizer if isinstance(self.optimizer, list) else [self.optimizer]
1081 learning_rates = []
1082 for optimizer in optimizers:
1083 inner = getattr(optimizer, "optimizer", optimizer)
1084 leaves = getattr(inner, "chained_optimizers", (inner,))
1085 for leaf in leaves:
1086 learning_rates.extend(
1087 group.get("lr")
1088 for group in getattr(leaf, "param_groups", ())
1089 if group.get("lr") is not None
1090 )
1091 if learning_rates:
1092 resolved_metrics["lr"] = float(learning_rates[0])
1093 token_counts = getattr(self, "step_token_counts", {})
1094 token_values = []
1095 for value in token_counts.values():
1096 token_values.append(float(value.item() if torch.is_tensor(value) else value))
1097 if token_values:
1098 resolved_metrics["tokens"] = sum(token_values)
1099 resolved_metrics["samples"] = self.config.training.global_batch_size
1100 self.model_integration.end_step(resolved_metrics)
1102 def train_step(
1103 self,
1104 data_iterator: Any,
1105 ) -> Dict[str, float]:
1106 """Execute one optimizer update from the next dataloader batch.
1108 Args:
1109 data_iterator: Iterator providing the next micro-batches.
1111 Returns:
1112 Aggregated loss and gradient norm.
1113 """
1114 memory_profiler.step()
1115 micro_batches: List[Dict[str, Any]] = next(data_iterator)
1116 self.state.global_step += 1
1118 self.on_step_begin(micro_batches=micro_batches)
1119 self.model_integration.begin_step(self.state.global_step)
1121 # Forward and backward for each micro batch
1122 synchronize()
1124 total_loss = 0.0
1125 total_loss_dict = defaultdict(int)
1127 # token num for fixed_ce_loss in postforward
1128 self.begin_loss_step()
1129 num_micro_steps = len(micro_batches)
1130 # forward and backward pass with gradient_accumulationsteps
1131 for micro_step, micro_batch in enumerate(micro_batches):
1132 self.model_reshard(micro_step, num_micro_steps)
1133 self._configure_fsdp_gradient_sync(micro_step, num_micro_steps)
1134 self.begin_fsdp_runtime_diagnostics(micro_step)
1135 loss: torch.Tensor
1136 loss_dict: Dict[str, torch.Tensor]
1137 # token num for fixed_ce_loss in postforward
1138 self.current_token_counts = count_loss_token(micro_batch)
1139 loss, loss_dict = self.forward_backward_step(micro_batch)
1140 self.on_micro_step_end(loss_dict)
1142 total_loss += loss.item()
1143 for k, v in loss_dict.items():
1144 total_loss_dict[k] += v.item()
1146 total_loss, grad_norm = self.prepare_optimizer_step(total_loss, total_loss_dict)
1147 self.step_optimizers_and_schedulers()
1149 grad_norm_value = grad_norm.item() if isinstance(grad_norm, torch.Tensor) else float(grad_norm)
1150 self._end_model_integration_step({"loss": total_loss, "grad_norm": grad_norm_value})
1151 self.on_step_end(loss=total_loss, loss_dict=total_loss_dict, grad_norm=grad_norm_value)
1152 return {
1153 "loss": total_loss,
1154 "grad_norm": grad_norm_value,
1155 }
1157 def abort_profilers(self) -> None:
1158 """Release process-local profiling resources while preserving a training error."""
1159 try:
1160 memory_profiler.abort()
1161 # Cleanup must also preserve the training error if interrupted.
1162 except BaseException: # pylint: disable=broad-exception-caught
1163 logger.exception("Failed to stop memory profiling while aborting training.")
1164 profiler_callback = getattr(self, "profiler_callback", None)
1165 if profiler_callback is not None:
1166 try:
1167 profiler_callback.close()
1168 # Cleanup must also preserve the training error if interrupted.
1169 except BaseException: # pylint: disable=broad-exception-caught
1170 logger.exception("Failed to stop performance profiling while aborting training.")
1172 def destroy_distributed(self) -> None:
1173 """Synchronize all ranks and tear down the distributed process group."""
1174 if not dist.is_available() or not dist.is_initialized():
1175 return
1177 empty_cache()
1178 dist.barrier()
1180 synchronize()
1181 destroy_process_group()
1183 def train(self) -> None:
1184 """Run the configured training loop."""
1185 try:
1186 config: TrainerConfig = self.config
1187 self.on_train_begin()
1188 self.data_iterator = HyperIter(
1189 self.train_dataloader, use_background_prefetcher=config.dataloader.use_background_prefetcher
1190 )
1191 logger.info(
1192 "Rank%s Start training. Global step: %s. Train iters: %s. Start epoch: %s. Train epochs: %s.",
1193 self.local_rank, self.state.global_step, self.train_iters, self.state.epoch, self.train_epochs,
1194 )
1196 try:
1197 start_epoch = self.state.epoch
1198 for epoch in range(start_epoch, self.train_epochs):
1199 if epoch != start_epoch:
1200 self.train_dataloader.set_epoch(epoch)
1201 self.data_iterator = HyperIter(
1202 self.train_dataloader, use_background_prefetcher=config.dataloader.use_background_prefetcher
1203 )
1204 self.state.epoch = epoch
1206 self.on_epoch_begin()
1208 start_step = self.state.global_step - epoch * self.train_steps
1209 train_steps = min(self.train_steps, self.train_iters - epoch * self.train_steps)
1210 for _ in range(start_step, train_steps):
1211 try:
1212 self.train_step(self.data_iterator)
1213 except StopIteration:
1214 logger.info(
1215 "epoch:%s Dataloader finished with drop_last %s",
1216 epoch,
1217 config.dataloader.drop_last,
1218 )
1219 break
1221 self.on_epoch_end()
1222 self.state.epoch = epoch + 1
1224 print_device_mem_info(f"VRAM usage after epoch {epoch + 1}")
1226 if config.dataloader.use_background_prefetcher:
1227 self.data_iterator.stop()
1229 memory_profiler.stop()
1230 self.on_train_end()
1231 finally:
1232 if config.dataloader.use_background_prefetcher:
1233 self.data_iterator.stop()
1235 except BaseException:
1236 try:
1237 memory_profiler.stop()
1238 # Snapshot export must preserve the training error even if interrupted.
1239 except BaseException: # pylint: disable=broad-exception-caught
1240 logger.exception("Failed to save memory snapshot after training error.")
1241 finally:
1242 self.abort_profilers()
1243 raise
1245 synchronize()
1247 self.destroy_distributed()