Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / trainer / base.py: 63%

556 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2025-2026 Bytedance Ltd. and/or its affiliates 

2# Copyright 2026 Huawei Technologies Co., Ltd 

3# 

4# Licensed under the Apache License, Version 2.0 (the "License"); 

5# you may not use this file except in compliance with the License. 

6# You may obtain a copy of the License at 

7# 

8# http://www.apache.org/licenses/LICENSE-2.0 

9# 

10# Unless required by applicable law or agreed to in writing, software 

11# distributed under the License is distributed on an "AS IS" BASIS, 

12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

13# See the License for the specific language governing permissions and 

14# limitations under the License. 

15 

16""" 

17Base Trainer class for distributed training. 

18 

19This module provides the BaseTrainer class which serves as the foundation 

20for all trainer implementations. Subclasses can override specific methods 

21to customize training behavior. 

22 

23Features: 

24 - Callback system for extensible training hooks 

25 - Distributed training support 

26 - Gradient accumulation 

27 - Checkpointing 

28""" 

29 

30from __future__ import annotations 

31 

32import json 

33import logging 

34import os 

35from abc import ABC 

36from collections import defaultdict 

37from collections.abc import Callable 

38from contextlib import nullcontext 

39from functools import partial 

40from typing import TYPE_CHECKING, Any, Dict, List, Optional 

41 

42import torch 

43import torch.distributed as dist 

44from torch.distributed.checkpoint.stateful import Stateful 

45from torch.optim.lr_scheduler import LRScheduler 

46from torch.utils.data import Dataset 

47from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase 

48from transformers.modeling_outputs import ModelOutput 

49 

50from hyper_parallel import DTensor, HSDPModule, SkipDTensorDispatch 

51from hyper_parallel.compile.compiler import GraphCompiler 

52from hyper_parallel.core.tensor_parallel import loss_parallel 

53from hyper_parallel.core.utils import clip_grad_norm_ 

54from hyper_parallel.trainer.config import ( 

55 OptimizerSwapConfig, 

56 TrainerConfig, 

57 normalize_distributed_setup_overrides, 

58 save_configs, 

59) 

60from hyper_parallel.data.dataset_logging import enable_dataset_logging 

61from hyper_parallel.data.batching import build_dataloader 

62from hyper_parallel.data.parallel import DataLoaderParallelContext, create_dataloader_parallel_context 

63from hyper_parallel.trainer.runtime.distributed import ( 

64 all_reduce, 

65 get_global_rank_safe, 

66 get_local_rank_safe, 

67 get_world_size_safe, 

68) 

69from hyper_parallel.trainer.runtime.distributed import ( 

70 create_distributed_setup_from_config, 

71 destroy_process_group, 

72 initialize_distributed, 

73) 

74from hyper_parallel.trainer.runtime.logging import setup_logging 

75from hyper_parallel.trainer.runtime.loss_aggregation import count_loss_token 

76from hyper_parallel.trainer.runtime.metrics import mean_global_loss 

77from hyper_parallel.components.functional.aux_loss import aux_loss_scale_context, bind_aux_loss_scale 

78from hyper_parallel.components.modules.moe_utils import MoEMonitorCallback 

79from hyper_parallel.models._transformers.loss_parallel import causal_lm_loss_parallel 

80from hyper_parallel.models._transformers.model_builder import _validate_optimize_dtype 

81from hyper_parallel.models.external_state import get_model_external_state 

82from hyper_parallel.components.losses.model_output import ModelOutputLoss 

83from hyper_parallel.core.optimizer import ( 

84 ChainedOptimizer, 

85 SwapOptimizerConfig, 

86 swap_optimizer, 

87) 

88from hyper_parallel.components.optim.mixed_precision_optimizer import ( 

89 Float16OptimizerWithFloat16Params, 

90 MixedPrecisionOptimizer, 

91) 

92from hyper_parallel.trainer.runtime import fsdp as fsdp_runtime 

93from hyper_parallel.trainer.runtime import model_integration as model_integration_runtime 

94from hyper_parallel.trainer.runtime.data_iterator import HyperIter 

95from hyper_parallel.trainer.runtime.logging import enable_third_party_logging 

96from hyper_parallel.trainer.runtime.memory import empty_cache, print_device_mem_info 

97from hyper_parallel.trainer.runtime.memory_profiler import memory_profiler 

98from hyper_parallel.trainer.runtime.random import enable_high_precision_for_bf16, set_seed 

99from hyper_parallel.trainer.runtime.device import ( # pylint: disable=syntax-error 

100 get_device_type, 

101 get_torch_device, 

102 synchronize, 

103) 

104 

105from hyper_parallel.trainer.callbacks import ( 

106 EnvironMeterCallback, 

107 EvaluateCallback, 

108 GarbageCollectionCallback, 

109 LoggingCallback, 

110 ProfilerCallback, 

111 StressDetectCallback, 

112 TqdmCallback, 

113 CheckpointerCallback, 

114 TrainerState, 

115) 

116 

117logger = logging.getLogger(__name__) 

118 

119if TYPE_CHECKING: 

120 from hyper_parallel.data.text.chat_template import ChatTemplate 

121 from hyper_parallel.distributed_data import BalancingAlgorithm, CostModel 

122 from hyper_parallel.distributed_data.schema import SampleMetadata 

123 

124 

125def _to_swap_optimizer_config(config: OptimizerSwapConfig) -> SwapOptimizerConfig: 

126 """Translate the trainer-facing swap section into the core swap config. 

127 

128 Args: 

129 config: Resolved ``optimizer.swap`` configuration section. 

130 

131 Returns: 

132 The backend-neutral swap configuration consumed by ``swap_optimizer``. 

133 """ 

134 return SwapOptimizerConfig( 

135 swap_times=config.swap_times, 

136 state_keys=config.state_keys, 

137 min_numel=config.min_numel, 

138 # ``None`` means "leave it to the backend default", not "off": the core 

139 # config declares ``packed_swap: bool = True``, so pass that default 

140 # through rather than letting ``None`` read as a hard False. 

141 packed_swap=True if config.packed_swap is None else config.packed_swap, 

142 ) 

143 

144 

145def _attach_optimizer_swap(optimizer: Any, config: OptimizerSwapConfig) -> Any: 

146 """Attach optimizer-state swap to every Adam/AdamW leaf of ``optimizer``. 

147 

148 Every YAML optimizer target returns a ``ChainedOptimizer`` of leaf 

149 optimizers -- also for a single AdamW -- and the fp32 main-parameter wrapper 

150 keeps the same leaf containers. The swap runtime wraps one concrete 

151 Adam/AdamW and delegates everything else, so it is attached to the leaves 

152 and the outer object keeps dispatching. 

153 

154 Args: 

155 optimizer: Optimizer built by the configured target, optionally already 

156 wrapped for fp32 main parameters. 

157 config: Resolved ``optimizer.swap`` configuration section. 

158 

159 Returns: 

160 The optimizer carrying swap-wrapped leaves. A leaf the swap runtime does 

161 not support fails here, before the first training step. 

162 """ 

163 swap_config = _to_swap_optimizer_config(config) 

164 chained = optimizer.optimizer if isinstance(optimizer, MixedPrecisionOptimizer) else optimizer 

165 if not isinstance(chained, ChainedOptimizer): 

166 return swap_optimizer(optimizer, swap_config) 

167 

168 names = list(chained.optimizers_dict) 

169 leaves = [swap_optimizer(chained.optimizers_dict[name], swap_config) for name in names] 

170 # In place: the fp32 main-parameter wrapper aliases both containers. 

171 chained.chained_optimizers[:] = leaves 

172 for name, leaf in zip(names, leaves): 

173 chained.optimizers_dict[name] = leaf 

174 return optimizer 

175 

176 

177class BaseTrainer(Stateful, ABC): 

178 """ 

179 Base trainer class for distributed model training. 

180 

181 This class provides the core training infrastructure including: 

182 - Distributed initialization and parallelism setup 

183 - Model, optimizer, and scheduler initialization 

184 - Training step execution with gradient accumulation 

185 - Checkpointing and fault tolerance 

186 - Metrics logging 

187 

188 Subclasses can override the following methods to customize behavior: 

189 - `_build_distributed_setup()`: Connect AutoModel DistributedSetup/MeshContext 

190 - `_build_model()`: Connect AutoModel from_pretrained/parallelize 

191 - `_build_model_assets()`: Build tokenizer/processor assets 

192 - `_build_data_transform()`: Build modality-specific sample transforms 

193 - `forward_backward_step()`: Customize forward/backward logic 

194 - `train_step()`: Customize training step execution 

195 - `train()`: Train the model 

196 

197 Callback Hooks: 

198 The trainer calls callback methods at various stages: 

199 - evaluate_callback: evaluation callback 

200 - trace_callback: tracing callback (meter, wandb, tqdm, profile) 

201 - checkpoint_callback: checkpointing callback 

202 """ 

203 

204 # Core configs 

205 config: TrainerConfig 

206 device: torch.device 

207 # Attached at runtime by the model-integration pipeline (the Omni dataset builder sets 

208 # ``data_transform``, the integration adapter attaches ``get_batch`` and 

209 # ``num_micro_batches``): bare annotations, which assign nothing, so runtime behaviour is 

210 # unchanged -- they exist so the type checker sees the contract this class reads. 

211 data_transform: Any 

212 get_batch: Any 

213 num_micro_batches: int 

214 

215 # AutoModel distributed setup. 

216 # These objects have one owner: ``../build_options.py`` and 

217 # ``../components/distributed/mesh.py``. Trainer code consumes them but must 

218 # not create a second DeviceMesh or a VeOmni ParallelState. 

219 dist_info: Any 

220 distributed_setup: Any 

221 mesh_context: Any 

222 device_mesh: Any 

223 moe_mesh: Any 

224 

225 # Data 

226 train_dataset: Dataset 

227 data_transform: Any 

228 collate_fn: Any 

229 get_batch: Any 

230 dataloader_context: DataLoaderParallelContext | None = None 

231 train_dataloader: Any = None 

232 num_micro_batches: int 

233 

234 # Model 

235 model: PreTrainedModel = None 

236 model_parts: Optional[List[torch.nn.Module]] = None 

237 hsdp_model_parts: List[torch.nn.Module] = [] 

238 fsdp_runtime_diagnostics: Optional[fsdp_runtime.FSDPRuntimeDiagnostics] = None 

239 model_integration: Optional[model_integration_runtime.ModelIntegrationSession] = None 

240 model_config: PretrainedConfig = PretrainedConfig() 

241 tokenizer: PreTrainedTokenizerBase = None 

242 processor: Any = None 

243 chat_template: Optional[ChatTemplate] = None 

244 model_assets: List[Any] = [] 

245 

246 # Training components 

247 optimizer = None 

248 lr_scheduler: LRScheduler = None 

249 loss_fn: Optional[torch.nn.Module] = None 

250 

251 # Training context 

252 model_fwd_context: Any 

253 model_bwd_context: Any 

254 

255 # # Runtime metrics, controlled by trace_callback 

256 # environ_meter: helper.EnvironMeter # see in trace_callback.EnvironMeterCallback 

257 # step_env_metrics: Dict[str, Any] # mfu, flops, tokens, etc 

258 # step_train_metrics: Dict[str, Any] # loss, grad_norm, lr, etc 

259 

260 # # Checkpointer 

261 # checkpointer: CheckpointerBase # see in checkpoint_callback.CheckpointerCallback 

262 

263 # Callback system 

264 state: TrainerState 

265 

266 # Default seed when training.seed is omitted. 

267 default_seed: int = 42 

268 

269 def __init__(self, config: TrainerConfig) -> None: 

270 """ 

271 Initialize the trainer. 

272 

273 Args: 

274 args: Global Arguments 

275 Should have attributes: model, data, train 

276 model: ModelArguments 

277 data: DataArguments 

278 train: TrainingArguments 

279 """ 

280 

281 self.config: TrainerConfig = config 

282 

283 # General setup owns distributed initialization as its first stage. 

284 # ``_build_distributed_setup`` remains a reserved backend hook; the 

285 # trainer only defines its contract here. 

286 self._setup() 

287 memory_profiler.reset( 

288 self.config.memory, 

289 global_rank=self.global_rank, 

290 tp_rank=self.mesh.tp_rank, 

291 dp_rank=self.mesh.dp_rank, 

292 ) 

293 

294 # Reserved atomic model-build interface. 

295 # 

296 # Owner: 

297 # - ``../_transformers/auto_model.py::HyperAutoModel*.from_pretrained`` 

298 # - ``../_transformers/model_builder.py`` for materialization, 

299 # checkpoint loading, PEFT/quantization, and parallelize. 

300 # 

301 # Contract: 

302 # - consume the ``DistributedSetup`` created above; 

303 # - set ``model`` and the resolved checkpoint ``model_config``; 

304 # - return a materialized, weight-loaded, already-parallelized model; 

305 # - never call a second trainer-side ``build_parallelize_model``. 

306 self._build_model() 

307 self._build_loss() 

308 

309 # Build trainer-owned data components after the model finalizes parameters and sharding. 

310 self._build_model_assets() 

311 self._build_data_transform() 

312 self._build_dataset() 

313 self._build_collate_fn() 

314 self._build_dataloader() 

315 self._compute_train_iters() 

316 

317 self._build_optimizer() 

318 self.model_integration.attach_optimizer(self.optimizer) 

319 self._build_lr_scheduler() 

320 self._build_training_context() 

321 self._init_callbacks() 

322 

323 def _setup(self): 

324 """Initialize logging, distributed state, and the local device.""" 

325 # log args 

326 setup_logging() 

327 

328 # Initialize distributed process group 

329 backend = getattr(self.config.training, "backend", "nccl") 

330 if backend == "hccl" and self.config.training.enable_full_determinism: 

331 # HCCL reads this when creating communicators, before set_seed runs below. 

332 os.environ["HCCL_DETERMINISTIC"] = "true" 

333 initialize_distributed(backend=backend) 

334 

335 self.local_rank = get_local_rank_safe() 

336 self.global_rank = get_global_rank_safe() 

337 self.world_size = get_world_size_safe() 

338 if self.config.debug.check_dataset: 

339 enable_dataset_logging(self.config.debug.check_dataset) 

340 

341 if self.config.training.seed is None: 

342 self.config.training.seed = self.default_seed 

343 

344 if self.global_rank == 0: 

345 logger.info( 

346 "Trainer config:\n%s", 

347 json.dumps(self.config.to_dict(), indent=2), 

348 ) 

349 

350 # init distributed environment 

351 device_str = f"{get_device_type()}:{self.local_rank}" 

352 get_torch_device().set_device(device_str) 

353 self.device = torch.device(device_str) 

354 

355 logger.info("Process rank: %s, world size: %s", self.global_rank, self.world_size) 

356 

357 # Register distributed_setup; the Trainer-side desugar fills its 

358 # normalized plan_overrides/module_replacements (05 §15.2.6) so the 

359 # AutoModels build pipeline never sees raw YAML PlanOverride DTOs. 

360 self.distributed_setup = create_distributed_setup_from_config(self.config) 

361 normalize_distributed_setup_overrides(self.distributed_setup, self.config) 

362 self.mesh = self.distributed_setup.mesh_context 

363 self.device_mesh = self.mesh.device_mesh 

364 self.dp_cp_mesh = self.mesh.dp_cp_mesh 

365 

366 # Set random seed 

367 set_seed(self.config.training.seed, self.config.training.enable_full_determinism) 

368 

369 # Enable high precision for bf16 

370 enable_high_precision_for_bf16() 

371 

372 # Enable third party logging 

373 if self.local_rank == 0: 

374 enable_third_party_logging() 

375 

376 # Save arguments 

377 if self.global_rank == 0: 

378 save_configs(self.config, self.config.checkpoint.checkpoint_dir) 

379 

380 def _build_model(self) -> None: 

381 """Build the model and derive Trainer-owned runtime state.""" 

382 self.peft_config = self.config.peft 

383 self.model = self.config.model.build( 

384 distributed_setup=self.distributed_setup, 

385 peft_config=self.peft_config, 

386 activation_checkpoint=self.config.activation_checkpoint.mode, 

387 activation_checkpoint_selection=self.config.activation_checkpoint.selection, 

388 swap_inputs=getattr(self.config.activation_checkpoint, "swap_inputs", False), 

389 allow_cache_entry_mutation=getattr( 

390 self.config.activation_checkpoint, "allow_cache_entry_mutation", False), 

391 activation_swap=self.config.activation_swap, 

392 compile_config=self.config.compile, 

393 # The final dtype is applied inside the atomic build (05 stage-5 

394 # item 5); the Trainer no longer patches it afterwards. 

395 model_init_dtype=self.config.model_init_dtype, 

396 ) 

397 _validate_optimize_dtype( 

398 self.model, 

399 fp32_main_params=self.config.optimizer.fp32_main_params, 

400 ) 

401 self.model_config = self.model.config 

402 if self.global_rank == 0: 

403 logger.info( 

404 "Model config:\n%s", 

405 json.dumps(self.model.config.to_dict(), indent=2), 

406 ) 

407 if self.config.accelerator.loss_parallel: 

408 self.model.loss_function = causal_lm_loss_parallel 

409 model_parts = getattr(self.model, "parts", None) 

410 self.model_parts = list(model_parts) if model_parts is not None else [self.model] 

411 self.hsdp_model_parts = [ 

412 model_part 

413 for model_part in self.model_parts 

414 if isinstance(model_part, HSDPModule) 

415 ] 

416 self.model_integration = model_integration_runtime.build_model_integration_session( 

417 self.model, 

418 self.config, 

419 self.global_rank, 

420 self.mesh, 

421 ) 

422 self.fsdp_runtime_diagnostics = ( 

423 fsdp_runtime.build_fsdp_runtime_diagnostics(self.hsdp_model_parts) 

424 if getattr(self.config.debug, "check_fsdp_runtime", False) 

425 else None 

426 ) 

427 

428 def _build_loss(self) -> None: 

429 """Build the configured loss module or use the model-output default.""" 

430 loss_fn = ( 

431 self.config.loss_fn.build() 

432 if self.config.loss_fn is not None 

433 else ModelOutputLoss() 

434 ) 

435 if not isinstance(loss_fn, torch.nn.Module): 

436 raise ValueError("config.loss_fn must build a torch.nn.Module") 

437 self.loss_fn = loss_fn 

438 # Model-integrated objectives may need to intercept the model before its 

439 # ordinary forward materializes an otherwise avoidable output tensor. 

440 loss_model_binding_hook = getattr(self.loss_fn, "bind_model", None) 

441 if callable(loss_model_binding_hook): 

442 loss_model_binding_hook(self.model, self.distributed_setup) 

443 

444 def _build_model_assets(self) -> None: 

445 """Require a concrete Trainer to build modality-specific model assets.""" 

446 raise NotImplementedError("Concrete Trainer must implement _build_model_assets") 

447 

448 def _build_data_transform(self) -> None: 

449 """Require a concrete Trainer to build its sample transform.""" 

450 raise NotImplementedError("Concrete Trainer must implement _build_data_transform") 

451 

452 def _build_dataset(self) -> None: 

453 """Build and assign one Dataset or a train-validation-test tuple.""" 

454 data_config = getattr(self.config.dataset, "data_config", {}) 

455 self.dataloader_context = create_dataloader_parallel_context( 

456 self.mesh, 

457 data_index_cache=bool(data_config.get("data_index_cache", False)), 

458 shared_storage=not bool(data_config.get("no_shared_storage", False)), 

459 tp_source_rank=None if self.config.dataloader.load_balance is not None else 0, 

460 ) 

461 dataset_result = self.config.dataset.build( 

462 transform=self.data_transform, 

463 tokenizer=self.tokenizer, 

464 mesh_context=self.mesh, 

465 training_config=self.config.training, 

466 sampler_type=self.config.dataloader.sampler_type, 

467 dataloader_context=self.dataloader_context, 

468 ) 

469 

470 if isinstance(dataset_result, tuple) and len(dataset_result) == 3: 

471 self.train_dataset, self.valid_dataset, self.test_dataset = dataset_result 

472 return 

473 

474 self.train_dataset = dataset_result 

475 self.valid_dataset = self.test_dataset = None 

476 

477 def _build_collate_fn(self) -> None: 

478 """Require a concrete Trainer to build its micro-batch collator.""" 

479 raise NotImplementedError("Concrete Trainer must implement _build_collate_fn") 

480 

481 def _build_dataloader( 

482 self, 

483 *, 

484 metadata_fn: Callable[[Any], SampleMetadata] | None = None, 

485 cost_model: CostModel | None = None, 

486 balancing_algorithm: BalancingAlgorithm | None = None, 

487 ) -> None: 

488 """Build and assign train, validation, and test dataloaders.""" 

489 split_names = ("train", "valid", "test") 

490 datasets = tuple(getattr(self, f"{split_name}_dataset", None) for split_name in split_names) 

491 dataloaders, batch_samplers = build_dataloader( 

492 self.config.dataloader, 

493 datasets=datasets, 

494 collate_fn=self.collate_fn, 

495 mesh_context=self.mesh, 

496 training_config=self.config.training, 

497 data_config=getattr(self.config.dataset, "data_config", {}), 

498 max_seq_len=getattr(self.data_transform, "max_seq_len", None), 

499 default_seed=self.default_seed, 

500 metadata_fn=metadata_fn, 

501 model_config=self.model_config, 

502 cost_model=cost_model, 

503 balancing_algorithm=balancing_algorithm, 

504 ) 

505 for split_name, dataloader, batch_sampler in zip(split_names, dataloaders, batch_samplers): 

506 setattr(self, f"{split_name}_dataloader", dataloader) 

507 setattr(self, f"{split_name}_batch_sampler", batch_sampler) 

508 

509 def attach_model_integration_data_pipeline(self) -> None: 

510 """Attach the final data pipeline through one shared Trainer path.""" 

511 self.model_integration.attach_data_pipeline( 

512 runtime_adapter=getattr(self.get_batch, "runtime_input_adapter", None), 

513 get_batch=self.get_batch, 

514 model_assets=tuple(self.model_assets), 

515 ) 

516 

517 def _compute_train_iters(self) -> None: 

518 """Resolve the run limit and the optimizer steps available per Dataset epoch.""" 

519 training_config = self.config.training 

520 if training_config.train_iters is not None and training_config.train_samples is not None: 

521 raise ValueError("configure only one of training.train_iters and training.train_samples") 

522 

523 if training_config.train_iters is None and training_config.train_samples is None: 

524 raise ValueError("configure one of training.train_iters or training.train_samples") 

525 

526 train_iters = training_config.train_iters 

527 if training_config.train_samples is not None: 

528 train_iters = training_config.train_samples // training_config.global_batch_size 

529 

530 if train_iters <= 0: 

531 raise ValueError("training configuration must produce at least one train iteration") 

532 

533 try: 

534 micro_batches_per_epoch = len(self.train_dataloader) 

535 except TypeError: 

536 micro_batches_per_epoch = 0 

537 

538 if micro_batches_per_epoch: 

539 train_steps = micro_batches_per_epoch // self.num_micro_batches 

540 if train_steps <= 0: 

541 raise ValueError("train DataLoader must provide one complete optimizer step per epoch") 

542 else: 

543 # Streaming and dynamic DataLoaders define an epoch only when their 

544 # source iterator is exhausted. 

545 train_steps = train_iters 

546 

547 self.train_iters = train_iters 

548 self.train_steps = train_steps 

549 self.train_epochs = (self.train_iters + self.train_steps - 1) // self.train_steps 

550 

551 def _build_optimizer(self) -> None: 

552 """Build the configured optimizer with the runtime model context.""" 

553 config: TrainerConfig = self.config 

554 external = get_model_external_state(self.model) 

555 if external is not None: 

556 def wrap_dense(dense_raw: Any) -> Any: 

557 """Wrap dense parameters in FP32 main and optional state swap. 

558 

559 Args: 

560 dense_raw: Constructed dense optimizer. 

561 """ 

562 dense = ( 

563 Float16OptimizerWithFloat16Params(dense_raw, self.model) 

564 if config.optimizer.fp32_main_params else dense_raw 

565 ) 

566 if config.optimizer.swap.enabled: 

567 dense = _attach_optimizer_swap(dense, config.optimizer.swap) 

568 return dense 

569 

570 self.optimizer = external.build_optimizer(config.optimizer, wrap_dense=wrap_dense) 

571 return 

572 optimizer = config.optimizer.target.build(model=self.model).get_optimizer() 

573 self.optimizer = ( 

574 Float16OptimizerWithFloat16Params(optimizer, self.model) 

575 if config.optimizer.fp32_main_params 

576 else optimizer 

577 ) 

578 # Swap is attached after the fp32 main-parameter wrap so leaf optimizers 

579 # already reference their final parameters, and the lr scheduler built 

580 # right after reads the wrapped ``self.optimizer.param_groups``. 

581 if config.optimizer.swap.enabled: 

582 self.optimizer = _attach_optimizer_swap(self.optimizer, config.optimizer.swap) 

583 

584 def _build_lr_scheduler(self): 

585 config: TrainerConfig = self.config 

586 scheduler_iters = config.training.lr_scheduler_iters or self.train_iters 

587 if scheduler_iters <= 0: 

588 raise ValueError("training.lr_scheduler_iters must be positive when configured") 

589 lr_scheduler = config.lr_scheduler.build( 

590 optimizer=self.optimizer, 

591 train_iters=scheduler_iters, 

592 ) 

593 self.lr_scheduler = lr_scheduler.get_lr_scheduler() 

594 

595 def _build_training_context(self): 

596 """Build training context for distributed training.""" 

597 if self.config.accelerator.loss_parallel: 

598 tp_mesh = self.device_mesh["tp"] 

599 self.model_fwd_context = partial(loss_parallel, mesh=tp_mesh) 

600 else: 

601 self.model_fwd_context = nullcontext() 

602 self.model_bwd_context = nullcontext() 

603 self.graph_compiler: Optional[GraphCompiler] = None 

604 

605 def _build_graph_compiler(self) -> None: 

606 """Build the optional joint-graph compiler for this trainer runtime.""" 

607 if not self.config.compile.selects_graph_compiler(): 

608 return 

609 self.graph_compiler = GraphCompiler( 

610 model=self.model, 

611 train_fn=self._graph_train_fn, 

612 trainer_config=self.config, 

613 device=self.device, 

614 mesh_context=self.mesh, 

615 source_shard_info=getattr(self.model, "_source_shard_info", None), 

616 ) 

617 

618 def _graph_train_fn( 

619 self, 

620 model: torch.nn.Module, 

621 model_inputs: Dict[str, Any], 

622 labels: Optional[torch.Tensor], 

623 current_token_counts: Dict[str, torch.Tensor], 

624 ) -> tuple[torch.Tensor, Dict[str, torch.Tensor]]: 

625 """Capture forward/loss with explicit runtime token counts.""" 

626 model_fwd_context = ( 

627 self.model_fwd_context() 

628 if callable(self.model_fwd_context) 

629 else self.model_fwd_context 

630 ) 

631 with model_fwd_context: 

632 outputs = model(**model_inputs, use_cache=False) 

633 # Counts must be graph inputs: packed lengths and ignored-label counts 

634 # can vary even when tensor shapes happen to match the first batch. 

635 loss, loss_dict = self.postforward( 

636 outputs, labels, 

637 current_token_counts=current_token_counts, 

638 ) 

639 return loss, loss_dict 

640 

641 def _init_callbacks(self): 

642 """Initialize callbacks.""" 

643 dp_cp_mesh = self.mesh.dp_cp_mesh 

644 self.moe_monitor = MoEMonitorCallback( 

645 self.model, 

646 dp_group=dp_cp_mesh.get_group() if dp_cp_mesh is not None else None, 

647 tp_group=self.device_mesh["tp"].get_group() if self.mesh.sequence_parallel else None, 

648 ) 

649 self.state = TrainerState() 

650 # Raw AUX is not normalized with the backward loss; average its observations. 

651 self.environ_meter_callback = EnvironMeterCallback(self, microbatch_mean_metrics=("router_aux_loss",)) 

652 self.tqdm_callback = TqdmCallback(self) 

653 self.logging_callback = LoggingCallback(self) 

654 self.evaluate_callback = EvaluateCallback(self) 

655 self.garbage_collection_callback = GarbageCollectionCallback(self) 

656 self.profiler_callback = ProfilerCallback(self) 

657 self._callbacks = [ 

658 self.environ_meter_callback, 

659 self.logging_callback, 

660 self.tqdm_callback, 

661 self.evaluate_callback, 

662 self.garbage_collection_callback, 

663 self.profiler_callback, 

664 ] 

665 if self.config.stress_detect.op_stress_detect.detection_interval > 0: 

666 self.stress_detect_callback = StressDetectCallback(self) 

667 self._callbacks.append(self.stress_detect_callback) 

668 # Registered to save, to restore, or both --- the two are independent, so 

669 # a run that only loads an existing checkpoint still needs the callback. 

670 if ( 

671 self.config.checkpoint.save_ckpt 

672 or self.config.checkpoint.save_hf_weights 

673 or self.config.checkpoint.restore_from is not None 

674 ): 

675 self._callbacks.append(CheckpointerCallback(self)) 

676 else: 

677 logger.info( 

678 "Checkpointing is inactive (save_ckpt=false, save_hf_weights=false, " 

679 "restore_from=None); " 

680 "no checkpoint callback registered." 

681 ) 

682 

683 def on_train_begin(self) -> None: 

684 """Run all registered callbacks at the start of training.""" 

685 for callback in self._callbacks: 

686 callback.on_train_begin(self.state) 

687 

688 def on_train_end(self) -> None: 

689 """Run all registered callbacks at the end of training.""" 

690 for callback in self._callbacks: 

691 callback.on_train_end(self.state) 

692 

693 def on_epoch_begin(self) -> None: 

694 """Run all registered callbacks at the start of an epoch.""" 

695 for callback in self._callbacks: 

696 callback.on_epoch_begin(self.state) 

697 

698 def on_epoch_end(self) -> None: 

699 """Run all registered callbacks at the end of an epoch.""" 

700 for callback in self._callbacks: 

701 callback.on_epoch_end(self.state) 

702 

703 def on_step_begin(self, **kwargs: Any) -> None: 

704 """Run all registered callbacks at the start of a training step.""" 

705 for callback in self._callbacks: 

706 callback.on_step_begin(self.state, **kwargs) 

707 

708 def on_micro_step_begin(self, micro_batch: Dict[str, Any], **kwargs: Any) -> None: 

709 """Run all registered callbacks before one forward-backward micro step. 

710 

711 Args: 

712 micro_batch: Prepared inputs for the current micro step. 

713 **kwargs: Additional callback context. 

714 """ 

715 for callback in self._callbacks: 

716 callback.on_micro_step_begin(self.state, micro_batch, **kwargs) 

717 

718 def on_micro_step_end(self, loss_dict: Dict[str, torch.Tensor], **kwargs: Any) -> None: 

719 """Run all registered callbacks after one forward-backward micro step. 

720 

721 Args: 

722 loss_dict: Named losses and metrics returned by the completed micro step. 

723 **kwargs: Additional callback context. 

724 """ 

725 for callback in self._callbacks: 

726 callback.on_micro_step_end(self.state, loss_dict=loss_dict, **kwargs) 

727 

728 def on_step_end( 

729 self, 

730 loss: Optional[float] = None, 

731 loss_dict: Optional[Dict[str, Any]] = None, 

732 grad_norm: Optional[float] = None, 

733 ) -> None: 

734 """Run all registered callbacks at the end of a training step. 

735 

736 Args: 

737 loss: Aggregated loss for the optimizer step. 

738 loss_dict: Named loss values for the optimizer step. 

739 grad_norm: Gradient norm measured before the optimizer update. 

740 """ 

741 for callback in self._callbacks: 

742 callback.on_step_end(self.state, loss=loss, loss_dict=loss_dict, grad_norm=grad_norm) 

743 

744 def preforward(self, micro_batch: List[Dict[str, Any]]) -> Dict[str, Any]: 

745 """Preprocess micro batches before forward pass. 

746 

747 Tensors are moved to ``self.device`` non-blockingly. Nested dicts 

748 (e.g. ``multimodal_metadata`` emitted by ``PackingCollator``) are 

749 recursed so inner tensor values land on the device too; Python ints 

750 / lists / etc. pass through unchanged. 

751 

752 Args: 

753 micro_batch: Batch fields to move to the training device. 

754 

755 Returns: 

756 Batch fields with tensors moved to the training device. 

757 """ 

758 

759 def _to_device(v: Any) -> Any: 

760 if isinstance(v, torch.Tensor): 

761 return v.to(self.device, non_blocking=True) 

762 if isinstance(v, dict): 

763 return {k: _to_device(vv) for k, vv in v.items()} 

764 return v 

765 

766 # chunk_mbs_config = getattr(self.config.training, "chunk_mbs_config", None) 

767 # self._chunk_mbs_ranges = build_chunk_mbs_ranges(micro_batch, chunk_mbs_config) 

768 micro_batch = {k: _to_device(v) for k, v in micro_batch.items()} # type: ignore 

769 # if getattr(self, "LOG_SAMPLE", True): 

770 # self.LOG_SAMPLE = False 

771 return micro_batch 

772 

773 def postforward( 

774 self, outputs: ModelOutput, labels: Optional[torch.Tensor], 

775 *, 

776 current_token_counts: Dict[str, torch.Tensor], 

777 ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: 

778 """Postprocess model outputs after forward pass. 

779 

780 Args: 

781 outputs: Model outputs consumed by the configured loss. 

782 labels: Labels associated with the current micro-batch. 

783 current_token_counts: Per-loss token counts for the current micro-batch. 

784 

785 Returns: 

786 Backward loss and named globally aggregated loss values. 

787 """ 

788 local_loss = self.loss_fn(model_output=outputs, labels=labels) 

789 # Names are static graph metadata; diagnostics added below are not losses. 

790 self.loss_names = tuple(local_loss) if isinstance(local_loss, dict) else ("foundation_loss",) 

791 # Every foundation-family objective receives the same external token weight. 

792 # Bind each branch before backward can reach its independently injected losses. 

793 foundation_losses = ( 

794 (value for key, value in local_loss.items() if key.split("_loss", maxsplit=1)[0] == "foundation") 

795 if isinstance(local_loss, dict) else (local_loss,) 

796 ) 

797 for foundation_loss in foundation_losses: 

798 if foundation_loss is not None: 

799 bind_aux_loss_scale(foundation_loss) 

800 loss_dict: Dict[str, torch.Tensor] = mean_global_loss( 

801 local_loss, 

802 current_token_counts, 

803 device_mesh=self.mesh, 

804 ) 

805 loss = torch.stack(list(loss_dict.values())).sum() 

806 # Model-declared extra metrics join the logged mapping only after 

807 # the backward loss is fixed: log-only, never part of the loss. 

808 extra_metrics = getattr(outputs, "extra_metrics", None) 

809 if extra_metrics is None: 

810 model_ref = getattr(self, "model", None) 

811 extra_metrics = getattr(model_ref, "pending_extra_metrics", None) 

812 if extra_metrics is not None: 

813 model_ref.pending_extra_metrics = None 

814 if extra_metrics: 

815 for name, value in extra_metrics.items(): 

816 loss_dict[name] = value.detach().float() 

817 # Log-only outputs share the eager/graph path without entering backward loss. 

818 moe_monitor = getattr(self, "moe_monitor", None) 

819 if moe_monitor is not None: 

820 aux_loss = moe_monitor.last_mean_aux_loss 

821 if aux_loss is not None: 

822 loss_dict["router_aux_loss"] = aux_loss 

823 return loss, loss_dict 

824 

825 def forward_backward_step( 

826 self, 

827 micro_batch: dict[str, torch.Tensor], 

828 loss_inputs: Optional[dict[str, torch.Tensor]] = None, 

829 ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: 

830 """Run forward and backward computation for one micro batch. 

831 

832 Args: 

833 micro_batch: Inputs forwarded to the model. 

834 loss_inputs: Loss-only labels and masks from the get-batch runtime. If 

835 omitted, the legacy single-dictionary path is preserved. 

836 

837 Returns: 

838 Backward loss and the named globally aggregated loss mapping. 

839 """ 

840 for name, count in self.current_token_counts.items(): 

841 self.step_token_counts[name] = self.step_token_counts.get(name, 0) + count.detach() 

842 channel_loss_callback = getattr(self, "channel_loss_callback", None) 

843 micro_step_context = ( 

844 channel_loss_callback.micro_step_context(self.state, micro_batch) 

845 if channel_loss_callback is not None 

846 else nullcontext() 

847 ) 

848 with micro_step_context, aux_loss_scale_context(): 

849 micro_batch = self.preforward(micro_batch) 

850 # TextTrainer passes loss-only fields separately. Keep the legacy 

851 # BaseTrainer/VLM one-dictionary call contract working as well. 

852 if loss_inputs is None: 

853 loss_inputs = { 

854 name: micro_batch[name] 

855 for name in ("labels", "shift_labels", "loss_mask") 

856 if name in micro_batch 

857 } 

858 else: 

859 loss_inputs = self.preforward(loss_inputs) 

860 labels = loss_inputs.get("labels", micro_batch.get("labels")) 

861 

862 # A model-integrated loss can translate the public batch fields into 

863 # model-family protocol arguments before the model forward starts. 

864 prepare_model_inputs = getattr( 

865 self.loss_fn, 

866 "prepare_model_inputs", 

867 None, 

868 ) 

869 if callable(prepare_model_inputs): 

870 micro_batch = prepare_model_inputs(micro_batch, loss_inputs) 

871 if channel_loss_callback is not None: 

872 channel_loss_callback.strip_model_inputs(micro_batch) 

873 self.model_integration.record_batch(micro_batch, loss_inputs) 

874 

875 if self.graph_compiler is not None: 

876 runtime_adapter = getattr(getattr(self, "get_batch", None), "runtime_input_adapter", None) 

877 if runtime_adapter is not None: 

878 micro_batch = runtime_adapter.prepare_graph_inputs(micro_batch) 

879 if self.config.training.empty_cache_before_backward: 

880 logger.warning( 

881 "training.empty_cache_before_backward is ignored in graph compiler mode " 

882 "because forward and backward execute in one joint graph" 

883 ) 

884 # The traced ``loss_dict`` values flow back as extra graph 

885 # outputs (emitted after the gradients), so the logging 

886 # metrics keep per-key parity with the eager path. 

887 loss, loss_dict = self.graph_compiler.forward_backward( 

888 model_inputs=micro_batch, 

889 labels=labels, 

890 current_token_counts=self.current_token_counts, 

891 ) 

892 else: 

893 model_fwd_context = ( 

894 self.model_fwd_context() 

895 if callable(self.model_fwd_context) 

896 else self.model_fwd_context 

897 ) 

898 with model_fwd_context: 

899 outputs: ModelOutput = self.model(**micro_batch, use_cache=False) 

900 

901 # with use_parallel_state("base"): 

902 loss, loss_dict = self.postforward( 

903 outputs, labels, 

904 current_token_counts=self.current_token_counts, 

905 ) 

906 prefetch_plan = getattr(self.train_dataloader, "prefetch_plan", None) 

907 if callable(prefetch_plan): 

908 prefetch_plan() # pylint: disable=not-callable # Guarded dynamic hook. 

909 # The loss graph owns everything required for backward. Releasing 

910 # the model output here avoids retaining large vocabulary logits 

911 # until the whole backward pass finishes. 

912 del outputs 

913 if self.config.training.empty_cache_before_backward: 

914 empty_cache() 

915 

916 # Backward pass 

917 with self.model_bwd_context: 

918 loss.backward() 

919 prefetch = getattr(self.train_dataloader, "prefetch", None) 

920 if callable(prefetch): 

921 prefetch() # pylint: disable=not-callable # Guarded dynamic hook. 

922 

923 del micro_batch 

924 return loss, loss_dict 

925 

926 def model_reshard(self, micro_step: int, num_micro_steps: int) -> None: 

927 """Reshard model after backward pass; policy lives in ``runtime/fsdp.py``. 

928 

929 Args: 

930 micro_step: Zero-based micro-step index. 

931 num_micro_steps: Number of micro-steps in the optimizer step. 

932 """ 

933 fsdp_runtime.model_reshard(self.hsdp_model_parts, self.config.fsdp_config, micro_step, num_micro_steps) 

934 

935 def _configure_fsdp_gradient_sync(self, micro_step: int, num_micro_steps: int): 

936 """Configure FSDP gradient synchronization; policy lives in ``runtime/fsdp.py``.""" 

937 fsdp_runtime.configure_fsdp_gradient_sync( 

938 self.hsdp_model_parts, 

939 self.config.fsdp_config, 

940 self.mesh.dp_replicate_size, 

941 micro_step, 

942 num_micro_steps, 

943 ) 

944 

945 def configure_fsdp_gradient_sync(self, micro_step: int, num_micro_steps: int) -> None: 

946 """Configure FSDP gradient synchronization for an external training loop. 

947 

948 Args: 

949 micro_step: Zero-based micro-step index. 

950 num_micro_steps: Number of micro-steps in the optimizer step. 

951 """ 

952 self._configure_fsdp_gradient_sync(micro_step, num_micro_steps) 

953 

954 def begin_fsdp_runtime_diagnostics(self, micro_step: int) -> None: 

955 """Begin observing one FSDP forward/backward micro-step when enabled. 

956 

957 Args: 

958 micro_step: Microbatch index. 

959 """ 

960 if self.fsdp_runtime_diagnostics is not None: 

961 self.fsdp_runtime_diagnostics.begin_micro_step( 

962 self.state.global_step, 

963 micro_step, 

964 ) 

965 

966 def validate_fsdp_runtime_before_optimizer(self) -> None: 

967 """Fail before optimizer updates when FSDP prefetch state is unsafe.""" 

968 if self.fsdp_runtime_diagnostics is not None: 

969 self.model_integration.record_fsdp_trace( 

970 self.fsdp_runtime_diagnostics.trace_evidence() 

971 ) 

972 self.fsdp_runtime_diagnostics.validate_before_optimizer_step() 

973 

974 def begin_loss_step(self) -> None: 

975 """Reset scalar counters before reading the next optimizer step.""" 

976 self.step_token_counts = {} 

977 

978 def _loss_denominator(self) -> float: 

979 """Return the optimizer-step valid-token count across DP/CP.""" 

980 family = self.loss_names[0].split("_loss", maxsplit=1)[0] 

981 count = self.step_token_counts[f"{family}_tokens"].detach().clone() 

982 mesh = self.mesh.dp_cp_mesh 

983 if mesh is not None and mesh.size() > 1: 

984 count = all_reduce(count.to(self.device), op="sum", group=mesh.get_group()) 

985 else: 

986 count = count.item() 

987 return max(count, 1) 

988 

989 @torch.no_grad() 

990 def loss_normalize(self, loss: float, loss_dict: Dict[str, float], *, denominator: float | None = None) -> float: 

991 """Normalize Trainer-owned gradients and loss metrics before clipping. 

992 

993 External providers expose their owned parameters through ``parameters`` 

994 and normalize those gradients in ``prepare_optimizer_step`` instead. 

995 """ 

996 if denominator is None: 

997 denominator = self._loss_denominator() 

998 external = get_model_external_state(self.model) 

999 excluded = {id(param) for param in external.parameters.values()} if external is not None else set() 

1000 with SkipDTensorDispatch(): 

1001 for parameter in self.model.parameters(): 

1002 if id(parameter) in excluded: 

1003 continue 

1004 grad = getattr(parameter, "main_grad", None) 

1005 if grad is None: 

1006 grad = parameter.grad 

1007 if grad is not None: 

1008 if isinstance(grad, DTensor): 

1009 grad = grad.to_local() 

1010 grad.div_(denominator) 

1011 for name in self.loss_names: 

1012 loss_dict[name] /= denominator 

1013 return loss / denominator 

1014 

1015 def prepare_optimizer_step(self, loss: float, loss_dict: Dict[str, float]) -> tuple[float, Any]: 

1016 """Normalize the step objective before clipping and optimizer preparation.""" 

1017 denominator = self._loss_denominator() 

1018 loss = self.loss_normalize(loss, loss_dict, denominator=denominator) 

1019 return loss, self.clip_grad_norm(gradient_divisor=denominator) 

1020 

1021 def clip_grad_norm(self, *, gradient_divisor: float = 1.0) -> Any: 

1022 """Run pre-optimizer diagnostics and clipping in their required order.""" 

1023 self.model_integration.after_backward_before_clip() 

1024 self.validate_fsdp_runtime_before_optimizer() 

1025 max_grad_norm = self.config.training.max_grad_norm 

1026 external = get_model_external_state(self.model) 

1027 if external is not None: 

1028 preparation = external.prepare_optimizer_step(max_norm=max_grad_norm, gradient_divisor=gradient_divisor) 

1029 self.model_integration.after_clip(preparation.post_clip_norm) 

1030 return preparation.global_norm 

1031 grad_norm: Any = 0.0 

1032 post_clip_norm = None 

1033 compiler = getattr(self, "graph_compiler", None) 

1034 clip_kwargs = {"parameter_layouts": compiler.parameter_layouts} if compiler is not None else {} 

1035 if max_grad_norm > 0: 

1036 grad_norm = clip_grad_norm_(self.model, max_grad_norm, **clip_kwargs) 

1037 if self.model_integration.runtime_enabled: 

1038 grad_norm_value = ( 

1039 float(grad_norm.item()) 

1040 if isinstance(grad_norm, torch.Tensor) 

1041 else float(grad_norm) 

1042 ) 

1043 clip_coefficient = min( 

1044 float(max_grad_norm) / (grad_norm_value + 1.0e-6), 

1045 1.0, 

1046 ) 

1047 post_clip_norm = grad_norm_value * clip_coefficient 

1048 elif self.model_integration.runtime_enabled: 

1049 grad_norm = clip_grad_norm_(self.model, float("inf"), **clip_kwargs) 

1050 post_clip_norm = grad_norm 

1051 self.model_integration.after_clip(post_clip_norm) 

1052 return grad_norm 

1053 

1054 def step_optimizers_and_schedulers(self) -> None: 

1055 """Step every optimizer and scheduler, then expose the final state.""" 

1056 optimizers = self.optimizer if isinstance(self.optimizer, list) else [self.optimizer] 

1057 for optimizer in optimizers: 

1058 # Keep zeros_like dispatched so lazily created optimizer states retain DTensor metadata. 

1059 with SkipDTensorDispatch(no_skip={torch.zeros_like}): 

1060 optimizer.step() 

1061 optimizer.zero_grad() 

1062 moe_monitor = getattr(self, "moe_monitor", None) 

1063 if moe_monitor is not None: 

1064 moe_monitor.on_step_end() 

1065 self.model_integration.after_optimizer() 

1066 

1067 schedulers = ( 

1068 self.lr_scheduler 

1069 if isinstance(self.lr_scheduler, list) 

1070 else ([self.lr_scheduler] if self.lr_scheduler is not None else []) 

1071 ) 

1072 for scheduler in schedulers: 

1073 scheduler.step() 

1074 

1075 def _end_model_integration_step(self, metrics: Dict[str, Any]) -> None: 

1076 """Persist structured step metrics and global input identity.""" 

1077 if not self.model_integration.runtime_enabled: 

1078 return 

1079 resolved_metrics = dict(metrics) 

1080 optimizers = self.optimizer if isinstance(self.optimizer, list) else [self.optimizer] 

1081 learning_rates = [] 

1082 for optimizer in optimizers: 

1083 inner = getattr(optimizer, "optimizer", optimizer) 

1084 leaves = getattr(inner, "chained_optimizers", (inner,)) 

1085 for leaf in leaves: 

1086 learning_rates.extend( 

1087 group.get("lr") 

1088 for group in getattr(leaf, "param_groups", ()) 

1089 if group.get("lr") is not None 

1090 ) 

1091 if learning_rates: 

1092 resolved_metrics["lr"] = float(learning_rates[0]) 

1093 token_counts = getattr(self, "step_token_counts", {}) 

1094 token_values = [] 

1095 for value in token_counts.values(): 

1096 token_values.append(float(value.item() if torch.is_tensor(value) else value)) 

1097 if token_values: 

1098 resolved_metrics["tokens"] = sum(token_values) 

1099 resolved_metrics["samples"] = self.config.training.global_batch_size 

1100 self.model_integration.end_step(resolved_metrics) 

1101 

1102 def train_step( 

1103 self, 

1104 data_iterator: Any, 

1105 ) -> Dict[str, float]: 

1106 """Execute one optimizer update from the next dataloader batch. 

1107 

1108 Args: 

1109 data_iterator: Iterator providing the next micro-batches. 

1110 

1111 Returns: 

1112 Aggregated loss and gradient norm. 

1113 """ 

1114 memory_profiler.step() 

1115 micro_batches: List[Dict[str, Any]] = next(data_iterator) 

1116 self.state.global_step += 1 

1117 

1118 self.on_step_begin(micro_batches=micro_batches) 

1119 self.model_integration.begin_step(self.state.global_step) 

1120 

1121 # Forward and backward for each micro batch 

1122 synchronize() 

1123 

1124 total_loss = 0.0 

1125 total_loss_dict = defaultdict(int) 

1126 

1127 # token num for fixed_ce_loss in postforward 

1128 self.begin_loss_step() 

1129 num_micro_steps = len(micro_batches) 

1130 # forward and backward pass with gradient_accumulationsteps 

1131 for micro_step, micro_batch in enumerate(micro_batches): 

1132 self.model_reshard(micro_step, num_micro_steps) 

1133 self._configure_fsdp_gradient_sync(micro_step, num_micro_steps) 

1134 self.begin_fsdp_runtime_diagnostics(micro_step) 

1135 loss: torch.Tensor 

1136 loss_dict: Dict[str, torch.Tensor] 

1137 # token num for fixed_ce_loss in postforward 

1138 self.current_token_counts = count_loss_token(micro_batch) 

1139 loss, loss_dict = self.forward_backward_step(micro_batch) 

1140 self.on_micro_step_end(loss_dict) 

1141 

1142 total_loss += loss.item() 

1143 for k, v in loss_dict.items(): 

1144 total_loss_dict[k] += v.item() 

1145 

1146 total_loss, grad_norm = self.prepare_optimizer_step(total_loss, total_loss_dict) 

1147 self.step_optimizers_and_schedulers() 

1148 

1149 grad_norm_value = grad_norm.item() if isinstance(grad_norm, torch.Tensor) else float(grad_norm) 

1150 self._end_model_integration_step({"loss": total_loss, "grad_norm": grad_norm_value}) 

1151 self.on_step_end(loss=total_loss, loss_dict=total_loss_dict, grad_norm=grad_norm_value) 

1152 return { 

1153 "loss": total_loss, 

1154 "grad_norm": grad_norm_value, 

1155 } 

1156 

1157 def abort_profilers(self) -> None: 

1158 """Release process-local profiling resources while preserving a training error.""" 

1159 try: 

1160 memory_profiler.abort() 

1161 # Cleanup must also preserve the training error if interrupted. 

1162 except BaseException: # pylint: disable=broad-exception-caught 

1163 logger.exception("Failed to stop memory profiling while aborting training.") 

1164 profiler_callback = getattr(self, "profiler_callback", None) 

1165 if profiler_callback is not None: 

1166 try: 

1167 profiler_callback.close() 

1168 # Cleanup must also preserve the training error if interrupted. 

1169 except BaseException: # pylint: disable=broad-exception-caught 

1170 logger.exception("Failed to stop performance profiling while aborting training.") 

1171 

1172 def destroy_distributed(self) -> None: 

1173 """Synchronize all ranks and tear down the distributed process group.""" 

1174 if not dist.is_available() or not dist.is_initialized(): 

1175 return 

1176 

1177 empty_cache() 

1178 dist.barrier() 

1179 

1180 synchronize() 

1181 destroy_process_group() 

1182 

1183 def train(self) -> None: 

1184 """Run the configured training loop.""" 

1185 try: 

1186 config: TrainerConfig = self.config 

1187 self.on_train_begin() 

1188 self.data_iterator = HyperIter( 

1189 self.train_dataloader, use_background_prefetcher=config.dataloader.use_background_prefetcher 

1190 ) 

1191 logger.info( 

1192 "Rank%s Start training. Global step: %s. Train iters: %s. Start epoch: %s. Train epochs: %s.", 

1193 self.local_rank, self.state.global_step, self.train_iters, self.state.epoch, self.train_epochs, 

1194 ) 

1195 

1196 try: 

1197 start_epoch = self.state.epoch 

1198 for epoch in range(start_epoch, self.train_epochs): 

1199 if epoch != start_epoch: 

1200 self.train_dataloader.set_epoch(epoch) 

1201 self.data_iterator = HyperIter( 

1202 self.train_dataloader, use_background_prefetcher=config.dataloader.use_background_prefetcher 

1203 ) 

1204 self.state.epoch = epoch 

1205 

1206 self.on_epoch_begin() 

1207 

1208 start_step = self.state.global_step - epoch * self.train_steps 

1209 train_steps = min(self.train_steps, self.train_iters - epoch * self.train_steps) 

1210 for _ in range(start_step, train_steps): 

1211 try: 

1212 self.train_step(self.data_iterator) 

1213 except StopIteration: 

1214 logger.info( 

1215 "epoch:%s Dataloader finished with drop_last %s", 

1216 epoch, 

1217 config.dataloader.drop_last, 

1218 ) 

1219 break 

1220 

1221 self.on_epoch_end() 

1222 self.state.epoch = epoch + 1 

1223 

1224 print_device_mem_info(f"VRAM usage after epoch {epoch + 1}") 

1225 

1226 if config.dataloader.use_background_prefetcher: 

1227 self.data_iterator.stop() 

1228 

1229 memory_profiler.stop() 

1230 self.on_train_end() 

1231 finally: 

1232 if config.dataloader.use_background_prefetcher: 

1233 self.data_iterator.stop() 

1234 

1235 except BaseException: 

1236 try: 

1237 memory_profiler.stop() 

1238 # Snapshot export must preserve the training error even if interrupted. 

1239 except BaseException: # pylint: disable=broad-exception-caught 

1240 logger.exception("Failed to save memory snapshot after training error.") 

1241 finally: 

1242 self.abort_profilers() 

1243 raise 

1244 

1245 synchronize() 

1246 

1247 self.destroy_distributed()