Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / optimizer.py: 63%

545 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15 

16"""Base distributed optimizer and chain optimizer composition.""" 

17 

18from collections import defaultdict 

19from contextlib import contextmanager, nullcontext 

20import logging 

21from typing import Any, Dict, Iterator, List, Optional, Tuple 

22 

23import torch 

24import torch.distributed as dist 

25from torch.distributed.checkpoint.state_dict import ( 

26 StateDictOptions, 

27 get_optimizer_state_dict, 

28 set_optimizer_state_dict, 

29) 

30 

31from hyper_parallel.core.optimizer.dtensor_compat import to_local_if_dtensor 

32from hyper_parallel.core.optimizer.sharding_category import ( 

33 HSDPGroupAssignment, 

34 build_owner_by_size, 

35 get_multi_dim_logical_info, 

36 select_owned_records, 

37 group_parameters_for_hsdp 

38) 

39from hyper_parallel.core.optimizer.swap_optimizer import is_swap_optimizer 

40from hyper_parallel.core.optimizer.swap_optimizer_base import _restore_dtensor_metadata 

41from hyper_parallel.core.optimizer.utils import empty_accelerator_cache, get_current_device, get_device_count 

42 

43logger = logging.getLogger(__name__) 

44 

45 

46class ChainedOptimizer: 

47 """Composition wrapper that dispatches step/zero_grad to sub-optimizers.""" 

48 

49 def __init__( 

50 self, 

51 model: torch.nn.Module, 

52 optimizers: Dict[str, torch.optim.Optimizer], 

53 flatten: bool = False, 

54 ) -> None: 

55 """Compose named optimizers over registered model parameters. 

56 

57 Args: 

58 model: Module containing the registered model parameters. 

59 optimizers: Leaf optimizers keyed by their configured names. 

60 flatten: Whether leaf optimizer state uses flattened FQN keys. 

61 """ 

62 self.optimizers_dict = optimizers 

63 self.chained_optimizers = list(optimizers.values()) 

64 self.optimizers_keys = list(optimizers.keys()) 

65 self.model = model 

66 self.flatten = flatten # not flatten adamw, flatten for multi-optimizer 

67 self._is_multi_optimizer = flatten 

68 self.model_param_by_optimizer_param: Dict[ 

69 torch.nn.Parameter, torch.nn.Parameter 

70 ] = {} 

71 self.optimizer_param_by_model_param: Dict[ 

72 torch.nn.Parameter, torch.nn.Parameter 

73 ] = {} 

74 

75 self._rebind_param_attrs() 

76 

77 def __iter__(self) -> Iterator[torch.optim.Optimizer]: 

78 """Allow iteration over the underlying optimizers.""" 

79 return iter(self.chained_optimizers) 

80 

81 def _rebind_param_attrs(self) -> None: 

82 """Restore model FQNs and optimizer-family name views.""" 

83 muon_param_ids: set = set() 

84 adamw_param_ids: set = set() 

85 self.muon_keys = [] 

86 self.no_muon_keys = [] 

87 for _, opt in self.optimizers_dict.items(): 

88 is_muon = hasattr(opt, "is_muon") and opt.is_muon 

89 target = muon_param_ids if is_muon else adamw_param_ids 

90 for group in opt.param_groups: 

91 for p in group.get("params", []): 

92 target.add(id(p)) 

93 

94 for param_name, param in self.model.named_parameters(): 

95 optimizer_param = self.optimizer_param_by_model_param.get(param, param) 

96 setattr(param, "model_name", param_name) 

97 setattr(optimizer_param, "model_name", param_name) 

98 if id(optimizer_param) in muon_param_ids: 

99 self.muon_keys.append(param_name) 

100 elif id(optimizer_param) in adamw_param_ids: 

101 self.no_muon_keys.append(param_name) 

102 

103 def reset_optimizer_parameters( 

104 self, 

105 model_param_by_optimizer_param: Dict[ 

106 torch.nn.Parameter, torch.nn.Parameter 

107 ], 

108 ) -> None: 

109 """Register replaced optimizer params and rebuild identity-based metadata. 

110 

111 Args: 

112 model_param_by_optimizer_param: Main-to-model parameter identity map. 

113 """ 

114 self.model_param_by_optimizer_param = dict( 

115 model_param_by_optimizer_param 

116 ) 

117 self.optimizer_param_by_model_param = { 

118 model_param: optimizer_param 

119 for optimizer_param, model_param in self.model_param_by_optimizer_param.items() 

120 } 

121 self._rebind_param_attrs() 

122 for optimizer in self.chained_optimizers: 

123 reset = getattr(optimizer, "reset_optimizer_parameters", None) 

124 if callable(reset): 

125 if is_swap_optimizer(optimizer): 

126 reset(self.model_param_by_optimizer_param) 

127 else: 

128 reset() 

129 

130 @contextmanager 

131 def _optimizer_parameter_view(self) -> Iterator[None]: 

132 """Temporarily register optimizer params under their model FQNs.""" 

133 if not self.model_param_by_optimizer_param: 

134 yield 

135 return 

136 

137 registrations = [] 

138 for module in self.model.modules(): 

139 for parameter_name, model_param in module._parameters.items(): # pylint: disable=protected-access 

140 if model_param is None: 

141 continue 

142 optimizer_param = self.optimizer_param_by_model_param.get(model_param) 

143 if optimizer_param is None: 

144 continue 

145 registrations.append((module, parameter_name, model_param)) 

146 module._parameters[parameter_name] = optimizer_param # pylint: disable=protected-access 

147 try: 

148 yield 

149 finally: 

150 for module, parameter_name, model_param in registrations: 

151 module._parameters[parameter_name] = model_param # pylint: disable=protected-access 

152 

153 def __str__(self) -> str: 

154 """Return each named leaf optimizer's representation.""" 

155 if not self.optimizers_dict: 

156 return f'{self.__class__.__name__}' 

157 

158 sections = [] 

159 for name, optimizer in self.optimizers_dict.items(): 

160 sections.append(f'{name}: {repr(optimizer)}') 

161 return "\n\n".join(sections) 

162 

163 __repr__ = __str__ 

164 

165 def step(self, closure: Any = None) -> None: 

166 """Call each sub-optimizer's step in order. 

167 

168 Args: 

169 closure: Optional callable forwarded to every sub-optimizer. 

170 """ 

171 for opt in self.chained_optimizers: 

172 opt.step(closure=closure) 

173 

174 def zero_grad(self, set_to_none: bool = True) -> None: 

175 """Clear gradients for all sub-optimizers. 

176 

177 Args: 

178 set_to_none: Whether gradients are reset to ``None``. 

179 """ 

180 for opt in self.chained_optimizers: 

181 opt.zero_grad(set_to_none=set_to_none) 

182 

183 @property 

184 def optimizer(self) -> torch.optim.Optimizer: 

185 """Access underlying optimizer when only one optimizer included for backward compatibility.""" 

186 if len(self.chained_optimizers) != 1: 

187 raise ValueError("ChainedOptimizer has more than one optimizer when accessing self.optimizer") 

188 return self.chained_optimizers[0] 

189 

190 @property 

191 def defaults(self) -> Dict[str, Any]: 

192 """Return the defaults of the first sub-optimizer.""" 

193 return self.chained_optimizers[0].defaults 

194 

195 def state_dict(self) -> Dict[str, Any]: 

196 """Return state dicts with owning checkpoint tensor copies. 

197 

198 In HSDP, optimizer state is only populated on the owner rank for replicated 

199 parameters. This method first broadcasts state to all replicate-group peers, 

200 then clones exported values while preserving every DTensor layout. 

201 """ 

202 prepared = [] 

203 try: 

204 # Swap optimizers must restore released device storage before the 

205 # wrapped HSDP implementation reads its state tensors. 

206 for opt in self.chained_optimizers: 

207 prepare = getattr(opt, "_prepare_swap_checkpoint", None) 

208 if callable(prepare): 

209 prepared.append(opt) 

210 prepare() 

211 

212 # Ensure all ranks have consistent optimizer state before snapshotting. 

213 for opt in self.chained_optimizers: 

214 if hasattr(opt, "_broadcast_state_fused_for_ckpt"): 

215 opt._broadcast_state_fused_for_ckpt() # pylint: disable=protected-access 

216 

217 merged: Dict[str, Any] = {} 

218 for name, optimizer in self.optimizers_dict.items(): 

219 with self._optimizer_parameter_view(): 

220 sd = get_optimizer_state_dict( 

221 self.model, 

222 optimizer, 

223 options=StateDictOptions(flatten_optimizer_state_dict=self.flatten) 

224 ) 

225 if optimizer in prepared: 

226 # The swap lifecycle releases device storage once the 

227 # checkpoint window closes, so the exported values must own 

228 # their memory instead of aliasing the slot tensors. 

229 self._localize_state_dict_tensors(sd) 

230 overlap = set(merged.keys()) & set(sd.keys()) 

231 if overlap: 

232 raise KeyError( 

233 f"Key clash detected while merging state dict for optimizer '{name}': " 

234 f"{', '.join(sorted(overlap))}" 

235 ) 

236 merged.update(sd) 

237 

238 return merged 

239 finally: 

240 for opt in reversed(prepared): 

241 cleanup = getattr(opt, "cleanup_synced_state", None) 

242 if callable(cleanup): 

243 cleanup() 

244 continue 

245 finish = getattr(opt, "_finish_swap_checkpoint", None) 

246 if callable(finish): 

247 finish() 

248 

249 @staticmethod 

250 def _localize_state_dict_tensors(state_dict: Dict[str, Any]) -> None: 

251 """Replace exported tensors with owning copies that retain DTensor layouts. 

252 

253 Swap-managed optimizer state lives in CPU mirrors while its device 

254 storage is released, so a checkpoint entry must not alias a tensor whose 

255 storage the swap runtime is about to free. 

256 """ 

257 def localize(value: Any) -> Any: 

258 """Clone tensors in one nested checkpoint value. 

259 

260 Args: 

261 value: Checkpoint value or nested container to localize. 

262 

263 Returns: 

264 The value with owning tensor copies and unchanged DTensor metadata. 

265 """ 

266 if isinstance(value, torch.Tensor): 

267 local_copy = to_local_if_dtensor(value).detach().cpu().clone() 

268 return _restore_dtensor_metadata(value, local_copy) 

269 if isinstance(value, dict): 

270 return {key: localize(item) for key, item in value.items()} 

271 if isinstance(value, list): 

272 return [localize(item) for item in value] 

273 if isinstance(value, tuple): 

274 return tuple(localize(item) for item in value) 

275 return value 

276 

277 for key, value in list(state_dict.items()): 

278 if key == "param_groups": 

279 continue 

280 state_dict[key] = localize(value) 

281 

282 def _get_param_groups(self) -> List[Dict[str, Any]]: 

283 """Get param_groups aggregated over underlying optimizers.""" 

284 param_groups: List[Dict[str, Any]] = [] 

285 for optimizer in self.chained_optimizers: 

286 param_groups += optimizer.param_groups 

287 return param_groups 

288 

289 def _set_param_groups(self, new_param_groups: List[Dict[str, Any]]) -> None: 

290 """Set param_groups distributed across underlying optimizers.""" 

291 if not isinstance(new_param_groups, list): 

292 raise TypeError("new_param_groups should be a list") 

293 if len(new_param_groups) != len(self.param_groups): 

294 raise ValueError("The size of new_param_groups must be equal to origin param_groups") 

295 

296 start = 0 

297 for optimizer in self.chained_optimizers: 

298 group_len = len(optimizer.param_groups) 

299 optimizer.param_groups = new_param_groups[start: start + group_len] 

300 start += group_len 

301 

302 param_groups = property(_get_param_groups, _set_param_groups) 

303 

304 def load_state_dict(self, state_dict: Dict[str, Any]) -> None: 

305 """Load optimizer state dicts and synchronize steps. 

306 

307 Args: 

308 state_dict: FQN-keyed optimizer state to restore. 

309 """ 

310 for optimizer in self.chained_optimizers: 

311 load_context = getattr(optimizer, "_swap_checkpoint_load_context", None) 

312 context = load_context(state_dict, self.flatten) if callable(load_context) else nullcontext() 

313 with self._optimizer_parameter_view(), context: 

314 set_optimizer_state_dict( 

315 self.model, 

316 optimizer, 

317 optim_state_dict=state_dict, 

318 options=StateDictOptions(flatten_optimizer_state_dict=self.flatten), 

319 ) 

320 

321 self._synchronize_steps() 

322 

323 def _synchronize_steps(self) -> Optional[int]: 

324 """Synchronize the step of all optimizers. 

325 

326 TE FusedAdam will not accumulate "step" for empty param groups, 

327 so we need to align the step across param groups before saving and after loading. 

328 """ 

329 steps = [] 

330 for optimizer in self.chained_optimizers: 

331 actual_opt = getattr(optimizer, 'optimizer', optimizer) 

332 for param_group in actual_opt.param_groups: 

333 if len(param_group['params']) > 0 and 'step' in param_group: 

334 steps.append(param_group['step']) 

335 

336 unique_steps = list(set(steps)) 

337 if len(unique_steps) > 1: 

338 raise ValueError(f"steps should <= 1, but got {unique_steps}") 

339 

340 step = unique_steps[0] if len(unique_steps) == 1 else None 

341 

342 for optimizer in self.chained_optimizers: 

343 actual_opt = getattr(optimizer, 'optimizer', optimizer) 

344 for param_group in actual_opt.param_groups: 

345 param_group['step'] = step 

346 

347 return step 

348 

349 

350class BaseDistributedOptimizer(torch.optim.Optimizer): 

351 """Base class for distributed optimizers with HSDP-aware topology communication. 

352 

353 Provides fused hierarchical broadcast for parameters and optimizer states. 

354 """ 

355 

356 def __init__( 

357 self, 

358 params: Any, 

359 defaults: Dict[str, Any], 

360 is_muon: bool, 

361 hsdp_replica_count: Optional[int] = None, 

362 ) -> None: 

363 """Initialize topology-aware optimizer state and parameter groups. 

364 

365 Args: 

366 params: Parameter groups optimized by this instance. 

367 defaults: Default hyperparameters applied to every group. 

368 is_muon: Whether parameters use the Muon update path. 

369 hsdp_replica_count: Optional optimizer-state replica group size. 

370 """ 

371 super().__init__(params, defaults) 

372 self.is_muon = is_muon 

373 self.hsdp_replica_count = hsdp_replica_count 

374 self._param_to_broadcast_info: Dict[ 

375 torch.nn.Parameter, Tuple[Tuple[int, ...], Tuple[dist.ProcessGroup, ...]] 

376 ] = {} 

377 

378 # Cache: (parent_ranks_tuple, sub_size) -> {sub_idx: sub_pg} 

379 self._split_sub_pg_cache: Dict[Tuple[Tuple[int, ...], int], Dict[int, dist.ProcessGroup]] = {} 

380 

381 def _group_dtensor_by_mesh(self): 

382 """Group DTensor parameters by mesh topology and shard layout.""" 

383 self._hsdp_grouping: Dict[int, Tuple[List, List]] = {} 

384 for group_key, group in enumerate(self.param_groups): 

385 no_comm_params, hsdp_groups = group_parameters_for_hsdp(group["params"]) 

386 self._hsdp_grouping[group_key] = (no_comm_params, hsdp_groups) 

387 

388 def reset_optimizer_parameters(self) -> None: 

389 """Rebuild subclass metadata after optimizer parameters are replaced.""" 

390 raise NotImplementedError 

391 

392 def _auto_deduce_replica_count(self) -> Optional[int]: 

393 """Deduce hsdp_replica_count based on cluster topology. 

394 

395 - Intra-node PGs: Full dedup (no split), high bandwidth makes broadcast cheap. 

396 - Inter-node PGs: Split at node boundaries to restrict communication domains 

397 within a single node, bypassing cross-node bottlenecks. 

398 """ 

399 devices_per_node = get_device_count() 

400 

401 dedup_per_dim: Dict[int, int] = {} 

402 needs_split = False 

403 

404 # group_key, (no_comm_params, hsdp_groups) 

405 for _, (_, hsdp_groups) in self._hsdp_grouping.items(): 

406 for hsdp_group in hsdp_groups: 

407 for dim_idx, pg in enumerate(hsdp_group.replicate_pgs): 

408 if pg is None: 

409 continue 

410 pg_size = dist.get_world_size(pg) 

411 if pg_size <= 1: 

412 continue 

413 

414 if pg_size > devices_per_node: 

415 # Inter-node: Find largest divisor safe for node boundary 

416 dedup = min(pg_size, devices_per_node) 

417 while pg_size % dedup != 0: 

418 dedup -= 1 

419 needs_split = True 

420 else: 

421 # origin Inter-node 

422 dedup = pg_size 

423 

424 # Enforce conservative (smallest) dedup across shared mesh axes 

425 if dim_idx not in dedup_per_dim: 

426 dedup_per_dim[dim_idx] = dedup 

427 else: 

428 dedup_per_dim[dim_idx] = min(dedup_per_dim[dim_idx], dedup) 

429 

430 if not needs_split: 

431 return None 

432 

433 sorted_dedups = [dedup_per_dim[k] for k in sorted(dedup_per_dim.keys())] 

434 return min(sorted_dedups) 

435 

436 def _split_replicate_groups(self) -> None: 

437 """Split replicate ProcessGroups into smaller sub-groups based on hsdp_replica_count.""" 

438 if self.hsdp_replica_count is None: 

439 return 

440 

441 # Argument validation 

442 if not isinstance(self.hsdp_replica_count, int): 

443 raise TypeError(f"Unsupported hsdp_replica_count type: {type(self.hsdp_replica_count).__name__}") 

444 if self.hsdp_replica_count <= 0: 

445 raise ValueError(f"hsdp_replica_count must be positive, got {self.hsdp_replica_count}") 

446 

447 for group_key, (_, hsdp_groups) in self._hsdp_grouping.items(): 

448 for hsdp_group in hsdp_groups: 

449 new_replicate_pgs: List[dist.ProcessGroup] = [] 

450 for dim_idx, pg in enumerate(hsdp_group.replicate_pgs): 

451 if pg is None: 

452 new_replicate_pgs.append(pg) 

453 continue 

454 

455 pg_size = dist.get_world_size(pg) 

456 dedup_size = self.hsdp_replica_count 

457 

458 if pg_size <= dedup_size: 

459 new_replicate_pgs.append(pg) 

460 continue 

461 

462 if pg_size % dedup_size != 0: 

463 raise ValueError( 

464 f"hsdp_replica_count {dedup_size} must evenly divide replicate group size {pg_size}" 

465 ) 

466 

467 sub_pg = self._get_or_create_sub_pg(pg, dedup_size) 

468 new_replicate_pgs.append(sub_pg) 

469 

470 logger.info_rank0( 

471 "[HSDP Split] group_key=%s, dim_idx=%s, original_size=%s, sub_size=%s", 

472 group_key, dim_idx, pg_size, dedup_size 

473 ) 

474 

475 # replace new sub groups of hsdp groups 

476 hsdp_group.replicate_pgs = tuple(new_replicate_pgs) 

477 

478 def _get_or_create_sub_pg( 

479 self, 

480 parent_pg: dist.ProcessGroup, 

481 sub_size: int, 

482 ) -> dist.ProcessGroup: 

483 """Retrieve or collectively create a synchronized sub-ProcessGroup.""" 

484 local_parent_ranks = tuple(sorted(list(dist.get_process_group_ranks(parent_pg)))) 

485 cache_key = (local_parent_ranks, sub_size) 

486 

487 # Cache Hit Check 

488 if cache_key in self._split_sub_pg_cache: 

489 sub_pg_map = self._split_sub_pg_cache[cache_key] 

490 for sub_pg in sub_pg_map.values(): 

491 if sub_pg is not None: 

492 try: 

493 dist.get_rank(group=sub_pg) 

494 return sub_pg 

495 except RuntimeError: 

496 continue 

497 raise RuntimeError(f"Current rank not found in cached sub-groups for parent_pg={local_parent_ranks}") 

498 

499 global_rank = dist.get_rank() 

500 world_size = dist.get_world_size() 

501 

502 # Global Rendezvous via GPU all_gather_into_tensor (NCCL/HCCL fast-path). 

503 # Far more scalable than all_gather_object for large world_size. 

504 device = get_current_device() 

505 num_parent_ranks = len(local_parent_ranks) 

506 local_tensor = torch.tensor(local_parent_ranks, dtype=torch.long, device=device) 

507 gathered_tensor = torch.empty((world_size, num_parent_ranks), dtype=torch.long, device=device) 

508 dist.all_gather_into_tensor(gathered_tensor.view(-1), local_tensor) 

509 

510 # Deduplicate: each row is one rank's parent_ranks; convert to 

511 # sorted set of tuples for deterministic iteration order. 

512 gathered_cpu_list = gathered_tensor.cpu().tolist() 

513 unique_all_parent_ranks = sorted(set( 

514 tuple(row) for row in gathered_cpu_list 

515 )) 

516 

517 sub_pg_map: Dict[int, dist.ProcessGroup] = {} 

518 my_sub_pg: Optional[dist.ProcessGroup] = None 

519 

520 # Synchronized collective creation loop 

521 for parent_ranks in unique_all_parent_ranks: 

522 num_sub_groups = len(parent_ranks) // sub_size 

523 

524 for sub_idx in range(num_sub_groups): 

525 sub_ranks = parent_ranks[sub_idx * sub_size: (sub_idx + 1) * sub_size] 

526 sub_pg = dist.new_group(sub_ranks) 

527 

528 # Map results only if this parent ranks list matches the current rank's context 

529 if parent_ranks == local_parent_ranks: 

530 if global_rank in sub_ranks: 

531 my_sub_pg = sub_pg 

532 sub_pg_map[sub_idx] = sub_pg 

533 else: 

534 sub_pg_map[sub_idx] = None 

535 

536 self._split_sub_pg_cache[cache_key] = sub_pg_map 

537 

538 if my_sub_pg is None: 

539 raise RuntimeError( 

540 f"Rank {global_rank} not found in any sub-group of parent_pg " 

541 f"with ranks {local_parent_ranks} and sub_size={sub_size}" 

542 ) 

543 

544 return my_sub_pg 

545 

546 def _build_hsdp_batch( 

547 self, 

548 max_batch_numel: Optional[int] = None, 

549 ) -> None: 

550 """Split HSDP groups into memory-capped batches for compute-broadcast overlap.""" 

551 if max_batch_numel is None: 

552 broadcast_max_bytes = getattr( 

553 self, "replicate_broadcast_max_bytes", 512 * 1024 * 1024, 

554 ) 

555 hsdp_size = self.hsdp_replica_count if self.hsdp_replica_count is not None else 1 

556 max_batch_numel = broadcast_max_bytes * hsdp_size # if hsdp_size > 1 else float('inf') 

557 

558 self._hsdp_batches: Dict[int, List[Dict]] = {} 

559 

560 for group_key, (_, hsdp_groups) in self._hsdp_grouping.items(): 

561 # Sort groups by total numel descending — large groups first. 

562 sorted_groups = sorted( 

563 hsdp_groups, 

564 key=lambda g: -sum(r.param.numel() for r in g.records), 

565 ) 

566 

567 batch_groups: List[Dict] = [] 

568 

569 for hsdp_group in sorted_groups: 

570 # Sort records within group by numel descending. 

571 sorted_records = sorted( 

572 hsdp_group.records, 

573 key=lambda r: -r.param.numel(), 

574 ) 

575 

576 sub_batches: List[List[Dict]] = [] 

577 current_batch: List[Dict] = [] 

578 current_numel = 0 

579 

580 for record in sorted_records: 

581 p_numel = record.param.numel() 

582 

583 current_batch.append({ 

584 "record": record, 

585 "hsdp_group": hsdp_group, 

586 }) 

587 current_numel += p_numel 

588 

589 # Soft limit: allow the bucket to slightly exceed the cap 

590 # so that symmetric structures stay together and fragmentation is reduced. 

591 if current_numel >= max_batch_numel: 

592 sub_batches.append(current_batch) 

593 current_batch = [] 

594 current_numel = 0 

595 

596 if current_batch: 

597 sub_batches.append(current_batch) 

598 

599 # Sort sub-batches by numel descending within this group. 

600 sub_batches.sort(key=lambda b: -sum(e["record"].param.numel() for e in b)) 

601 

602 batch_groups.append({ 

603 "hsdp_group": hsdp_group, 

604 "sub_batches": sub_batches, 

605 }) 

606 

607 # log batch split info. 

608 total_sub_batches = sum(len(bg["sub_batches"]) for bg in batch_groups) 

609 logger.info_rank0( 

610 "[HSDP Batch] group_key=%s, num_hsdp_groups=%s, num_batch_groups=%s, " 

611 "total_sub_batches=%s, group_numels=%s, max_batch_numel=%s", 

612 group_key, 

613 len(hsdp_groups), 

614 len(batch_groups), 

615 total_sub_batches, 

616 [sum(bg['hsdp_group'].records_numel) if hasattr(bg['hsdp_group'], 'records_numel') else sum( 

617 r.param.numel() for r in bg['hsdp_group'].records) for bg in batch_groups], 

618 max_batch_numel 

619 ) 

620 self._hsdp_batches[group_key] = batch_groups 

621 

622 def _build_sub_batch_assignment( 

623 self, 

624 sub_batch_entries: List[Dict], 

625 hsdp_group: Any, 

626 ) -> Optional[HSDPGroupAssignment]: 

627 """Build an HSDPGroupAssignment for one sub-batch and update broadcast info.""" 

628 records = [e["record"] for e in sub_batch_entries] 

629 if not records: 

630 return None 

631 

632 device_mesh = records[0].param.device_mesh 

633 

634 replicate_group_ranks, replicate_sizes = get_multi_dim_logical_info( 

635 device_mesh, 

636 hsdp_group.comm_key.replicate_mesh_dims, 

637 ) 

638 

639 # When hsdp_replica_count is set, remap coordinates into 

640 # sub-groups: each original coord maps to (coord % sub_size) 

641 # within its sub-group, and the effective group size shrinks. 

642 if self.hsdp_replica_count is not None: 

643 replicate_group_ranks = tuple( 

644 r % self.hsdp_replica_count for r in replicate_group_ranks 

645 ) 

646 replicate_sizes = tuple( 

647 min(s, self.hsdp_replica_count) for s in replicate_sizes 

648 ) 

649 

650 # Greedy owner assignment on this sub-batch's records. 

651 owner_by_index = build_owner_by_size( 

652 records=records, 

653 replicate_sizes=replicate_sizes, 

654 ) 

655 

656 owned_records = select_owned_records( 

657 records=records, 

658 owner_by_index=owner_by_index, 

659 replicate_group_ranks=replicate_group_ranks, 

660 ) 

661 

662 is_shard_for_ns = ( 

663 hsdp_group.comm_key.has_shard_group 

664 and hsdp_group.layout_spec.is_last2d_sharded 

665 ) 

666 

667 hsdp_assign = HSDPGroupAssignment( 

668 owned_records=owned_records, 

669 all_records=records, 

670 owner_by_index=owner_by_index, 

671 replicate_group_ranks=replicate_group_ranks, 

672 replicate_sizes=replicate_sizes, 

673 replicate_pgs=hsdp_group.replicate_pgs, 

674 shard_pgs=hsdp_group.shard_pgs, 

675 is_shard=is_shard_for_ns, 

676 layout_spec=hsdp_group.layout_spec, 

677 ) 

678 logger.debug_rank0(f'[Hyper-optimizer] hsdp_assign: {hsdp_assign}') 

679 

680 # Build broadcast reverse mapping for replicated groups 

681 if hsdp_assign.is_replicated and hsdp_assign.replicate_pgs: 

682 for record in hsdp_assign.all_records: 

683 src_coord = hsdp_assign.owner_rank_coord(record) 

684 if not src_coord or any(c < 0 for c in src_coord): 

685 continue 

686 self._param_to_broadcast_info[record.param] = ( 

687 src_coord, hsdp_assign.replicate_pgs, 

688 ) 

689 

690 return hsdp_assign 

691 

692 def _build_param_broadcast_info(self) -> None: 

693 """Build per-batch HSDP assignments and param broadcast reverse mapping. 

694 

695 When ``hsdp_replica_count`` is set, replicate coordinates and sizes 

696 are remapped so that owner assignment and broadcast happen within 

697 sub-groups of size ``hsdp_replica_count`` instead of the full 

698 replicate group. 

699 

700 Sub-group broadcast already covers all ranks (each rank belongs to 

701 exactly one sub-group), so param and state broadcast use the same 

702 sub-group mapping — no separate full-group path is needed. 

703 """ 

704 self._hsdp_assignment_batches: Dict[int, Dict] = {} 

705 self._param_to_broadcast_info: Dict[ 

706 torch.nn.Parameter, Tuple[Tuple[int, ...], Tuple[dist.ProcessGroup, ...]] 

707 ] = {} 

708 

709 for group_key, batch_groups in self._hsdp_batches.items(): 

710 no_comm_params = self._hsdp_grouping[group_key][0] 

711 assignment_batch_groups = [] 

712 

713 for bg in batch_groups: 

714 hsdp_group = bg["hsdp_group"] 

715 sub_batch_assigns: List[HSDPGroupAssignment] = [] 

716 

717 for sub_batch_entries in bg["sub_batches"]: 

718 hsdp_assign = self._build_sub_batch_assignment(sub_batch_entries, hsdp_group) 

719 if hsdp_assign is not None: 

720 sub_batch_assigns.append(hsdp_assign) 

721 

722 assignment_batch_groups.append({ 

723 "hsdp_group": hsdp_group, 

724 "sub_batches": sub_batch_assigns, 

725 }) 

726 

727 self._hsdp_assignment_batches[group_key] = { 

728 "no_comm": no_comm_params, 

729 "batch_groups": assignment_batch_groups, 

730 } 

731 

732 def _broadcast_replicate_params_after_step(self) -> None: 

733 """Broadcast updated params from assigned rank to replicate-group peers.""" 

734 self._broadcast_op_fused(target="param") 

735 

736 def _broadcast_state_fused_for_ckpt(self) -> None: 

737 """Broadcast optimizer state before checkpoint save.""" 

738 self._broadcast_op_fused( 

739 target="state", 

740 state_keys=list(self._state_keys_for_checkpoint_broadcast()), 

741 ) 

742 

743 def _state_keys_for_checkpoint_broadcast(self) -> Tuple[str, ...]: 

744 """Return optimizer state fields synchronized for an HSDP checkpoint.""" 

745 return ("momentum_buffer",) if self.is_muon else ("exp_avg", "exp_avg_sq") 

746 

747 def _collect_broadcast_tensors( 

748 self, 

749 target: str, 

750 state_keys: Optional[List[str]] = None, 

751 ) -> Dict[Tuple, List[torch.Tensor]]: 

752 """Collect tensors to broadcast, grouped by (src_coord, dtype, replicate_pgs). 

753 

754 Args: 

755 target: "param" or "state". 

756 state_keys: State dict keys to broadcast when target="state". 

757 

758 Returns: 

759 Dict mapping (src_coord, dtype, replicate_pgs) to list of local tensors. 

760 """ 

761 rank_dtype_tensors = defaultdict(list) 

762 

763 for p, (src_coord, replicate_pgs) in self._param_to_broadcast_info.items(): 

764 if target == "param": 

765 local_tensor = to_local_if_dtensor(p.data) 

766 rank_dtype_tensors[(src_coord, local_tensor.dtype, replicate_pgs)].append(local_tensor) 

767 

768 elif target == "state" and state_keys: 

769 param_state = self.state.setdefault(p, {}) 

770 for key in state_keys: 

771 if key in param_state: 

772 state_tensor = param_state[key] 

773 local_tensor = to_local_if_dtensor(state_tensor) 

774 else: 

775 local_tensor = torch.empty_like(p, dtype=torch.float32) 

776 param_state[key] = local_tensor 

777 local_tensor = to_local_if_dtensor(local_tensor) 

778 

779 rank_dtype_tensors[(src_coord, local_tensor.dtype, replicate_pgs)].append(local_tensor) 

780 

781 return rank_dtype_tensors 

782 

783 @staticmethod 

784 def _compute_broadcast_batches( 

785 tensors: List[torch.Tensor], 

786 alignment_elements: int, 

787 max_broadcast_elements: int 

788 ) -> List[Tuple[List[Tuple[torch.Tensor, int, int, int]], int]]: 

789 """Split tensors into memory-capped batches with alignment padding. 

790 

791 Args: 

792 tensors: List of tensors to batch. 

793 alignment_elements: Alignment granularity in elements. 

794 max_broadcast_elements: Maximum elements per batch. 

795 

796 Returns: 

797 List of (batch_offsets, batch_total_size) tuples. 

798 Each batch_offsets entry is (tensor, offset, actual_numel, padded_numel). 

799 """ 

800 batches = [] 

801 current_batch = [] 

802 current_total_size = 0 

803 

804 for t in tensors: 

805 actual_numel = t.numel() 

806 padded_numel = ((actual_numel + alignment_elements - 1) // alignment_elements) * alignment_elements 

807 

808 if current_batch and current_total_size + padded_numel > max_broadcast_elements: 

809 batches.append((current_batch, current_total_size)) 

810 current_batch = [] 

811 current_total_size = 0 

812 

813 current_batch.append((t, current_total_size, actual_numel, padded_numel)) 

814 current_total_size += padded_numel 

815 

816 if current_batch: 

817 batches.append((current_batch, current_total_size)) 

818 

819 return batches 

820 

821 @staticmethod 

822 def _hierarchical_broadcast_buffer( 

823 batch_buffer: torch.Tensor, 

824 src_coord: Tuple[int, ...], 

825 replicate_pgs: Tuple[dist.ProcessGroup, ...], 

826 local_coord: Tuple[int, ...] 

827 ) -> None: 

828 """Broadcast a buffer dimension-by-dimension across replicate groups. 

829 

830 Args: 

831 batch_buffer: Contiguous buffer to broadcast. 

832 src_coord: Source coordinate tuple. 

833 replicate_pgs: Tuple of ProcessGroups (one per dimension). 

834 local_coord: Local rank coordinate tuple. 

835 """ 

836 

837 for dim_idx, pg in enumerate(replicate_pgs): 

838 if pg is None: 

839 continue 

840 

841 participate = True 

842 for subsequent_dim in range(dim_idx + 1, len(replicate_pgs)): 

843 if local_coord[subsequent_dim] != src_coord[subsequent_dim]: 

844 participate = False 

845 break 

846 

847 if not participate: 

848 continue 

849 

850 src_rank_in_pg = src_coord[dim_idx] 

851 global_src_rank = dist.get_global_rank(pg, src_rank_in_pg) 

852 dist.broadcast(batch_buffer, src=global_src_rank, group=pg) 

853 

854 @staticmethod 

855 def _hierarchical_broadcast_buffer_async( 

856 batch_buffer: torch.Tensor, 

857 src_coord: Tuple[int, ...], 

858 replicate_pgs: Tuple[dist.ProcessGroup, ...], 

859 local_coord: Tuple[int, ...], 

860 ) -> List[dist.Work]: 

861 """Async version of _hierarchical_broadcast_buffer. 

862 

863 Same dimension-by-dimension relay logic, but each dist.broadcast uses 

864 async_op=True. Returns a list of Work handles to wait on later. 

865 

866 Note: dimensions within a single buffer are still sequential (dim N+1 

867 depends on dim N completing), but different buffers can overlap. 

868 """ 

869 broadcast_ops: List[Tuple[dist.ProcessGroup, int]] = [] 

870 

871 for dim_idx, pg in enumerate(replicate_pgs): 

872 if pg is None: 

873 continue 

874 

875 participate = True 

876 for subsequent_dim in range(dim_idx + 1, len(replicate_pgs)): 

877 if local_coord[subsequent_dim] != src_coord[subsequent_dim]: 

878 participate = False 

879 break 

880 

881 if not participate: 

882 continue 

883 

884 src_rank_in_pg = src_coord[dim_idx] 

885 global_src_rank = dist.get_global_rank(pg, src_rank_in_pg) 

886 broadcast_ops.append((pg, global_src_rank)) 

887 

888 handles: List[dist.Work] = [] 

889 for op_idx, (pg, global_src_rank) in enumerate(broadcast_ops): 

890 handle = dist.broadcast( 

891 batch_buffer, src=global_src_rank, group=pg, async_op=True, 

892 ) 

893 if op_idx + 1 < len(broadcast_ops): 

894 # The next mesh dimension relays the data received here. 

895 # It must not read the shared buffer until this hop finishes. 

896 handle.wait() 

897 else: 

898 # Only the final hop may overlap with subsequent computation. 

899 handles.append(handle) 

900 

901 return handles 

902 

903 def _broadcast_op_fused( 

904 self, 

905 target: str, 

906 state_keys: Optional[List[str]] = None, 

907 ) -> None: 

908 """Fused hierarchical broadcast for param or state across replicate groups. 

909 

910 Groups tensors by (src_coord, dtype, replicate_pgs), packs into contiguous device buffers 

911 with 512-byte alignment, and broadcasts dimension-by-dimension in memory-capped batches. 

912 State tensors may be CPU swap mirrors; only the current fused buffer is then resident on 

913 the accelerator while packing and unpacking each batch. 

914 

915 Args: 

916 target: "param" or "state". 

917 state_keys: State dict keys to broadcast when target="state". 

918 """ 

919 device = get_current_device() 

920 

921 alignment = 512 # bytes 

922 rank_dtype_tensors = self._collect_broadcast_tensors(target, state_keys) 

923 

924 for (src_coord, dtype, replicate_pgs), tensors in rank_dtype_tensors.items(): 

925 if not tensors: 

926 continue 

927 

928 local_coord = tuple( 

929 dist.get_rank(group=pg) if pg is not None else -1 

930 for pg in replicate_pgs 

931 ) 

932 

933 element_size = torch.empty(0, dtype=dtype, device=device).element_size() 

934 alignment_elements = alignment // element_size 

935 max_broadcast_bytes = getattr(self, "replicate_broadcast_max_bytes", 512 * 1024 * 1024) 

936 max_broadcast_elements = max_broadcast_bytes // element_size 

937 max_broadcast_elements = max( 

938 alignment_elements, 

939 (max_broadcast_elements // alignment_elements) * alignment_elements, 

940 ) 

941 

942 batches = self._compute_broadcast_batches(tensors, alignment_elements, max_broadcast_elements) 

943 if not batches: 

944 continue 

945 

946 max_batch_size = max(batch_size for _, batch_size in batches) 

947 buffer = torch.empty(max_batch_size, dtype=dtype, device=device) 

948 

949 for batch_tensor_offsets, batch_total_size in batches: 

950 batch_buffer = buffer[:batch_total_size] 

951 

952 # Pack: owner rank 

953 if local_coord == src_coord: 

954 for t, offset, actual_numel, padded_numel in batch_tensor_offsets: 

955 batch_buffer[offset:offset + actual_numel].copy_(t.view(-1)) 

956 if padded_numel > actual_numel: 

957 batch_buffer[offset + actual_numel:offset + padded_numel].zero_() 

958 

959 # Hierarchical Broadcast 

960 self._hierarchical_broadcast_buffer(batch_buffer, src_coord, replicate_pgs, local_coord) 

961 

962 # Unpack: copy buffer data back to individual tensors 

963 for t, offset, actual_numel, _ in batch_tensor_offsets: 

964 t.view(-1).copy_(batch_buffer[offset:offset + actual_numel]) 

965 

966 buffer.untyped_storage().resize_(0) 

967 del buffer 

968 

969 def cleanup_synced_state(self) -> None: 

970 """Release optimizer state for non-owned params after checkpoint saving. 

971 

972 In HSDP mode, _broadcast_state_fused_for_ckpt broadcasts state to all 

973 ranks so every rank can save a complete checkpoint. This method removes 

974 the non-owned state to restore per-rank memory savings. Call once all 

975 checkpoint consumers have finished reading the live state, either after 

976 persistence or after creating an independent snapshot for async I/O. 

977 """ 

978 params_to_remove = [] 

979 for assignment_info in self._hsdp_assignment_batches.values(): 

980 for bg in assignment_info["batch_groups"]: 

981 for hsdp_info in bg["sub_batches"]: 

982 if not hsdp_info.is_replicated: 

983 continue 

984 for record in hsdp_info.all_records: 

985 # Records without an owner are never broadcast, so their state is not a replica copy. 

986 if record.index not in hsdp_info.owner_by_index: 

987 continue 

988 if not hsdp_info.is_owned(record) and record.param in self.state: 

989 params_to_remove.append(record.param) 

990 

991 for p in params_to_remove: 

992 self.state[p].clear() 

993 del self.state[p] 

994 

995 empty_accelerator_cache() 

996 

997 

998class AsyncReplicateBroadcaster: 

999 """Incremental async replicate broadcast with HSDP-group-level flushing. 

1000 

1001 Designed to overlap HSDP replicate-group broadcasts with Muon NS 

1002 computation. After all sub_batches for an HSDP group are done, the 

1003 caller invokes flush_group(hsdp_assign) to issue an async hierarchical 

1004 broadcast — overlapping with the next HSDP group's NS computation. 

1005 

1006 Flush is driven by the caller at HSDP-group boundaries (not by a 

1007 threshold) so that all ranks issue the same collective operations in 

1008 the same order, which is required by NCCL/HCCL. 

1009 

1010 Usage:: 

1011 

1012 broadcaster = AsyncReplicateBroadcaster(optimizer) 

1013 for hsdp_assign in hsdp_assignments: 

1014 # ... NS compute + apply for this group ... 

1015 broadcaster.flush_group(hsdp_assign) 

1016 broadcaster.wait_all() 

1017 """ 

1018 

1019 def __init__( 

1020 self, 

1021 optimizer: BaseDistributedOptimizer, 

1022 ) -> None: 

1023 """Bind the distributed optimizer whose broadcasts are batched. 

1024 

1025 Args: 

1026 optimizer: Distributed optimizer owning the replica assignments. 

1027 """ 

1028 self._optimizer = optimizer 

1029 

1030 # Inflight async broadcasts: list of (buffer, batch_offsets, handles) 

1031 # Buffer must stay alive until handles are waited on. 

1032 self._inflight: List[ 

1033 Tuple[ 

1034 torch.Tensor, 

1035 List[Tuple[torch.Tensor, int, int, int]], 

1036 List[dist.Work], 

1037 ] 

1038 ] = [] 

1039 # Allow one batch of replicate communication to overlap with the 

1040 # next batch's compute, but avoid unbounded inflight buffer growth. 

1041 self._max_inflight_batches = 1 

1042 

1043 def flush_group( 

1044 self, 

1045 hsdp_assign: Any, 

1046 records: Optional[List] = None, 

1047 ) -> None: 

1048 """Flush params belonging to the given HSDP group. 

1049 

1050 Can be called per sub_batch (passing only the records in that 

1051 sub_batch) so that the async replicate broadcast overlaps with 

1052 the next sub_batch's NS computation. When *records* is None, 

1053 all records in the group are flushed (backward compatible). 

1054 

1055 All ranks must call this at the same point in the execution flow 

1056 to ensure collective communication consistency. 

1057 """ 

1058 if not hsdp_assign.is_replicated or not hsdp_assign.replicate_pgs: 

1059 return 

1060 

1061 flush_records = records if records is not None else hsdp_assign.all_records 

1062 

1063 # Collect local tensors for the given records, grouped by 

1064 # (src_coord, dtype, replicate_pgs) — same logic as 

1065 # _collect_broadcast_tensors but scoped to the provided records. 

1066 rank_dtype_tensors: Dict[ 

1067 Tuple[Tuple[int, ...], torch.dtype, Tuple[dist.ProcessGroup, ...]], 

1068 List[torch.Tensor], 

1069 ] = defaultdict(list) 

1070 

1071 for record in flush_records: 

1072 src_coord = hsdp_assign.owner_rank_coord(record) 

1073 if not src_coord or any(c < 0 for c in src_coord): 

1074 continue 

1075 local_tensor = to_local_if_dtensor(record.param.data) 

1076 key = (src_coord, local_tensor.dtype, hsdp_assign.replicate_pgs) 

1077 rank_dtype_tensors[key].append(local_tensor) 

1078 

1079 for key, tensors in rank_dtype_tensors.items(): 

1080 if tensors: 

1081 self._flush_key(key, tensors, async_op=True) 

1082 

1083 while len(self._inflight) > self._max_inflight_batches: 

1084 self._wait_and_release_oldest() 

1085 

1086 def _flush_key( 

1087 self, 

1088 key: Tuple[Tuple[int, ...], torch.dtype, Tuple[dist.ProcessGroup, ...]], 

1089 tensors: List[torch.Tensor], 

1090 async_op: bool = True, 

1091 ) -> None: 

1092 """Pack and broadcast tensors for one broadcast key.""" 

1093 device = get_current_device() 

1094 src_coord, dtype, replicate_pgs = key 

1095 alignment = 512 # bytes 

1096 

1097 local_coord = tuple( 

1098 dist.get_rank(group=pg) if pg is not None else -1 

1099 for pg in replicate_pgs 

1100 ) 

1101 

1102 element_size = torch.empty(0, dtype=dtype, device=device).element_size() 

1103 alignment_elements = alignment // element_size 

1104 max_broadcast_bytes = getattr( 

1105 self._optimizer, "replicate_broadcast_max_bytes", 512 * 1024 * 1024, 

1106 ) 

1107 max_broadcast_elements = max_broadcast_bytes // element_size 

1108 max_broadcast_elements = max( 

1109 alignment_elements, 

1110 (max_broadcast_elements // alignment_elements) * alignment_elements, 

1111 ) 

1112 

1113 # pylint: disable=protected-access 

1114 batches = BaseDistributedOptimizer._compute_broadcast_batches( 

1115 tensors, alignment_elements, max_broadcast_elements, 

1116 ) 

1117 if not batches: 

1118 return 

1119 

1120 buffer: torch.Tensor 

1121 if not async_op: 

1122 max_batch_size = max(batch_size for _, batch_size in batches) 

1123 buffer = torch.empty(max_batch_size, dtype=dtype, device=device) 

1124 

1125 for batch_tensor_offsets, batch_total_size in batches: 

1126 if async_op: 

1127 batch_buffer = torch.empty(batch_total_size, dtype=dtype, device=device) 

1128 else: 

1129 batch_buffer = buffer[:batch_total_size] 

1130 

1131 # Pack: owner rank 

1132 if local_coord == src_coord: 

1133 for t, offset, actual_numel, padded_numel in batch_tensor_offsets: 

1134 batch_buffer[offset:offset + actual_numel].copy_(t.view(-1)) 

1135 if padded_numel > actual_numel: 

1136 batch_buffer[offset + actual_numel:offset + padded_numel].zero_() 

1137 

1138 if async_op: 

1139 handles = BaseDistributedOptimizer._hierarchical_broadcast_buffer_async( 

1140 batch_buffer, src_coord, replicate_pgs, local_coord, 

1141 ) 

1142 if handles: 

1143 # Pin buffer + offsets until wait_all unpacks them. 

1144 self._inflight.append((batch_buffer, batch_tensor_offsets, handles)) 

1145 else: 

1146 # No async work was enqueued on this rank, so unpack and free now. 

1147 for t, offset, actual_numel, _ in batch_tensor_offsets: 

1148 t.view(-1).copy_(batch_buffer[offset:offset + actual_numel]) 

1149 batch_buffer.untyped_storage().resize_(0) 

1150 del batch_buffer 

1151 else: 

1152 BaseDistributedOptimizer._hierarchical_broadcast_buffer( 

1153 batch_buffer, src_coord, replicate_pgs, local_coord, 

1154 ) 

1155 # Unpack immediately for sync path 

1156 for t, offset, actual_numel, _ in batch_tensor_offsets: 

1157 t.view(-1).copy_(batch_buffer[offset:offset + actual_numel]) 

1158 

1159 if not async_op: 

1160 # Sync path: buffer can be freed immediately 

1161 buffer.untyped_storage().resize_(0) 

1162 del buffer 

1163 

1164 def _wait_and_release_oldest(self) -> None: 

1165 """Wait, unpack, and release the oldest inflight async batch.""" 

1166 batch_buffer, batch_tensor_offsets, handles = self._inflight.pop(0) 

1167 for handle in handles: 

1168 handle.wait() 

1169 

1170 # Unpack after the async broadcast completes. 

1171 for t, offset, actual_numel, _ in batch_tensor_offsets: 

1172 t.view(-1).copy_(batch_buffer[offset:offset + actual_numel]) 

1173 

1174 batch_buffer.untyped_storage().resize_(0) 

1175 del batch_buffer 

1176 

1177 def wait_all(self) -> None: 

1178 """Wait for all inflight async broadcasts and unpack results.""" 

1179 while self._inflight: 

1180 self._wait_and_release_oldest()