Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / muon.py: 70%

626 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15 

16"""Muon optimizer with HSDP shard-group-aware communication.""" 

17 

18import math 

19from collections.abc import Callable, Iterable 

20from collections import defaultdict 

21from dataclasses import dataclass 

22import logging 

23from typing import Any, Dict, List, Optional, Sequence, Tuple, Union 

24 

25import torch 

26import torch.distributed as dist 

27 

28from hyper_parallel.core.optimizer.optimizer import AsyncReplicateBroadcaster, BaseDistributedOptimizer 

29from hyper_parallel.core.optimizer.dtensor_compat import to_local_if_dtensor 

30from hyper_parallel.core.optimizer.sharding_category import ( 

31 HSDPGroupAssignment, 

32 ParamShardMeta, 

33 build_owner_by_size, 

34) 

35from hyper_parallel.core.optimizer.muon_shard import ( 

36 _debug_param_shard_metadata, 

37 build_pad_ns_inputs, 

38 build_param_shard_metadata_for_group, 

39 chunk_update_by_layout, 

40 fused_allgather_dtensor_params, 

41) 

42 

43logger = logging.getLogger(__name__) 

44 

45# Legacy: single-coefficient quintic NS (Keller Jordan / Moonlight). 

46# Same (a, b, c) applied every step; typically 5 steps. 

47_NS_LEGACY_COEFFS: Tuple[float, float, float] = (3.4445, -4.7750, 2.0315) 

48 

49# Asym5: 5-step asymmetric NS (dion). Each step uses different coefficients. 

50_NS_ASYM5_COEFFS: Tuple[Tuple[float, float, float], ...] = ( 

51 (4.0848, -6.8946, 2.9270), 

52 (3.9505, -6.3029, 2.6377), 

53 (3.7418, -5.5913, 2.3037), 

54 (2.8769, -3.1427, 1.2046), 

55 (2.8366, -3.0525, 1.2012), 

56) 

57 

58 

59@dataclass 

60class NSInputTransform: 

61 """Prepared NS tensors and the callback that restores their updates.""" 

62 

63 tensors: List[torch.Tensor] 

64 restore: Callable[[List[torch.Tensor], torch.Tensor], None] 

65 

66 

67@dataclass(frozen=True) 

68class MuonPostUpdateContext: 

69 """Context supplied to an optional parameter post-update callback.""" 

70 

71 param_fqn: Optional[str] 

72 logical_shape: Tuple[int, ...] 

73 lr: float 

74 weight_decay: float 

75 step: int 

76 

77 

78_MUON_ADVANCED_DEFAULTS: Dict[str, Any] = { 

79 "ns_coefficients": None, 

80 "ns_epsilon": 1e-10, 

81 "zeropower_fn": None, 

82 "momentum_update_fn": None, 

83 "reshape_fn": None, 

84 "ns_transform_fn": None, 

85 "post_update_fn": None, 

86 "zero_rms_scale_mode": "zero", 

87 "apply_lr_in_update": False, 

88} 

89 

90 

91@dataclass(frozen=True) 

92class _MuonAdvancedOptions: 

93 """Validated low-frequency Muon options.""" 

94 

95 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]] 

96 ns_epsilon: float 

97 zeropower_fn: Optional[Callable[..., torch.Tensor]] 

98 momentum_update_fn: Optional[Callable[..., torch.Tensor]] 

99 reshape_fn: Optional[Callable[..., Any]] 

100 ns_transform_fn: Optional[Callable[..., Optional[NSInputTransform]]] 

101 post_update_fn: Optional[Callable[..., None]] 

102 zero_rms_scale_mode: str 

103 apply_lr_in_update: bool 

104 

105 

106def _resolve_muon_advanced_options(options: Dict[str, Any]) -> _MuonAdvancedOptions: 

107 """Validate and fill low-frequency Muon keyword options.""" 

108 unknown = options.keys() - _MUON_ADVANCED_DEFAULTS.keys() 

109 if unknown: 

110 raise TypeError(f"Muon got unexpected keyword arguments: {', '.join(sorted(unknown))}") 

111 resolved = dict(_MUON_ADVANCED_DEFAULTS) 

112 resolved.update(options) 

113 return _MuonAdvancedOptions(**resolved) 

114 

115 

116_TransformNSState = Dict[str, Any] 

117 

118 

119def _resolve_ns_coefficients( 

120 steps: int, 

121 ns_variant: str = "asym5", 

122 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]] = None, 

123) -> Tuple[Tuple[float, float, float], ...]: 

124 """Resolve the coefficient prefix shared by native NS and backend construction.""" 

125 if ns_variant == "legacy": 

126 return (_NS_LEGACY_COEFFS,) * steps 

127 if ns_variant == "asym5": 

128 return _NS_ASYM5_COEFFS[:steps] 

129 if ns_variant == "custom": 

130 if ns_coefficients is None: 

131 raise ValueError("ns_coefficients must be provided when ns_variant='custom'") 

132 if steps > len(ns_coefficients): 

133 raise ValueError( 

134 f"ns_steps={steps} exceeds the {len(ns_coefficients)} provided coefficient groups" 

135 ) 

136 return tuple(tuple(row) for row in ns_coefficients[:steps]) 

137 raise ValueError(f"ns_variant must be 'legacy', 'asym5', or 'custom', got {ns_variant!r}") 

138 

139 

140def zeropower_via_newtonschulz5( 

141 ns_inputs: torch.Tensor, 

142 steps: int, 

143 ns_variant: str = "asym5", 

144 epsilon: float = 1e-10, 

145 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]] = None, 

146) -> torch.Tensor: 

147 """Newton-Schulz orthogonalization with preallocated matmul buffers. 

148 

149 Args: 

150 ns_inputs: Matrix or matrix batch to orthogonalize. 

151 steps: Requested iteration count; asym5 uses at most five iterations. 

152 ns_variant: Built-in legacy/asym5 schedule or an explicit custom schedule. 

153 epsilon: Normalization epsilon added to the input norm. 

154 ns_coefficients: Coefficient triples required by the custom variant. 

155 

156 Returns: 

157 Orthogonalized updates with the input shape, dtype and device. 

158 """ 

159 mat_x = ns_inputs 

160 transposed = ns_inputs.size(-2) > ns_inputs.size(-1) 

161 if transposed: 

162 mat_x = mat_x.mT 

163 

164 # Normalize input before Newton-Schulz iteration. 

165 mat_x = mat_x / (mat_x.norm(dim=(-2, -1), keepdim=True) + epsilon) 

166 

167 n_size = mat_x.size(-2) 

168 buf_a = torch.empty(mat_x.shape[:-2] + (n_size, n_size), dtype=mat_x.dtype, device=mat_x.device) 

169 buf_b = torch.empty_like(buf_a) 

170 buf_c = torch.empty(mat_x.shape, dtype=mat_x.dtype, device=mat_x.device) 

171 

172 step_coeffs = _resolve_ns_coefficients(steps, ns_variant, ns_coefficients) 

173 

174 for coeff_a, coeff_b, coeff_c in step_coeffs: 

175 torch.matmul(mat_x, mat_x.mT, out=buf_a) 

176 torch.matmul(buf_a, buf_a, out=buf_b) 

177 buf_a.mul_(coeff_b) 

178 buf_b.mul_(coeff_c) 

179 buf_a.add_(buf_b) 

180 torch.matmul(buf_a, mat_x, out=buf_c) 

181 mat_x.mul_(coeff_a).add_(buf_c) 

182 

183 del buf_a, buf_b, buf_c 

184 

185 if transposed: 

186 mat_x = mat_x.mT 

187 

188 return mat_x 

189 

190 

191def compute_muon_slice_scale( 

192 slice_tensor: torch.Tensor, 

193 matched_adamw_rms: float, 

194 zero_rms_scale_mode: str = "zero", 

195) -> float: 

196 """Compute Muon scale from the logical matrix dims of a reshaped slice.""" 

197 if not matched_adamw_rms: 

198 return 1.0 if zero_rms_scale_mode == "use_lr" else 0.0 

199 shape = tuple(slice_tensor.shape) 

200 if len(shape) == 3 and shape[1] == 1: 

201 logical_dims = (shape[0], shape[2]) 

202 else: 

203 logical_dims = shape[-2:] 

204 return math.sqrt(max(logical_dims)) * matched_adamw_rms 

205 

206 

207class Muon(BaseDistributedOptimizer): 

208 """Muon optimizer with HSDP shard-group-aware Newton-Schulz orthogonalization. 

209 

210 Implements the Muon optimizer which uses Newton-Schulz iteration for matrix 

211 orthogonalization of gradient updates, with HSDP-aware communication for 

212 sharded parameters. 

213 """ 

214 

215 ADDITIONAL_CONFIG_KEYS = set(_MUON_ADVANCED_DEFAULTS) 

216 

217 def __init__( 

218 self, 

219 params: Any, 

220 lr: float = 2e-2, 

221 weight_decay: float = 0.1, 

222 matched_adamw_rms: float = 0.2, 

223 momentum: Union[float, List[float], Tuple[float, ...]] = 0.95, 

224 nesterov: bool = True, 

225 ns_steps: int = 5, 

226 ns_variant: str = "asym5", 

227 hsdp_replica_count: Optional[Union[int, Tuple[int, ...]]] = None, 

228 momentum_mode: str = "legacy", 

229 **advanced_options: Any, 

230 ) -> None: 

231 """Initialize Muon and build parameter-identity runtime caches. 

232 

233 Args: 

234 params: Parameters or parameter groups optimized by Muon. 

235 lr: Learning rate. 

236 weight_decay: Decoupled weight-decay coefficient. 

237 matched_adamw_rms: RMS scale matched to an AdamW update. 

238 momentum: Momentum coefficient or coefficient pair. 

239 nesterov: Whether to use Nesterov momentum. 

240 ns_steps: Newton-Schulz iteration count. 

241 ns_variant: Newton-Schulz coefficient variant. 

242 ns_coefficients: Optional custom Newton-Schulz coefficients. 

243 ns_epsilon: Numerical stability term used during normalization. 

244 zeropower_fn: Optional ``fn(local_tensor, steps=ns_steps)`` override. 

245 Return a writable tensor with the same shape, dtype and device. 

246 The callback owns its coefficient schedule and epsilon; these 

247 group options are not forwarded. Recreate matching callback 

248 configuration when restoring an optimizer checkpoint. 

249 momentum_update_fn: Optional momentum update implementation override. 

250 reshape_fn: Optional logical-matrix reshape function. 

251 ns_transform_fn: Optional Newton-Schulz input transformation. 

252 post_update_fn: Optional callback applied after a parameter update. 

253 zero_rms_scale_mode: Scaling behavior when matched AdamW RMS is zero. 

254 apply_lr_in_update: Whether the update callback applies the learning rate. 

255 hsdp_replica_count: Optional optimizer-state replica group size. 

256 momentum_mode: Momentum formula, either "legacy" or "vanilla". 

257 Both modes support distinct update and buffer coefficients. 

258 """ 

259 advanced = _resolve_muon_advanced_options(advanced_options) 

260 if momentum_mode not in ("legacy", "vanilla"): 

261 raise ValueError( 

262 f"momentum_mode must be 'legacy' or 'vanilla', got {momentum_mode!r}" 

263 ) 

264 if ns_variant not in ("legacy", "asym5", "custom"): 

265 raise ValueError( 

266 f"ns_variant must be 'legacy', 'asym5', or 'custom', got {ns_variant!r}" 

267 ) 

268 if advanced.ns_epsilon < 0.0 or not math.isfinite(advanced.ns_epsilon): 

269 raise ValueError(f"ns_epsilon must be a finite non-negative value, got {advanced.ns_epsilon}") 

270 if advanced.zero_rms_scale_mode not in ("zero", "use_lr"): 

271 raise ValueError( 

272 "zero_rms_scale_mode must be 'zero' or 'use_lr', " 

273 f"got {advanced.zero_rms_scale_mode!r}" 

274 ) 

275 normalized_coefficients = self._validate_ns_coefficients( 

276 ns_variant, ns_steps, advanced.ns_coefficients 

277 ) 

278 if not isinstance(momentum, (list, tuple)): 

279 momentum = [momentum] 

280 if len(momentum) == 1: 

281 momentum = (momentum[0], momentum[0]) 

282 defaults = { 

283 "lr": lr, 

284 "weight_decay": weight_decay, 

285 "matched_adamw_rms": matched_adamw_rms, 

286 "momentum": momentum, 

287 "nesterov": nesterov, 

288 "ns_steps": ns_steps, 

289 "ns_variant": ns_variant, 

290 "ns_coefficients": normalized_coefficients, 

291 "ns_epsilon": advanced.ns_epsilon, 

292 "zero_rms_scale_mode": advanced.zero_rms_scale_mode, 

293 "apply_lr_in_update": advanced.apply_lr_in_update, 

294 } 

295 super().__init__(params, defaults, is_muon=True, hsdp_replica_count=hsdp_replica_count) 

296 self.reshape_fn = advanced.reshape_fn 

297 self.zeropower_fn = advanced.zeropower_fn 

298 self.momentum_update_fn = advanced.momentum_update_fn 

299 self._update_muon_momentum_mode = momentum_mode 

300 self.ns_transform_fn = advanced.ns_transform_fn 

301 self.post_update_fn = advanced.post_update_fn 

302 self.reset_optimizer_parameters() 

303 

304 def reset_optimizer_parameters(self) -> None: 

305 """Rebuild all parameter-identity caches from current param groups.""" 

306 self._group_dtensor_by_mesh() 

307 self._build_param_shard_metadata() 

308 deduced_count = self._auto_deduce_replica_count() 

309 if deduced_count is None: 

310 self.hsdp_replica_count = None 

311 elif self.hsdp_replica_count is None: 

312 self.hsdp_replica_count = deduced_count 

313 self._split_replicate_groups() 

314 self._build_hsdp_batch() 

315 self._build_param_broadcast_info() 

316 self._classify_parameters_for_step() 

317 

318 @staticmethod 

319 def _validate_ns_coefficients( 

320 ns_variant: str, 

321 ns_steps: int, 

322 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]], 

323 ) -> Optional[Tuple[Tuple[float, float, float], ...]]: 

324 """Validate and normalize externally supplied Newton-Schulz coefficients.""" 

325 if ns_variant != "custom": 

326 if ns_coefficients is not None: 

327 raise ValueError("ns_coefficients can only be provided when ns_variant='custom'") 

328 return None 

329 if not ns_coefficients: 

330 raise ValueError("ns_coefficients must be provided when ns_variant='custom'") 

331 if ns_steps > len(ns_coefficients): 

332 raise ValueError( 

333 f"ns_steps={ns_steps} exceeds the {len(ns_coefficients)} provided coefficient groups" 

334 ) 

335 

336 normalized = [] 

337 for index, coefficients in enumerate(ns_coefficients): 

338 if len(coefficients) != 3: 

339 raise ValueError( 

340 f"ns_coefficients[{index}] must contain exactly three values, got {coefficients!r}" 

341 ) 

342 normalized_coefficients = tuple(float(value) for value in coefficients) 

343 if not all(math.isfinite(value) for value in normalized_coefficients): 

344 raise ValueError(f"ns_coefficients[{index}] must contain only finite values") 

345 normalized.append(normalized_coefficients) 

346 return tuple(normalized) 

347 

348 def __str__(self) -> str: 

349 """Return the optimizer representation.""" 

350 return super().__repr__() 

351 

352 __repr__ = __str__ 

353 

354 @torch.no_grad() 

355 def step( 

356 self, 

357 closure: Optional[Callable[[], Any]] = None, 

358 ) -> Optional[float]: 

359 """Perform a single optimization step. 

360 

361 De-duplication is controlled by the caller: ``param_to_ns_input`` should already contain only the owned 

362 params (via ``hsdp_assign.owned_params``). The caller is responsible for broadcasting the updated params to 

363 replica peers via ``AsyncReplicateBroadcaster.flush_group``. 

364 

365 Args: 

366 closure: Optional callable that reevaluates the model and returns loss. 

367 

368 Returns: 

369 The closure loss when provided, otherwise ``None``. 

370 """ 

371 loss = None 

372 if closure is not None: 

373 with torch.enable_grad(): 

374 loss = closure() 

375 broadcaster = AsyncReplicateBroadcaster(self) 

376 

377 num_groups = len(self.param_groups) 

378 

379 for group in self.param_groups: 

380 group['step'] = (group.get('step') or 0) + 1 

381 

382 # Compute momentum only for no-comm params upfront. 

383 no_comm_ns: Dict[int, Dict] = {} 

384 for group_idx in range(num_groups): 

385 info = self._hsdp_assignment_batches.get(group_idx) 

386 if not info: 

387 # No HSDP info — all params are no_comm. 

388 unshard_params = self.unshard_params_by_group.get(group_idx, []) 

389 no_comm_ns[group_idx] = self._update_muon_momentum( 

390 self.param_groups[group_idx], unshard_params 

391 ) 

392 else: 

393 no_comm_params = info.get("no_comm", []) 

394 if no_comm_params: 

395 no_comm_ns[group_idx] = self._update_muon_momentum( 

396 self.param_groups[group_idx], no_comm_params 

397 ) 

398 else: 

399 no_comm_ns[group_idx] = {} 

400 

401 # Process no-comm params. 

402 for group_idx in range(num_groups): 

403 if no_comm_ns[group_idx]: 

404 group = self.param_groups[group_idx] 

405 self._process_unshard_params(group, no_comm_ns[group_idx]) 

406 if self.post_update_fn is not None: 

407 self._run_post_update_fn(group, no_comm_ns[group_idx].keys()) 

408 

409 # Flatten nested batch into a linear schedule. 

410 group_linear_batches: Dict[int, List[HSDPGroupAssignment]] = {} 

411 max_num_batches = 0 

412 for group_idx in range(num_groups): 

413 info = self._hsdp_assignment_batches.get(group_idx) 

414 linear_batches = [] 

415 if info: 

416 for bg in info.get("batch_groups", []): 

417 linear_batches.extend(bg.get("sub_batches", [])) 

418 group_linear_batches[group_idx] = linear_batches 

419 max_num_batches = max(max_num_batches, len(linear_batches)) 

420 

421 # Process batches: compute momentum per-batch for owned params only (HSDP de-duplication). 

422 for batch_idx in range(max_num_batches): 

423 for group_idx in range(num_groups): 

424 linear_batches = group_linear_batches[group_idx] 

425 if batch_idx >= len(linear_batches): 

426 continue 

427 

428 hsdp_assign = linear_batches[batch_idx] 

429 group = self.param_groups[group_idx] 

430 

431 # Compute momentum for this assignment's owned params only. 

432 ns_inputs = self._update_muon_momentum(group, hsdp_assign.owned_params) 

433 if not hsdp_assign.is_shard: 

434 if ns_inputs: 

435 self._process_unshard_params(group, ns_inputs) 

436 else: 

437 self._process_shard_params( 

438 group, ns_inputs, [hsdp_assign], group_idx, 

439 buffer_cache={}, 

440 ) 

441 if self.post_update_fn is not None: 

442 self._run_post_update_fn(group, ns_inputs.keys()) 

443 

444 # Flush broadcasts after each assignment. 

445 broadcaster.flush_group(hsdp_assign) 

446 

447 broadcaster.wait_all() 

448 return loss 

449 

450 def _run_post_update_fn( 

451 self, 

452 group: Dict[str, Any], 

453 params: Iterable[torch.nn.Parameter], 

454 ) -> None: 

455 """Run the optional model callback on updated owner parameters before broadcast.""" 

456 if self.post_update_fn is None: 

457 return 

458 context_values = { 

459 "lr": group["lr"], 

460 "weight_decay": group["weight_decay"], 

461 "step": group["step"], 

462 } 

463 for param in params: 

464 local_tensor = to_local_if_dtensor(param.data) 

465 context = MuonPostUpdateContext( 

466 param_fqn=getattr(param, "model_name", None), 

467 logical_shape=tuple(param.shape), 

468 **context_values, 

469 ) 

470 self.post_update_fn(param, local_tensor, context) 

471 

472 def _classify_parameters_for_step(self) -> None: 

473 """Classify params by whether the last two dims are sharded. 

474 

475 unshard: run Newton-Schulz locally. 

476 shard: all-gather before Newton-Schulz. 

477 

478 Reads from self._hsdp_assignment_batches which is organized by batch. 

479 """ 

480 self.unshard_params_by_group: Dict[int, List] = {} 

481 self.shard_params_by_group: Dict[int, List] = {} 

482 self.shard_assignments_by_group: Dict[int, List[HSDPGroupAssignment]] = {} 

483 # Per group: record.index -> shard coord that computes NS. 

484 self._shard_compute_coord: Dict[int, Dict[int, Tuple[int, ...]]] = {} 

485 

486 for group_idx, group in enumerate(self.param_groups): 

487 assignment_info = self._hsdp_assignment_batches.get(group_idx) 

488 

489 unshard_params = [] 

490 shard_hsdp_assignments: List[HSDPGroupAssignment] = [] 

491 

492 if assignment_info: 

493 unshard_params.extend(assignment_info["no_comm"]) 

494 for bg in assignment_info["batch_groups"]: 

495 for hsdp_assign in bg["sub_batches"]: 

496 if hsdp_assign.is_shard: 

497 shard_hsdp_assignments.append(hsdp_assign) 

498 else: 

499 unshard_params.extend(hsdp_assign.owned_params) 

500 else: 

501 unshard_params.extend(group["params"]) 

502 

503 # Shard params only include locally owned params; replica ownership is enforced at apply time. 

504 shard_params = [p for a in shard_hsdp_assignments for p in a.owned_params] 

505 

506 # Greedily assign NS compute across shard ranks. 

507 self._shard_compute_coord[group_idx] = {} 

508 for hsdp_assign in shard_hsdp_assignments: 

509 shard_sizes, _, _, _ = self._get_shard_info(hsdp_assign) 

510 compute_by_index = build_owner_by_size( 

511 records=hsdp_assign.owned_records, 

512 replicate_sizes=shard_sizes, 

513 ) 

514 self._shard_compute_coord[group_idx].update(compute_by_index) 

515 

516 self.unshard_params_by_group[group_idx] = unshard_params 

517 self.shard_assignments_by_group[group_idx] = shard_hsdp_assignments 

518 self.shard_params_by_group[group_idx] = shard_params 

519 

520 def _update_muon_momentum( 

521 self, 

522 group: Dict[str, Any], 

523 params: List[torch.Tensor], 

524 ) -> Dict[torch.nn.Parameter, torch.Tensor]: 

525 """Compute first-order momentum and return bfloat16 NS inputs.""" 

526 momentum = group["momentum"] 

527 if not isinstance(momentum, (list, tuple)): 

528 momentum = [momentum] 

529 if len(momentum) == 1: 

530 momentum = (momentum[0], momentum[0]) 

531 momentum1, momentum2 = momentum 

532 # Pre-filter params with valid grads and ensure momentum buffers exist 

533 valid_params = [] 

534 grads = [] 

535 bufs = [] 

536 for p in params: 

537 g = p.grad 

538 if g is None: 

539 continue 

540 if "momentum_buffer" not in self.state[p]: 

541 self.state[p]["momentum_buffer"] = torch.zeros_like(g) 

542 valid_params.append(p) 

543 grads.append(g) 

544 bufs.append(self.state[p]["momentum_buffer"]) 

545 

546 if not valid_params: 

547 return {} 

548 

549 grads = [to_local_if_dtensor(grad) for grad in grads] 

550 bufs = [to_local_if_dtensor(buffer) for buffer in bufs] 

551 if self.momentum_update_fn is not None: 

552 custom_updates = list(self.momentum_update_fn(grads, bufs, momentum1, momentum2, group["nesterov"])) 

553 if len(custom_updates) != len(valid_params): 

554 raise ValueError( 

555 "momentum_update_fn must return one update tensor per input gradient, " 

556 f"got {len(custom_updates)} updates for {len(valid_params)} gradients" 

557 ) 

558 custom_updates = [update.to(torch.bfloat16) for update in custom_updates] 

559 param_updates = dict(zip(valid_params, custom_updates)) 

560 return param_updates 

561 

562 if self._update_muon_momentum_mode == "vanilla": 

563 # Match vanilla implementation in Muon paper & PyTorch implementation: 

564 # m_for_update = (1 - momentum1) * grad + momentum1 * m_old 

565 # m_new = (1 - momentum2) * grad + momentum2 * m_old 

566 # Fold the Nesterov interpolation into one lerp with momentum1 squared. 

567 # pylint: disable=protected-access 

568 update_momentum = momentum1 ** 2 if group["nesterov"] else momentum1 

569 local_us = torch._foreach_lerp(grads, bufs, weight=update_momentum) 

570 torch._foreach_lerp_(bufs, grads, weight=1 - momentum2) 

571 elif self._update_muon_momentum_mode == "legacy": 

572 # Match muon_update_core(): 

573 # m_for_update = grad + momentum1 * m_old 

574 # m_new = grad + momentum2 * m_old 

575 # pylint: disable=protected-access 

576 local_us = torch._foreach_add(grads, bufs, alpha=momentum1) 

577 

578 torch._foreach_mul_(bufs, momentum2) 

579 torch._foreach_add_(bufs, grads) 

580 

581 if group["nesterov"]: 

582 torch._foreach_mul_(local_us, momentum1) 

583 torch._foreach_add_(local_us, grads) 

584 else: 

585 raise ValueError( 

586 "Muon._update_muon_momentum_mode must be 'vanilla' or 'legacy', " 

587 f"got {self._update_muon_momentum_mode!r}" 

588 ) 

589 

590 if local_us[0].dtype != torch.bfloat16: 

591 local_us = [update.to(torch.bfloat16) for update in local_us] 

592 

593 param_updates = dict(zip(valid_params, local_us)) 

594 return param_updates 

595 

596 def _process_unshard_params( 

597 self, 

598 group: Dict[str, Any], 

599 param_to_ns_input: Dict[torch.nn.Parameter, torch.Tensor], 

600 ) -> None: 

601 """Process un-sharded params: shape-group -> memory-batch -> NS -> local update.""" 

602 lr = group["lr"] 

603 weight_decay = group["weight_decay"] 

604 

605 # empty params are not processed in muon, for uneven shard 

606 param_to_ns_input = { 

607 p: ns_input for p, ns_input in param_to_ns_input.items() if ns_input.numel() > 0 

608 } 

609 if not param_to_ns_input: 

610 return 

611 

612 shape_groups = self._group_by_shape(list(param_to_ns_input.keys())) 

613 for _, p_list in shape_groups.items(): 

614 safe_batches = self._split_into_memory_safe_batches(p_list, shard_size=1) 

615 

616 for sub_batch in safe_batches: 

617 updates_dict = self._compute_batched_ns_updates( 

618 sub_batch, param_to_ns_input, group, no_shard=True 

619 ) 

620 

621 local_params = [to_local_if_dtensor(p.data) for p in sub_batch] 

622 local_updates = [updates_dict[p].view(lp.shape) for p, lp in zip(sub_batch, local_params)] 

623 

624 if weight_decay != 0.0: 

625 # pylint: disable=protected-access 

626 torch._foreach_mul_(local_params, 1 - lr * weight_decay) 

627 # pylint: disable=protected-access 

628 apply_alpha = 1.0 if group["apply_lr_in_update"] else -lr 

629 torch._foreach_add_(local_params, local_updates, alpha=apply_alpha) 

630 

631 def _gather_and_compute_shard_updates( 

632 self, 

633 valid_params: List[torch.nn.Parameter], 

634 param_to_ns_input: Dict[torch.nn.Parameter, torch.Tensor], 

635 hsdp_assign: HSDPGroupAssignment, 

636 shard_compute_coord: Dict[int, Tuple[int, ...]], 

637 group: Dict[str, Any], 

638 buffer_cache: Optional[Dict], 

639 ) -> Tuple[ 

640 Dict[torch.nn.Parameter, torch.Tensor], 

641 Dict[torch.nn.Parameter, Tuple[int, ...]], 

642 ]: 

643 """Gather NS inputs and compute updates for locally-assigned shard params. 

644 

645 Returns: 

646 (my_updates, param_compute_coord) for this HSDP assignment. 

647 """ 

648 shard_sizes, local_coords, shard_pgs, total_shard_size = self._get_shard_info(hsdp_assign) 

649 param_to_index = {record.param: record.index for record in hsdp_assign.owned_records} 

650 

651 my_params: List[torch.nn.Parameter] = [] 

652 my_param_ids: set = set() 

653 param_compute_coord: Dict[torch.nn.Parameter, Tuple[int, ...]] = {} 

654 my_indices: set = set() 

655 

656 for idx, p in enumerate(valid_params): 

657 p_index = param_to_index[p] 

658 compute_coord = shard_compute_coord.get(p_index, (0,) * len(shard_sizes)) 

659 param_compute_coord[p] = compute_coord 

660 if compute_coord == local_coords: 

661 my_params.append(p) 

662 my_param_ids.add(id(p)) 

663 my_indices.add(idx) 

664 

665 # Fused all-gather full NS inputs; keep only tensors computed locally. 

666 gathered_inputs: Dict[torch.nn.Parameter, torch.Tensor] = {} 

667 local_inputs = build_pad_ns_inputs( 

668 valid_params, 

669 param_to_ns_input, 

670 self._param_shard_metadata, 

671 ) 

672 param_shard_metadata = [self._param_shard_metadata.get(p) for p in valid_params] 

673 gathered_list = fused_allgather_dtensor_params( 

674 local_inputs, shard_pgs, hsdp_assign.layout_spec, 

675 param_shard_metadata=param_shard_metadata, 

676 buffer_cache=buffer_cache, 

677 keep_indices=my_indices, 

678 ) 

679 for p, full_inp in zip(valid_params, gathered_list): 

680 if id(p) in my_param_ids: 

681 gathered_inputs[p] = full_inp 

682 local_inputs.clear() 

683 gathered_list.clear() 

684 

685 # Compute NS updates in shape groups and memory-safe batches. 

686 my_updates: Dict[torch.nn.Parameter, torch.Tensor] = {} 

687 if my_params: 

688 shape_groups = self._group_by_shape(my_params) 

689 for _, p_list in shape_groups.items(): 

690 safe_batches = self._split_into_memory_safe_batches(p_list, shard_size=total_shard_size) 

691 for sub_batch in safe_batches: 

692 updates_dict = self._compute_batched_ns_updates( 

693 sub_batch, gathered_inputs, group 

694 ) 

695 for p in sub_batch: 

696 my_updates[p] = updates_dict[p].contiguous() 

697 del updates_dict 

698 gathered_inputs.clear() 

699 

700 return my_updates, param_compute_coord 

701 

702 def _process_shard_params( 

703 self, 

704 group: Dict[str, Any], 

705 param_to_ns_input: Dict[torch.nn.Parameter, torch.Tensor], 

706 hsdp_assignments: List[HSDPGroupAssignment], 

707 group_idx: int, 

708 buffer_cache: Optional[Dict] = None, 

709 ) -> None: 

710 """Process sharded params with greedy shard-group compute assignment.""" 

711 shard_compute_coord = self._shard_compute_coord.get(group_idx, {}) 

712 

713 for hsdp_assign in hsdp_assignments: 

714 owned_params = hsdp_assign.owned_params 

715 # Communication membership is static per assignment. Ranks with empty 

716 # local shards still participate via build_pad_ns_inputs(). 

717 valid_params = owned_params 

718 

719 shard_sizes, _, shard_pgs, total_shard_size = self._get_shard_info(hsdp_assign) 

720 

721 # uneven shard check 

722 shard_pgs_rank = [] 

723 for pg in shard_pgs: 

724 pg_rank = torch.distributed.get_process_group_ranks(pg) 

725 shard_pgs_rank.append(pg_rank) 

726 

727 for p in valid_params: 

728 if tuple(p.shape)[0] % shard_sizes[0] != 0: 

729 logger.debug_rank0( 

730 "[hyper-optimizer uneven-shard] p %s placement %s device_mesh %s " 

731 "local_shape %s global_shape %s pg_rank %s shard_sizes %s", 

732 p.model_name, 

733 p.placements, 

734 p.device_mesh, 

735 p.to_local().shape, 

736 p.shape, 

737 shard_pgs_rank, 

738 shard_sizes, 

739 ) 

740 

741 if valid_params: 

742 safe_batches = self._split_into_memory_safe_batches( 

743 valid_params, shard_size=total_shard_size, min_batch_size=total_shard_size 

744 ) 

745 for sub_batch in safe_batches: 

746 my_updates, param_compute_coord = self._gather_and_compute_shard_updates( 

747 sub_batch, param_to_ns_input, hsdp_assign, 

748 shard_compute_coord, group, buffer_cache, 

749 ) 

750 

751 self._fused_broadcast_and_apply( 

752 sub_batch, my_updates, param_compute_coord, 

753 group, hsdp_assign, 

754 ) 

755 

756 def _group_by_shape( 

757 self, 

758 params: List[torch.nn.Parameter], 

759 ) -> Dict[tuple, List[torch.nn.Parameter]]: 

760 """Group parameters by their last-2-dim shape (A, B) for batched NS. 

761 

762 [1024, 1024], [1024, 1, 1024], and [3, 1024, 1024] all map to 

763 key (1024, 1024) for maximum batch merging. 

764 """ 

765 groups = defaultdict(list) 

766 

767 for p in params: 

768 core_shape = self._shape_to_core_shape(tuple(p.shape)) 

769 groups[core_shape].append(p) 

770 

771 return groups 

772 

773 @staticmethod 

774 def _shape_to_core_shape(shape: Tuple[int, ...]) -> Tuple[int, int]: 

775 if len(shape) == 2: 

776 return (shape[0], shape[1]) 

777 if len(shape) == 3 and shape[1] == 1: 

778 return (shape[0], shape[2]) 

779 if len(shape) >= 3: 

780 return (shape[-2], shape[-1]) 

781 raise ValueError('1D parameters are not supported in Muon') 

782 

783 def _reshape_ns_input( 

784 self, 

785 param: torch.nn.Parameter, 

786 ns_input: torch.Tensor, 

787 ) -> Tuple[torch.Tensor, List[torch.Tensor]]: 

788 """Return the contiguous NS input and any reshape views used for NS.""" 

789 working_input = ns_input if ns_input.is_contiguous() else ns_input.contiguous() 

790 if self.reshape_fn is None: 

791 return working_input, [working_input] 

792 

793 param_fqn = param.model_name 

794 if param_fqn is None: 

795 return working_input, [working_input] 

796 

797 reshaped_inputs = list(self.reshape_fn(param_fqn, working_input)) 

798 if not reshaped_inputs: 

799 return working_input, [working_input] 

800 

801 if reshaped_inputs[0].shape != working_input.shape: 

802 logger.info_rank0( 

803 "Reshape %s from %s to %s", 

804 param_fqn, 

805 working_input.shape, 

806 reshaped_inputs[0].shape, 

807 ) 

808 

809 for reshaped_input in reshaped_inputs: 

810 if reshaped_input.untyped_storage().data_ptr() != working_input.untyped_storage().data_ptr(): 

811 raise ValueError("reshape_fn must return views that share storage with the working NS input tensor") 

812 

813 return working_input, reshaped_inputs 

814 

815 def _prepare_ns_transform( 

816 self, 

817 param: torch.nn.Parameter, 

818 ns_input: torch.Tensor, 

819 ) -> Tuple[torch.Tensor, NSInputTransform]: 

820 """Route one parameter through custom, zero-copy, or identity NS preparation.""" 

821 working_input = ns_input if ns_input.is_contiguous() else ns_input.contiguous() 

822 param_fqn = getattr(param, "model_name", None) 

823 if self.ns_transform_fn is not None and param_fqn is not None: 

824 transform = self.ns_transform_fn(param_fqn, working_input) 

825 if transform is not None: 

826 self._validate_ns_transform(transform, working_input) 

827 return working_input, transform 

828 

829 _, reshaped_inputs = self._reshape_ns_input(param, working_input) 

830 

831 def restore_view_updates(updates: List[torch.Tensor], output: torch.Tensor) -> None: 

832 """Write reshaped NS updates back into the views and the working input.""" 

833 for reshaped_input, update in zip(reshaped_inputs, updates): 

834 reshaped_input.copy_(update.contiguous().view_as(reshaped_input)) 

835 if output.untyped_storage().data_ptr() != working_input.untyped_storage().data_ptr(): 

836 output.copy_(working_input) 

837 

838 return working_input, NSInputTransform(tensors=reshaped_inputs, restore=restore_view_updates) 

839 

840 @staticmethod 

841 def _validate_ns_transform(transform: NSInputTransform, working_input: torch.Tensor) -> None: 

842 """Validate a model-provided reversible NS transform.""" 

843 if not isinstance(transform, NSInputTransform): 

844 raise ValueError( 

845 "ns_transform_fn must return NSInputTransform or None, " 

846 f"got {type(transform).__name__}" 

847 ) 

848 if not transform.tensors: 

849 raise ValueError("NSInputTransform.tensors must not be empty") 

850 for index, tensor in enumerate(transform.tensors): 

851 if not isinstance(tensor, torch.Tensor): 

852 raise ValueError(f"NSInputTransform.tensors[{index}] must be a torch.Tensor") 

853 if tensor.device != working_input.device: 

854 raise ValueError("NSInputTransform tensors must be on the same device as the NS input") 

855 if tensor.dtype != working_input.dtype: 

856 raise ValueError("NSInputTransform tensors must have the same dtype as the NS input") 

857 if tensor.dim() < 2: 

858 raise ValueError("NSInputTransform tensors must have at least two dimensions") 

859 if not callable(transform.restore): 

860 raise ValueError("NSInputTransform.restore must be callable") 

861 

862 def _compute_batched_ns_outputs_for_tensors( 

863 self, 

864 tensor_list: List[torch.Tensor], 

865 ns_steps: int, 

866 ns_variant: str = "asym5", 

867 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]] = None, 

868 ns_epsilon: float = 1e-10, 

869 ) -> List[torch.Tensor]: 

870 """Run batched NS on mixed-shape tensors and restore their original shapes.""" 

871 

872 if not tensor_list: 

873 return [] 

874 

875 state = {"inputs_3d": [], "slice_sizes": [], "shapes_info": [], "restored_updates": [], "current_idx": 0} 

876 

877 for tensor in tensor_list: 

878 origin_shape = tuple(tensor.shape) 

879 is_conv = False 

880 

881 if len(origin_shape) == 2: 

882 inp_3d = tensor.unsqueeze(0) 

883 n_dim = 1 

884 elif len(origin_shape) == 3 and origin_shape[1] == 1: 

885 inp_3d = tensor.squeeze(1).unsqueeze(0) 

886 is_conv = True 

887 n_dim = 1 

888 elif len(origin_shape) == 3: 

889 inp_3d = tensor 

890 n_dim = origin_shape[0] 

891 else: 

892 inp_3d = tensor.reshape(-1, origin_shape[-2], origin_shape[-1]) 

893 n_dim = inp_3d.shape[0] 

894 

895 state["inputs_3d"].append(inp_3d) 

896 state["slice_sizes"].append(n_dim) 

897 state["shapes_info"].append((origin_shape, is_conv)) 

898 

899 merged_input = torch.cat(state["inputs_3d"], dim=0) 

900 

901 if merged_input.shape[0] == 1: 

902 merged_input = merged_input.squeeze(0) 

903 

904 if self.zeropower_fn is None: 

905 merged_update = zeropower_via_newtonschulz5( 

906 merged_input, 

907 steps=ns_steps, 

908 ns_variant=ns_variant, 

909 epsilon=ns_epsilon, 

910 ns_coefficients=ns_coefficients, 

911 ) 

912 else: 

913 merged_update = self.zeropower_fn(merged_input, steps=ns_steps) 

914 

915 if merged_input.dim() == 2: 

916 merged_update = merged_update.unsqueeze(0) 

917 del merged_input 

918 

919 for n_dim, (origin_shape, is_conv) in zip(state["slice_sizes"], state["shapes_info"]): 

920 update = merged_update[state["current_idx"]:state["current_idx"] + n_dim] 

921 state["current_idx"] += n_dim 

922 

923 if is_conv: 

924 update = update.squeeze(0).unsqueeze(1) 

925 elif len(origin_shape) == 2: 

926 update = update.squeeze(0) 

927 elif len(origin_shape) >= 4: 

928 update = update.reshape(origin_shape) 

929 

930 state["restored_updates"].append(update) 

931 

932 del merged_update 

933 return state["restored_updates"] 

934 

935 def _split_into_memory_safe_batches( 

936 self, 

937 p_list: List[torch.nn.Parameter], 

938 shard_size: int = 1, 

939 min_batch_size: int = 1, 

940 ) -> List[List[torch.nn.Parameter]]: 

941 """Split parameters into memory-safe batches to prevent OOM during NS. 

942 

943 The per-batch element limit is scaled down by shard_size to account 

944 for the memory amplification from allgather. Sharded paths prioritize 

945 filling the compute-owner ranks before applying this soft limit. 

946 """ 

947 max_numel_per_batch = 512 * 1024 * 1024 // shard_size 

948 min_batch_size = max(1, min_batch_size) 

949 

950 batches = [] 

951 current_batch = [] 

952 current_count = 0 

953 

954 for p in p_list: 

955 p_count = p.numel() 

956 if ( 

957 len(current_batch) >= min_batch_size 

958 and current_count + p_count > max_numel_per_batch 

959 ): 

960 batches.append(current_batch) 

961 current_batch = [p] 

962 current_count = p_count 

963 else: 

964 current_batch.append(p) 

965 current_count += p_count 

966 

967 if current_batch: 

968 batches.append(current_batch) 

969 

970 return batches 

971 

972 def _compute_batched_ns_updates( 

973 self, 

974 p_list: List[torch.nn.Parameter], 

975 ns_inputs: Dict[torch.nn.Parameter, torch.Tensor], 

976 group: Dict[str, Any], 

977 no_shard: bool = False 

978 ) -> Dict[torch.nn.Parameter, torch.Tensor]: 

979 """Route batched NS through the native or reversible-transform path.""" 

980 if self.ns_transform_fn is None: 

981 return self._compute_batched_ns_updates_fast(p_list, ns_inputs, group, no_shard) 

982 return self._compute_batched_ns_updates_with_transform(p_list, ns_inputs, group, no_shard) 

983 

984 def _compute_batched_ns_updates_fast( 

985 self, 

986 p_list: List[torch.nn.Parameter], 

987 ns_inputs: Dict[torch.nn.Parameter, torch.Tensor], 

988 group: Dict[str, Any], 

989 no_shard: bool = False, 

990 ) -> Dict[torch.nn.Parameter, torch.Tensor]: 

991 """Compute native reshape/view NS updates without transform bookkeeping.""" 

992 if not p_list: 

993 return {} 

994 state = {"reshape_groups": defaultdict(list), "origin_shapes": {}, "working_inputs": {}, "restored": {}} 

995 for param in p_list: 

996 local_shape = getattr(param, "local_shape", None) 

997 if local_shape is None: 

998 local_shape = to_local_if_dtensor(param.data).shape 

999 state["origin_shapes"][param] = tuple(local_shape) if no_shard else tuple(param.shape) 

1000 working_input, reshaped_inputs = self._reshape_ns_input( 

1001 param, ns_inputs[param].view(state["origin_shapes"][param]) 

1002 ) 

1003 state["working_inputs"][param] = working_input 

1004 for reshaped_input in reshaped_inputs: 

1005 state["reshape_groups"][self._shape_to_core_shape(tuple(reshaped_input.shape))].append( 

1006 reshaped_input 

1007 ) 

1008 for tensor_list in state["reshape_groups"].values(): 

1009 updates = self._compute_batched_ns_outputs_for_tensors( 

1010 tensor_list, group["ns_steps"], group["ns_variant"], group["ns_coefficients"], group["ns_epsilon"] 

1011 ) 

1012 for reshaped_input, update in zip(tensor_list, updates): 

1013 scale = compute_muon_slice_scale( 

1014 update, group["matched_adamw_rms"], zero_rms_scale_mode=group["zero_rms_scale_mode"] 

1015 ) 

1016 if group["apply_lr_in_update"]: 

1017 scale *= -group["lr"] 

1018 update.mul_(scale) 

1019 reshaped_input.copy_(update.contiguous().view_as(reshaped_input)) 

1020 

1021 for param in p_list: 

1022 update = ns_inputs[param].view(state["origin_shapes"][param]) 

1023 working_input = state["working_inputs"][param] 

1024 if working_input.untyped_storage().data_ptr() != update.untyped_storage().data_ptr(): 

1025 update.copy_(working_input) 

1026 state["restored"][param] = update 

1027 return state["restored"] 

1028 

1029 def _compute_batched_ns_updates_with_transform( 

1030 self, 

1031 p_list: List[torch.nn.Parameter], 

1032 ns_inputs: Dict[torch.nn.Parameter, torch.Tensor], 

1033 group: Dict[str, Any], 

1034 no_shard: bool = False, 

1035 ) -> Dict[torch.nn.Parameter, torch.Tensor]: 

1036 """Batched Newton-Schulz update for mixed 2D / Conv3D / 3D parameters. 

1037 

1038 Normalizes all inputs to 3D, concatenates along dim 0, runs a single 

1039 NS iteration, then slices results back to original shapes. 

1040 

1041 """ 

1042 

1043 if not p_list: 

1044 return {} 

1045 

1046 state: _TransformNSState = { 

1047 "reshape_groups": defaultdict(list), 

1048 "origin_shapes": {}, 

1049 "working_inputs": {}, 

1050 "transforms": {}, 

1051 "transform_updates": [], 

1052 "restored": {}, 

1053 } 

1054 

1055 for p in p_list: 

1056 state["origin_shapes"][p] = ( 

1057 tuple(to_local_if_dtensor(p.data).shape) if no_shard else tuple(p.shape) 

1058 ) 

1059 prepared = self._prepare_ns_transform(p, ns_inputs[p].view(state["origin_shapes"][p])) 

1060 state["working_inputs"][p] = prepared[0] 

1061 state["transforms"][p] = prepared[1] 

1062 state["transform_updates"].append([None] * len(prepared[1].tensors)) 

1063 

1064 for tensor_index, transformed_input in enumerate(prepared[1].tensors): 

1065 state["reshape_groups"][self._shape_to_core_shape(tuple(transformed_input.shape))].append( 

1066 (transformed_input, len(state["transform_updates"]) - 1, tensor_index) 

1067 ) 

1068 

1069 for tensor_records in state["reshape_groups"].values(): 

1070 reshaped_updates = self._compute_batched_ns_outputs_for_tensors( 

1071 [record[0] for record in tensor_records], 

1072 group["ns_steps"], 

1073 ns_variant=group["ns_variant"], 

1074 ns_coefficients=group["ns_coefficients"], 

1075 ns_epsilon=group["ns_epsilon"], 

1076 ) 

1077 

1078 # Scale updates. 

1079 for record, reshaped_update in zip(tensor_records, reshaped_updates): 

1080 slice_scale = compute_muon_slice_scale( 

1081 reshaped_update, 

1082 group["matched_adamw_rms"], 

1083 zero_rms_scale_mode=group["zero_rms_scale_mode"], 

1084 ) 

1085 

1086 if group["apply_lr_in_update"]: 

1087 slice_scale *= -group["lr"] 

1088 reshaped_update.mul_(slice_scale) 

1089 state["transform_updates"][record[1]][record[2]] = reshaped_update 

1090 

1091 for p, reshaped_updates in zip(p_list, state["transform_updates"]): 

1092 

1093 if any(update is None for update in reshaped_updates): 

1094 raise RuntimeError("Missing Newton-Schulz update for a transformed input") 

1095 state["transforms"][p].restore(reshaped_updates, state["working_inputs"][p]) 

1096 

1097 for p in p_list: 

1098 reshaped_update = ns_inputs[p].view(state["origin_shapes"][p]) 

1099 

1100 if (state["working_inputs"][p].untyped_storage().data_ptr() 

1101 != reshaped_update.untyped_storage().data_ptr()): 

1102 reshaped_update.copy_(state["working_inputs"][p]) 

1103 state["restored"][p] = reshaped_update 

1104 

1105 return state["restored"] 

1106 

1107 def _fused_broadcast_and_apply( 

1108 self, 

1109 valid_params: List[torch.nn.Parameter], 

1110 my_updates: Dict[torch.nn.Parameter, torch.Tensor], 

1111 param_compute_coord: Dict[torch.nn.Parameter, Tuple[int, ...]], 

1112 group: Dict[str, Any], 

1113 hsdp_assign: HSDPGroupAssignment, 

1114 ) -> None: 

1115 """Fused broadcast and apply for shard-group updates.""" 

1116 lr = group["lr"] 

1117 weight_decay = group["weight_decay"] 

1118 shard_sizes, local_coords, shard_pgs, total_shard_size = self._get_shard_info(hsdp_assign) 

1119 device = to_local_if_dtensor(valid_params[0].data).device 

1120 

1121 coord_groups: Dict[Tuple[int, ...], List[torch.nn.Parameter]] = defaultdict(list) 

1122 for p in valid_params: 

1123 coord_groups[param_compute_coord[p]].append(p) 

1124 

1125 all_local_params: List[torch.Tensor] = [] 

1126 all_update_shards: List[torch.Tensor] = [] 

1127 

1128 alignment_bytes = 512 

1129 element_size = torch.empty(0, dtype=torch.bfloat16, device=device).element_size() 

1130 alignment_elements = max(1, alignment_bytes // element_size) 

1131 

1132 pack_buffers: Dict[Tuple[int, ...], torch.Tensor] = {} 

1133 coord_param_offsets: Dict[Tuple[int, ...], List[Tuple[int, int, int]]] = {} 

1134 

1135 for coord, coord_params in coord_groups.items(): 

1136 is_compute_rank = coord == local_coords 

1137 param_offsets: List[Tuple[int, int, int]] = [] 

1138 total_padded_numel = 0 

1139 

1140 for p in coord_params: 

1141 actual_numel = p.numel() 

1142 padded_numel = ((actual_numel + alignment_elements - 1) // alignment_elements) * alignment_elements 

1143 param_offsets.append((total_padded_numel, actual_numel, padded_numel)) 

1144 total_padded_numel += padded_numel 

1145 

1146 coord_param_offsets[coord] = param_offsets 

1147 pack_buffer = torch.empty(total_padded_numel, dtype=torch.bfloat16, device=device) 

1148 

1149 if is_compute_rank: 

1150 for p, (offset, actual_numel, padded_numel) in zip(coord_params, param_offsets): 

1151 update = my_updates[p] 

1152 pack_buffer[offset:offset + actual_numel].copy_(update.reshape(-1)) 

1153 if padded_numel > actual_numel: 

1154 pack_buffer[offset + actual_numel:offset + padded_numel].zero_() 

1155 

1156 pack_buffers[coord] = pack_buffer 

1157 

1158 if total_shard_size > 1: 

1159 self._batched_relay_broadcast( 

1160 pack_buffers, shard_pgs, shard_sizes, local_coords 

1161 ) 

1162 

1163 layout_spec = hsdp_assign.layout_spec 

1164 for coord, coord_params in coord_groups.items(): 

1165 pack_buffer = pack_buffers[coord] 

1166 param_offsets = coord_param_offsets[coord] 

1167 

1168 for p, (offset, actual_numel, _) in zip(coord_params, param_offsets): 

1169 full_update = pack_buffer[offset:offset + actual_numel].view(tuple(p.shape)) 

1170 update_to_apply = chunk_update_by_layout( 

1171 full_update, 

1172 p, 

1173 layout_spec, 

1174 self._param_shard_metadata.get(p), 

1175 ) 

1176 

1177 local_param = to_local_if_dtensor(p.data) 

1178 all_local_params.append(local_param) 

1179 all_update_shards.append(update_to_apply.view(local_param.shape)) 

1180 

1181 if not all_local_params: 

1182 return 

1183 

1184 if weight_decay != 0.0: 

1185 coeff = 1.0 - lr * weight_decay 

1186 # pylint: disable=protected-access 

1187 torch._foreach_mul_(all_local_params, coeff) 

1188 

1189 # Slice-wise Muon scaling has already been applied during NS postprocess. 

1190 # pylint: disable=protected-access 

1191 apply_alpha = 1.0 if group["apply_lr_in_update"] else -lr 

1192 torch._foreach_add_(all_local_params, all_update_shards, alpha=apply_alpha) 

1193 

1194 def _build_param_shard_metadata(self) -> None: 

1195 """Build shard metadata once during optimizer init.""" 

1196 self._param_shard_metadata: Dict[torch.nn.Parameter, ParamShardMeta] = {} 

1197 

1198 for _, hsdp_groups in self._hsdp_grouping.values(): 

1199 for hsdp_group in hsdp_groups: 

1200 if hsdp_group.layout_spec is None or not hsdp_group.layout_spec.shard_axes: 

1201 continue 

1202 group_param_to_meta = build_param_shard_metadata_for_group(hsdp_group) 

1203 for param, shard_meta in group_param_to_meta.items(): 

1204 self._param_shard_metadata[param] = shard_meta 

1205 _debug_param_shard_metadata(hsdp_group, group_param_to_meta) 

1206 

1207 @staticmethod 

1208 def _get_shard_info( 

1209 hsdp_assign: HSDPGroupAssignment, 

1210 ) -> Tuple[Tuple[int, ...], Tuple[int, ...], Tuple[dist.ProcessGroup, ...], int]: 

1211 """Extract shard topology from HSDPGroupAssignment. 

1212 

1213 Returns: 

1214 shard_sizes: Size of each shard mesh dimension. 

1215 local_coords: Current rank's coordinate in each shard mesh dim. 

1216 shard_pgs: ProcessGroup for each shard mesh dim. 

1217 total_shard_size: Product of all shard_sizes. 

1218 """ 

1219 shard_pgs = hsdp_assign.shard_pgs 

1220 shard_sizes = tuple( 

1221 dist.get_world_size(pg) if pg is not None else 1 

1222 for pg in shard_pgs 

1223 ) 

1224 local_coords = tuple( 

1225 dist.get_rank(pg) if pg is not None else 0 

1226 for pg in shard_pgs 

1227 ) 

1228 total_shard_size = 1 

1229 for s in shard_sizes: 

1230 total_shard_size *= s 

1231 

1232 return shard_sizes, local_coords, shard_pgs, total_shard_size 

1233 

1234 @staticmethod 

1235 def _batched_relay_broadcast( 

1236 tensor_dict: Dict[Tuple[int, ...], torch.Tensor], 

1237 shard_pgs: Tuple[dist.ProcessGroup, ...], 

1238 shard_sizes: Tuple[int, ...], 

1239 local_coords: Tuple[int, ...], 

1240 ) -> None: 

1241 """ 

1242 Batched asynchronous multi-dimensional relay broadcast. 

1243 By operating asynchronously within each dimension, we eliminate CPU overhead bubbles 

1244 while strictly preserving the multidimensional relay dependency. 

1245 """ 

1246 for dim_idx, pg in enumerate(shard_pgs): 

1247 if pg is None or shard_sizes[dim_idx] <= 1: 

1248 continue 

1249 

1250 work_handles = [] 

1251 

1252 for coord, tensor in tensor_dict.items(): 

1253 aligned = all( 

1254 local_coords[sub_dim] == coord[sub_dim] 

1255 for sub_dim in range(dim_idx + 1, len(shard_pgs)) 

1256 ) 

1257 if not aligned: 

1258 continue 

1259 

1260 src_rank_in_pg = coord[dim_idx] 

1261 global_src_rank = dist.get_global_rank(pg, src_rank_in_pg) 

1262 

1263 work = dist.broadcast(tensor, src=global_src_rank, group=pg, async_op=True) 

1264 if work is not None: 

1265 work_handles.append(work) 

1266 

1267 for work in work_handles: 

1268 work.wait()