Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / swap_muon.py: 95%

234 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Muon momentum-only optimizer-state swap adapter.""" 

16 

17# pylint: disable=protected-access 

18 

19from __future__ import annotations 

20 

21from dataclasses import dataclass 

22import math 

23from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple 

24 

25import torch 

26 

27from hyper_parallel.core.optimizer.muon import Muon 

28from hyper_parallel.core.optimizer.optimizer import AsyncReplicateBroadcaster 

29from hyper_parallel.core.optimizer.sharding_category import HSDPGroupAssignment 

30from hyper_parallel.core.optimizer.swap_optimizer_base import ( 

31 StateSwapAdapter, 

32 SwapOptimizer as CoreSwapOptimizer, 

33 SwapSlot, 

34) 

35 

36#: Muon's only persistent optimizer state that participates in swap. 

37MUON_STATE_KEYS = ("momentum_buffer",) 

38 

39_NO_COMM = "no_comm" 

40_HSDP = "hsdp" 

41 

42 

43@dataclass 

44class MuonSwapUnit: 

45 """One atomic Muon update unit and the momentum slots it needs. 

46 

47 A unit is never split by the pipeline: its parameters run through momentum, 

48 Newton-Schulz and the parameter update as one indivisible piece of work, so 

49 a single matrix is never partially swapped. ``kind`` only selects which 

50 Muon execution path consumes the unit, and ``hsdp_assign`` keeps the 

51 original HSDP communication boundary intact. 

52 

53 Attributes: 

54 group_index: Index of the owning parameter group, derived from 

55 ``schedule_key``. 

56 kind: ``"no_comm"`` for locally orthonormalized params, ``"hsdp"`` for 

57 an HSDP shard/replicate assignment. Derived from ``schedule_key``. 

58 params: Parameters this unit updates, in schedule order. 

59 slots: Swappable momentum-buffer slots for ``params``. 

60 schedule_key: Rank-independent group, kind, assignment and sub-batch indices. 

61 logical_nbytes: Full logical momentum bytes, independent of local ownership. 

62 hsdp_assign: The original assignment for ``"hsdp"`` units, else ``None``. 

63 post_update_params: Parameters whose post-update callbacks run after this 

64 unit. Only the final unit of the original Muon update boundary owns 

65 this tuple, in the original parameter order. 

66 flushes_broadcast: Whether this unit commits its assignment's parameter 

67 broadcast. Only the last unit of an assignment sets it, so an 

68 assignment split into several NS batches still issues the single 

69 flush the bare schedule issues. 

70 """ 

71 

72 params: List[torch.nn.Parameter] 

73 slots: List[SwapSlot] 

74 schedule_key: Tuple[int, str, int, int] 

75 logical_nbytes: int 

76 hsdp_assign: Optional[HSDPGroupAssignment] = None 

77 post_update_params: Tuple[torch.nn.Parameter, ...] = () 

78 flushes_broadcast: bool = False 

79 

80 @property 

81 def group_index(self) -> int: 

82 """Return the owning parameter-group index from the schedule identity.""" 

83 return self.schedule_key[0] 

84 

85 @property 

86 def kind(self) -> str: 

87 """Return the execution-path kind from the schedule identity.""" 

88 return self.schedule_key[1] 

89 

90 

91class MuonSwapAdapter(StateSwapAdapter): 

92 """Swap adapter that offloads and prefetches Muon's momentum buffers. 

93 

94 The adapter is the swap *coordinator* for Muon, never a second Muon 

95 implementation. It replays the schedule of the unmodified ``Muon`` and 

96 calls that optimizer's existing execution methods -- momentum, 

97 Newton-Schulz, all-gather, broadcast and the parameter write-back all stay 

98 in ``muon.py`` as the single source of truth. What the adapter owns is 

99 (a) building the atomic update-unit schedule in the original order, and 

100 (b) making sure the momentum buffer each unit reads is materialized on the 

101 device only for the duration of that unit. 

102 """ 

103 

104 @staticmethod 

105 def default_state_keys() -> Tuple[str, ...]: 

106 """Return the swap state keys a Muon optimizer owns.""" 

107 return MUON_STATE_KEYS 

108 

109 def validate(self) -> None: 

110 """Reject configurations the momentum-only swap path cannot preserve. 

111 

112 Raises: 

113 ValueError: If a configured state key is not one Muon owns, or a 

114 parameter holds momentum state the runtime cannot rebind, or a 

115 custom momentum update would lose its original call boundary. 

116 """ 

117 self._validate_configured_state_keys() 

118 if self.optimizer.momentum_update_fn is not None: 

119 raise ValueError( 

120 "Muon swap optimizer does not support momentum_update_fn because " 

121 "state partitioning changes its call boundary." 

122 ) 

123 for group in self.optimizer.param_groups: 

124 for param in group["params"]: 

125 self._validate_param(param) 

126 

127 def _validate_configured_state_keys(self) -> None: 

128 """Reject a configured swap key that is not Muon state. 

129 

130 ``_configured_state_keys`` shares a vocabulary with the other families 

131 and silently drops keys none of them implement, so Muon has to check its 

132 own selection explicitly: asking to swap ``exp_avg`` on a Muon optimizer 

133 must fail rather than quietly swap nothing. 

134 

135 Raises: 

136 ValueError: If the configured selection names a foreign state key. 

137 """ 

138 if self.config.state_keys is None: 

139 return 

140 unavailable = sorted(set(self.config.state_keys) - set(self.default_state_keys())) 

141 if unavailable: 

142 raise ValueError( 

143 f"Requested state keys {unavailable} are not available for {type(self.optimizer)!r}; " 

144 f"Muon only swaps {self.default_state_keys()}." 

145 ) 

146 

147 def _validate_param(self, param: torch.nn.Parameter) -> None: 

148 """Validate one parameter's momentum state is swap-compatible.""" 

149 state = self.optimizer.state.get(param) 

150 if not state: 

151 return 

152 buffer = state.get("momentum_buffer") 

153 if buffer is None: 

154 return 

155 if getattr(param, "grad", None) is not None and getattr(param.grad, "is_sparse", False): 

156 raise ValueError("Muon swap optimizer only supports dense gradients.") 

157 local = self._runtime_local_tensor(buffer) 

158 if not isinstance(local, torch.Tensor): 

159 raise ValueError( 

160 f"Muon swap optimizer requires tensor momentum state, got {type(buffer)!r}." 

161 ) 

162 if local.layout != torch.strided: 

163 raise ValueError("Muon swap optimizer requires dense momentum state.") 

164 if not local.is_contiguous(): 

165 raise ValueError("Muon swap optimizer requires contiguous momentum state.") 

166 

167 @staticmethod 

168 def _runtime_local_tensor(tensor: Any) -> Any: 

169 """Return the local shard of a possibly distributed tensor.""" 

170 to_local = getattr(tensor, "to_local", None) 

171 return to_local() if callable(to_local) else tensor 

172 

173 def _scheduled_param_ids(self) -> Optional[set[int]]: 

174 """Return parameters this rank owns in the current Muon schedule. 

175 

176 ``None`` means the optimizer has no HSDP schedule and therefore all 

177 parameters are local. With HSDP metadata present, only no-comm 

178 parameters and assignment owners may carry Muon state on this rank. 

179 """ 

180 schedule = getattr(self.optimizer, "_hsdp_assignment_batches", None) 

181 # ``Muon`` initialises this as an empty dict in non-HSDP runs, so an 

182 # empty schedule carries no ownership metadata at all: every parameter 

183 # is local and pre-existing momentum state must be kept. 

184 if not schedule: 

185 return None 

186 scheduled: set[int] = set() 

187 for group_index, group in enumerate(self.optimizer.param_groups): 

188 info = schedule.get(group_index) 

189 if info is None: 

190 scheduled.update(id(param) for param in group["params"]) 

191 continue 

192 scheduled.update(id(param) for param in info.get("no_comm", [])) 

193 for batch_group in info.get("batch_groups", []): 

194 for assignment in batch_group.get("sub_batches", []): 

195 scheduled.update(id(param) for param in assignment.owned_params) 

196 return scheduled 

197 

198 def _params_with_local_state(self, params: Iterable[Any]) -> Iterable[Any]: 

199 """Keep checkpoint-restored momentum only for local schedule owners.""" 

200 scheduled_ids = self._scheduled_param_ids() 

201 if scheduled_ids is None: 

202 return params 

203 return tuple(param for param in params if id(param) in scheduled_ids) 

204 

205 def initial_slots(self) -> Iterable[SwapSlot]: 

206 """Discover only local-owner momentum state before the first step.""" 

207 scheduled_ids = self._scheduled_param_ids() 

208 slots = [] 

209 for group in self.optimizer.param_groups: 

210 for param in group["params"]: 

211 if scheduled_ids is not None and id(param) not in scheduled_ids: 

212 # State may have been primed on every rank before this 

213 # wrapper was created. Drop non-owner entries before the 

214 # swap runtime publishes any slots. 

215 self.optimizer.state.pop(param, None) 

216 continue 

217 state = self.optimizer.state.get(param) 

218 if not state: 

219 continue 

220 for key in self._configured_state_keys(): 

221 if key in state and (id(param), key) not in self._slots: 

222 self._slots[(id(param), key)] = self._make_slot(key, state[key]) 

223 slots.extend(self._slots_for_params([param])) 

224 return tuple(slots) 

225 

226 # ------------------------------------------------------------------ schedule 

227 def prepare_step(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: 

228 """Advance steps, register momentum slots and build the unit schedule. 

229 

230 The unit schedule mirrors ``Muon.step()`` exactly: the no-comm params of 

231 every group first, then the flattened HSDP assignment schedule, keeping 

232 the per-group-then-per-batch nesting so collective order across ranks is 

233 unchanged. 

234 

235 Returns: 

236 Step context holding the ordered units and the step's broadcaster. 

237 

238 Raises: 

239 ValueError: If extra step arguments are passed. 

240 """ 

241 if args or kwargs: 

242 raise ValueError("Muon swap optimizer step does not support closure or extra arguments.") 

243 for group in self.optimizer.param_groups: 

244 group['step'] = (group.get('step') or 0) + 1 

245 

246 units: List[MuonSwapUnit] = [] 

247 num_groups = len(self.optimizer.param_groups) 

248 for group_index in range(num_groups): 

249 no_comm_params = self._no_comm_params(group_index) 

250 no_comm_batches = list(self._iter_unshard_batches(no_comm_params)) 

251 for batch_index, params in enumerate(no_comm_batches): 

252 post_update_params = no_comm_params if batch_index == len(no_comm_batches) - 1 else () 

253 units.append(self._make_unit( 

254 (group_index, _NO_COMM, -1, batch_index), params, params, 

255 post_update_params=post_update_params, 

256 )) 

257 

258 linear_batches = [ 

259 self._hsdp_linear_batches(group_index) for group_index in range(num_groups) 

260 ] 

261 max_num_batches = max((len(batches) for batches in linear_batches), default=0) 

262 for batch_index in range(max_num_batches): 

263 for group_index in range(num_groups): 

264 group_batches = linear_batches[group_index] 

265 if batch_index >= len(group_batches): 

266 continue 

267 units.extend(self._make_hsdp_units(group_index, group_batches[batch_index], batch_index)) 

268 

269 return { 

270 "units": units, 

271 "broadcaster": AsyncReplicateBroadcaster(self.optimizer), 

272 } 

273 

274 def _no_comm_params(self, group_index: int) -> List[torch.nn.Parameter]: 

275 """Return the params of ``group_index`` that need no HSDP collective. 

276 

277 This reads the same private schedule metadata ``Muon.step()`` reads, and 

278 falls back to every parameter in the group when no HSDP assignment 

279 metadata was built for it -- the same "no HSDP info means all params are 

280 no_comm" rule the bare step applies. 

281 """ 

282 info = getattr(self.optimizer, "_hsdp_assignment_batches", {}).get(group_index) 

283 if not info: 

284 return list(self.optimizer.unshard_params_by_group.get(group_index, [])) 

285 return list(info.get("no_comm", [])) 

286 

287 def _hsdp_linear_batches(self, group_index: int) -> List[HSDPGroupAssignment]: 

288 """Flatten one group's nested HSDP schedule into a linear batch list.""" 

289 info = getattr(self.optimizer, "_hsdp_assignment_batches", {}).get(group_index) 

290 linear_batches: List[HSDPGroupAssignment] = [] 

291 if info: 

292 for batch_group in info.get("batch_groups", []): 

293 linear_batches.extend(batch_group.get("sub_batches", [])) 

294 return linear_batches 

295 

296 def _iter_unshard_batches( 

297 self, 

298 params: List[torch.nn.Parameter], 

299 ) -> Iterator[List[torch.nn.Parameter]]: 

300 """Plan logical batches using Muon's existing shape and memory rules.""" 

301 for shape_group in self.optimizer._group_by_shape(params).values(): 

302 yield from self.optimizer._split_into_memory_safe_batches(shape_group, shard_size=1) 

303 

304 def _make_unit( 

305 self, 

306 schedule_key: Tuple[int, str, int, int], 

307 logical_params: Iterable[torch.nn.Parameter], 

308 params: Iterable[torch.nn.Parameter], 

309 hsdp_assign: Optional[HSDPGroupAssignment] = None, 

310 post_update_params: Iterable[torch.nn.Parameter] = (), 

311 flushes_broadcast: bool = False, 

312 ) -> MuonSwapUnit: 

313 """Project a logical unit onto local gradient-bearing owners and slots.""" 

314 logical_nbytes = sum( 

315 math.prod(param.shape) * self._runtime_local_tensor(param).element_size() 

316 for param in logical_params 

317 ) 

318 params = [param for param in params if param.grad is not None] 

319 post_update_params = tuple( 

320 param for param in post_update_params if param.grad is not None 

321 ) 

322 # HSDP assignments contain every logical parameter, but only the local 

323 # owner participates in this rank's momentum update. Register state 

324 # only for parameters that are actually present in this atomic unit; 

325 # otherwise a non-owner would retain a device/host slot indefinitely. 

326 for param in params: 

327 self._ensure_momentum_state(param) 

328 return MuonSwapUnit( 

329 params=params, 

330 slots=self._slots_for_params(params), 

331 schedule_key=schedule_key, 

332 logical_nbytes=logical_nbytes, 

333 hsdp_assign=hsdp_assign, 

334 post_update_params=post_update_params, 

335 flushes_broadcast=flushes_broadcast, 

336 ) 

337 

338 def _make_hsdp_units( 

339 self, 

340 group_index: int, 

341 hsdp_assign: HSDPGroupAssignment, 

342 assignment_index: int, 

343 ) -> List[MuonSwapUnit]: 

344 """Build the units for one HSDP assignment, in assignment order. 

345 

346 Sharded assignments stay atomic because the shard executor consumes the 

347 entire assignment. For unsharded assignments, every rank derives all 

348 owners' NS batches before selecting its own local batch. 

349 """ 

350 if hsdp_assign.is_shard: 

351 return [ 

352 self._make_unit( 

353 (group_index, _HSDP, assignment_index, 0), 

354 hsdp_assign.all_params, hsdp_assign.owned_params, 

355 hsdp_assign, post_update_params=hsdp_assign.owned_params, 

356 flushes_broadcast=True, 

357 ) 

358 ] 

359 params_by_owner: Dict[Tuple[int, ...], List[torch.nn.Parameter]] = {} 

360 for record in hsdp_assign.all_records: 

361 owner = hsdp_assign.owner_by_index.get(record.index) 

362 # Frozen records remain in all_records but have no assigned owner. 

363 # () is a valid owner when there are no replicate dimensions. 

364 if owner is None: 

365 continue 

366 params_by_owner.setdefault(owner, []).append(record.param) 

367 batches_by_owner = { 

368 owner: list(self._iter_unshard_batches(params_by_owner[owner])) 

369 for owner in sorted(params_by_owner) 

370 } 

371 batch_count = max((len(batches) for batches in batches_by_owner.values()), default=1) 

372 local_batches = batches_by_owner.get(hsdp_assign.replicate_group_ranks, []) 

373 units = [] 

374 for index in range(batch_count): 

375 logical_params = [ 

376 param for batches in batches_by_owner.values() if index < len(batches) 

377 for param in batches[index] 

378 ] 

379 local_params = local_batches[index] if index < len(local_batches) else [] 

380 post_update_params = hsdp_assign.owned_params if index == batch_count - 1 else () 

381 units.append(self._make_unit( 

382 (group_index, _HSDP, assignment_index, index), 

383 logical_params, local_params, hsdp_assign, 

384 post_update_params=post_update_params, 

385 flushes_broadcast=index == batch_count - 1, 

386 )) 

387 return units 

388 

389 def _ensure_momentum_state(self, param: torch.nn.Parameter) -> None: 

390 """Create the host-resident momentum slot for one parameter. 

391 

392 First-step state is created host-side on purpose: the design requires 

393 that a freshly created momentum buffer never becomes device-resident 

394 outside its own update unit. Parameters without a gradient keep Muon's 

395 original skip semantics and get no slot at all. 

396 

397 The device shell published into ``state[param]`` is what 

398 ``Muon._update_muon_momentum`` reads during this step. That method only 

399 allocates momentum when the key is missing, so a registered slot is 

400 written in place and never replaced. 

401 """ 

402 if self._runtime_local_tensor(param.grad).layout != torch.strided: 

403 raise ValueError("Muon swap optimizer only supports dense gradients.") 

404 state = self.optimizer.state[param] 

405 for key in self._configured_state_keys(): 

406 if (id(param), key) in self._slots: 

407 continue 

408 device_tensor = self.runtime.make_empty_state_like(param) 

409 if not self._runtime_local_tensor(device_tensor).is_contiguous(): 

410 raise ValueError("Muon swap optimizer requires contiguous momentum state.") 

411 state[key] = device_tensor 

412 slot = self._make_slot(key, device_tensor) 

413 if not slot.swappable: 

414 # Too small or otherwise ineligible: keep ordinary device state. 

415 device_tensor.zero_() 

416 self._slots[(id(param), key)] = slot 

417 continue 

418 slot.cpu_tensor = self.runtime.make_zero_cpu_tensor_like(param) 

419 slot.state = "host" 

420 self._slots[(id(param), key)] = slot 

421 self.runtime.release_device_storage(slot) 

422 

423 def _slots_for_params(self, params: Iterable[torch.nn.Parameter]) -> List[SwapSlot]: 

424 """Return the registered momentum slots for ``params``, in order. 

425 

426 Only slots the adapter already owns are returned: they carry the swap 

427 state (host mirror, released device storage, pending event) that the 

428 runtime manipulates, so rebuilding a slot here would drop that state. 

429 Momentum state materialized before the wrapper was created is registered 

430 separately by :meth:`initial_slots`. 

431 """ 

432 keys = self._configured_state_keys() 

433 slots = [] 

434 seen = set() 

435 for param in params: 

436 for key in keys: 

437 slot = self._slots.get((id(param), key)) 

438 if slot is None or id(slot) in seen: 

439 continue 

440 slots.append(slot) 

441 seen.add(id(slot)) 

442 return slots 

443 

444 # ------------------------------------------------------------------ execution 

445 def iter_update_units(self, step_context: Dict[str, Any]) -> List[MuonSwapUnit]: 

446 """Return the units built by :meth:`prepare_step`.""" 

447 return step_context["units"] 

448 

449 def partition_costs(self, units: Sequence[MuonSwapUnit], step_context: Any) -> Sequence[int]: 

450 """Return logical momentum bytes without consulting local slots or gradients.""" 

451 del step_context 

452 return [unit.logical_nbytes for unit in units] 

453 

454 def step_batch(self, batch: List[MuonSwapUnit], step_context: Dict[str, Any]) -> None: 

455 """Run one prefetched batch of Muon update units. 

456 

457 The runtime has already waited for this batch's H2D before calling in, 

458 so every slot the batch reads is device-resident here. 

459 """ 

460 broadcaster = step_context["broadcaster"] 

461 for unit in batch: 

462 self._run_unit(unit, broadcaster) 

463 

464 def _run_unit(self, unit: MuonSwapUnit, broadcaster: AsyncReplicateBroadcaster) -> None: 

465 """Run one atomic update unit with the matching Muon method.""" 

466 kind = unit.kind 

467 if kind not in (_NO_COMM, _HSDP): 

468 raise ValueError(f"Unknown Muon swap unit kind {kind!r}.") 

469 group = self.optimizer.param_groups[unit.group_index] 

470 ns_inputs = self.optimizer._update_muon_momentum(group, unit.params) 

471 if kind == _NO_COMM: 

472 self._apply_unshard_update(group, ns_inputs) 

473 else: 

474 hsdp_assign = unit.hsdp_assign 

475 if hsdp_assign is None: 

476 raise RuntimeError("Muon swap HSDP unit is missing its group assignment.") 

477 if hsdp_assign.is_shard: 

478 self.optimizer._process_shard_params( 

479 group, ns_inputs, [hsdp_assign], unit.group_index, 

480 buffer_cache={}, 

481 ) 

482 else: 

483 self._apply_unshard_update(group, ns_inputs) 

484 if self.optimizer.post_update_fn is not None and unit.post_update_params: 

485 self.optimizer._run_post_update_fn(group, unit.post_update_params) 

486 # Flush at the global assignment boundary, after owner callbacks, even 

487 # when this rank has no local updates in the last logical unit. 

488 if unit.flushes_broadcast: 

489 broadcaster.flush_group(unit.hsdp_assign) 

490 

491 def _apply_unshard_update( 

492 self, 

493 group: Dict[str, Any], 

494 ns_inputs: Dict[torch.nn.Parameter, torch.Tensor], 

495 ) -> None: 

496 """Run the unsharded NS and parameter update for one unit.""" 

497 if not ns_inputs: 

498 return 

499 # Muon's own helper re-groups by shape and re-splits into memory-safe 

500 # batches; a unit's params are already one such batch, so the split is a 

501 # single group and the boundary cannot drift. 

502 self.optimizer._process_unshard_params(group, ns_inputs) 

503 

504 def finish_step(self, step_context: Any) -> None: 

505 """Wait for every parameter broadcast issued during this step.""" 

506 step_context["broadcaster"].wait_all() 

507 

508 

509def build_muon_swap_adapter(optimizer: Any, config: Any, runtime: Any) -> MuonSwapAdapter: 

510 """Build the Muon swap adapter for ``optimizer``. 

511 

512 Args: 

513 optimizer: Muon optimizer instance. 

514 config: Resolved ``SwapOptimizerConfig``. 

515 runtime: Shared ``PipelineSwapRuntime``. 

516 

517 Returns: 

518 The Muon swap adapter. 

519 

520 Raises: 

521 ValueError: If ``optimizer`` is not a Muon optimizer. 

522 """ 

523 if not isinstance(optimizer, Muon): 

524 raise ValueError( 

525 "Swap muon only supports hyper_parallel.core.optimizer.muon.Muon. " 

526 f"Got {type(optimizer)!r}." 

527 ) 

528 return MuonSwapAdapter(optimizer, config, runtime) 

529 

530 

531def swap_muon(optimizer: Any, config: Any) -> Any: 

532 """Wrap a Muon optimizer with momentum-only state swap. 

533 

534 Args: 

535 optimizer: Muon optimizer instance. 

536 config: Resolved ``SwapOptimizerConfig``. Packed swap preserves Muon's 

537 logical units and only coalesces their momentum transfers. 

538 

539 Returns: 

540 The shared core swap-optimizer wrapper. 

541 """ 

542 return CoreSwapOptimizer( 

543 optimizer, 

544 config, 

545 adapter_factory=build_muon_swap_adapter, 

546 )