Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / swap_muon.py: 95%
234 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Muon momentum-only optimizer-state swap adapter."""
17# pylint: disable=protected-access
19from __future__ import annotations
21from dataclasses import dataclass
22import math
23from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple
25import torch
27from hyper_parallel.core.optimizer.muon import Muon
28from hyper_parallel.core.optimizer.optimizer import AsyncReplicateBroadcaster
29from hyper_parallel.core.optimizer.sharding_category import HSDPGroupAssignment
30from hyper_parallel.core.optimizer.swap_optimizer_base import (
31 StateSwapAdapter,
32 SwapOptimizer as CoreSwapOptimizer,
33 SwapSlot,
34)
36#: Muon's only persistent optimizer state that participates in swap.
37MUON_STATE_KEYS = ("momentum_buffer",)
39_NO_COMM = "no_comm"
40_HSDP = "hsdp"
43@dataclass
44class MuonSwapUnit:
45 """One atomic Muon update unit and the momentum slots it needs.
47 A unit is never split by the pipeline: its parameters run through momentum,
48 Newton-Schulz and the parameter update as one indivisible piece of work, so
49 a single matrix is never partially swapped. ``kind`` only selects which
50 Muon execution path consumes the unit, and ``hsdp_assign`` keeps the
51 original HSDP communication boundary intact.
53 Attributes:
54 group_index: Index of the owning parameter group, derived from
55 ``schedule_key``.
56 kind: ``"no_comm"`` for locally orthonormalized params, ``"hsdp"`` for
57 an HSDP shard/replicate assignment. Derived from ``schedule_key``.
58 params: Parameters this unit updates, in schedule order.
59 slots: Swappable momentum-buffer slots for ``params``.
60 schedule_key: Rank-independent group, kind, assignment and sub-batch indices.
61 logical_nbytes: Full logical momentum bytes, independent of local ownership.
62 hsdp_assign: The original assignment for ``"hsdp"`` units, else ``None``.
63 post_update_params: Parameters whose post-update callbacks run after this
64 unit. Only the final unit of the original Muon update boundary owns
65 this tuple, in the original parameter order.
66 flushes_broadcast: Whether this unit commits its assignment's parameter
67 broadcast. Only the last unit of an assignment sets it, so an
68 assignment split into several NS batches still issues the single
69 flush the bare schedule issues.
70 """
72 params: List[torch.nn.Parameter]
73 slots: List[SwapSlot]
74 schedule_key: Tuple[int, str, int, int]
75 logical_nbytes: int
76 hsdp_assign: Optional[HSDPGroupAssignment] = None
77 post_update_params: Tuple[torch.nn.Parameter, ...] = ()
78 flushes_broadcast: bool = False
80 @property
81 def group_index(self) -> int:
82 """Return the owning parameter-group index from the schedule identity."""
83 return self.schedule_key[0]
85 @property
86 def kind(self) -> str:
87 """Return the execution-path kind from the schedule identity."""
88 return self.schedule_key[1]
91class MuonSwapAdapter(StateSwapAdapter):
92 """Swap adapter that offloads and prefetches Muon's momentum buffers.
94 The adapter is the swap *coordinator* for Muon, never a second Muon
95 implementation. It replays the schedule of the unmodified ``Muon`` and
96 calls that optimizer's existing execution methods -- momentum,
97 Newton-Schulz, all-gather, broadcast and the parameter write-back all stay
98 in ``muon.py`` as the single source of truth. What the adapter owns is
99 (a) building the atomic update-unit schedule in the original order, and
100 (b) making sure the momentum buffer each unit reads is materialized on the
101 device only for the duration of that unit.
102 """
104 @staticmethod
105 def default_state_keys() -> Tuple[str, ...]:
106 """Return the swap state keys a Muon optimizer owns."""
107 return MUON_STATE_KEYS
109 def validate(self) -> None:
110 """Reject configurations the momentum-only swap path cannot preserve.
112 Raises:
113 ValueError: If a configured state key is not one Muon owns, or a
114 parameter holds momentum state the runtime cannot rebind, or a
115 custom momentum update would lose its original call boundary.
116 """
117 self._validate_configured_state_keys()
118 if self.optimizer.momentum_update_fn is not None:
119 raise ValueError(
120 "Muon swap optimizer does not support momentum_update_fn because "
121 "state partitioning changes its call boundary."
122 )
123 for group in self.optimizer.param_groups:
124 for param in group["params"]:
125 self._validate_param(param)
127 def _validate_configured_state_keys(self) -> None:
128 """Reject a configured swap key that is not Muon state.
130 ``_configured_state_keys`` shares a vocabulary with the other families
131 and silently drops keys none of them implement, so Muon has to check its
132 own selection explicitly: asking to swap ``exp_avg`` on a Muon optimizer
133 must fail rather than quietly swap nothing.
135 Raises:
136 ValueError: If the configured selection names a foreign state key.
137 """
138 if self.config.state_keys is None:
139 return
140 unavailable = sorted(set(self.config.state_keys) - set(self.default_state_keys()))
141 if unavailable:
142 raise ValueError(
143 f"Requested state keys {unavailable} are not available for {type(self.optimizer)!r}; "
144 f"Muon only swaps {self.default_state_keys()}."
145 )
147 def _validate_param(self, param: torch.nn.Parameter) -> None:
148 """Validate one parameter's momentum state is swap-compatible."""
149 state = self.optimizer.state.get(param)
150 if not state:
151 return
152 buffer = state.get("momentum_buffer")
153 if buffer is None:
154 return
155 if getattr(param, "grad", None) is not None and getattr(param.grad, "is_sparse", False):
156 raise ValueError("Muon swap optimizer only supports dense gradients.")
157 local = self._runtime_local_tensor(buffer)
158 if not isinstance(local, torch.Tensor):
159 raise ValueError(
160 f"Muon swap optimizer requires tensor momentum state, got {type(buffer)!r}."
161 )
162 if local.layout != torch.strided:
163 raise ValueError("Muon swap optimizer requires dense momentum state.")
164 if not local.is_contiguous():
165 raise ValueError("Muon swap optimizer requires contiguous momentum state.")
167 @staticmethod
168 def _runtime_local_tensor(tensor: Any) -> Any:
169 """Return the local shard of a possibly distributed tensor."""
170 to_local = getattr(tensor, "to_local", None)
171 return to_local() if callable(to_local) else tensor
173 def _scheduled_param_ids(self) -> Optional[set[int]]:
174 """Return parameters this rank owns in the current Muon schedule.
176 ``None`` means the optimizer has no HSDP schedule and therefore all
177 parameters are local. With HSDP metadata present, only no-comm
178 parameters and assignment owners may carry Muon state on this rank.
179 """
180 schedule = getattr(self.optimizer, "_hsdp_assignment_batches", None)
181 # ``Muon`` initialises this as an empty dict in non-HSDP runs, so an
182 # empty schedule carries no ownership metadata at all: every parameter
183 # is local and pre-existing momentum state must be kept.
184 if not schedule:
185 return None
186 scheduled: set[int] = set()
187 for group_index, group in enumerate(self.optimizer.param_groups):
188 info = schedule.get(group_index)
189 if info is None:
190 scheduled.update(id(param) for param in group["params"])
191 continue
192 scheduled.update(id(param) for param in info.get("no_comm", []))
193 for batch_group in info.get("batch_groups", []):
194 for assignment in batch_group.get("sub_batches", []):
195 scheduled.update(id(param) for param in assignment.owned_params)
196 return scheduled
198 def _params_with_local_state(self, params: Iterable[Any]) -> Iterable[Any]:
199 """Keep checkpoint-restored momentum only for local schedule owners."""
200 scheduled_ids = self._scheduled_param_ids()
201 if scheduled_ids is None:
202 return params
203 return tuple(param for param in params if id(param) in scheduled_ids)
205 def initial_slots(self) -> Iterable[SwapSlot]:
206 """Discover only local-owner momentum state before the first step."""
207 scheduled_ids = self._scheduled_param_ids()
208 slots = []
209 for group in self.optimizer.param_groups:
210 for param in group["params"]:
211 if scheduled_ids is not None and id(param) not in scheduled_ids:
212 # State may have been primed on every rank before this
213 # wrapper was created. Drop non-owner entries before the
214 # swap runtime publishes any slots.
215 self.optimizer.state.pop(param, None)
216 continue
217 state = self.optimizer.state.get(param)
218 if not state:
219 continue
220 for key in self._configured_state_keys():
221 if key in state and (id(param), key) not in self._slots:
222 self._slots[(id(param), key)] = self._make_slot(key, state[key])
223 slots.extend(self._slots_for_params([param]))
224 return tuple(slots)
226 # ------------------------------------------------------------------ schedule
227 def prepare_step(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
228 """Advance steps, register momentum slots and build the unit schedule.
230 The unit schedule mirrors ``Muon.step()`` exactly: the no-comm params of
231 every group first, then the flattened HSDP assignment schedule, keeping
232 the per-group-then-per-batch nesting so collective order across ranks is
233 unchanged.
235 Returns:
236 Step context holding the ordered units and the step's broadcaster.
238 Raises:
239 ValueError: If extra step arguments are passed.
240 """
241 if args or kwargs:
242 raise ValueError("Muon swap optimizer step does not support closure or extra arguments.")
243 for group in self.optimizer.param_groups:
244 group['step'] = (group.get('step') or 0) + 1
246 units: List[MuonSwapUnit] = []
247 num_groups = len(self.optimizer.param_groups)
248 for group_index in range(num_groups):
249 no_comm_params = self._no_comm_params(group_index)
250 no_comm_batches = list(self._iter_unshard_batches(no_comm_params))
251 for batch_index, params in enumerate(no_comm_batches):
252 post_update_params = no_comm_params if batch_index == len(no_comm_batches) - 1 else ()
253 units.append(self._make_unit(
254 (group_index, _NO_COMM, -1, batch_index), params, params,
255 post_update_params=post_update_params,
256 ))
258 linear_batches = [
259 self._hsdp_linear_batches(group_index) for group_index in range(num_groups)
260 ]
261 max_num_batches = max((len(batches) for batches in linear_batches), default=0)
262 for batch_index in range(max_num_batches):
263 for group_index in range(num_groups):
264 group_batches = linear_batches[group_index]
265 if batch_index >= len(group_batches):
266 continue
267 units.extend(self._make_hsdp_units(group_index, group_batches[batch_index], batch_index))
269 return {
270 "units": units,
271 "broadcaster": AsyncReplicateBroadcaster(self.optimizer),
272 }
274 def _no_comm_params(self, group_index: int) -> List[torch.nn.Parameter]:
275 """Return the params of ``group_index`` that need no HSDP collective.
277 This reads the same private schedule metadata ``Muon.step()`` reads, and
278 falls back to every parameter in the group when no HSDP assignment
279 metadata was built for it -- the same "no HSDP info means all params are
280 no_comm" rule the bare step applies.
281 """
282 info = getattr(self.optimizer, "_hsdp_assignment_batches", {}).get(group_index)
283 if not info:
284 return list(self.optimizer.unshard_params_by_group.get(group_index, []))
285 return list(info.get("no_comm", []))
287 def _hsdp_linear_batches(self, group_index: int) -> List[HSDPGroupAssignment]:
288 """Flatten one group's nested HSDP schedule into a linear batch list."""
289 info = getattr(self.optimizer, "_hsdp_assignment_batches", {}).get(group_index)
290 linear_batches: List[HSDPGroupAssignment] = []
291 if info:
292 for batch_group in info.get("batch_groups", []):
293 linear_batches.extend(batch_group.get("sub_batches", []))
294 return linear_batches
296 def _iter_unshard_batches(
297 self,
298 params: List[torch.nn.Parameter],
299 ) -> Iterator[List[torch.nn.Parameter]]:
300 """Plan logical batches using Muon's existing shape and memory rules."""
301 for shape_group in self.optimizer._group_by_shape(params).values():
302 yield from self.optimizer._split_into_memory_safe_batches(shape_group, shard_size=1)
304 def _make_unit(
305 self,
306 schedule_key: Tuple[int, str, int, int],
307 logical_params: Iterable[torch.nn.Parameter],
308 params: Iterable[torch.nn.Parameter],
309 hsdp_assign: Optional[HSDPGroupAssignment] = None,
310 post_update_params: Iterable[torch.nn.Parameter] = (),
311 flushes_broadcast: bool = False,
312 ) -> MuonSwapUnit:
313 """Project a logical unit onto local gradient-bearing owners and slots."""
314 logical_nbytes = sum(
315 math.prod(param.shape) * self._runtime_local_tensor(param).element_size()
316 for param in logical_params
317 )
318 params = [param for param in params if param.grad is not None]
319 post_update_params = tuple(
320 param for param in post_update_params if param.grad is not None
321 )
322 # HSDP assignments contain every logical parameter, but only the local
323 # owner participates in this rank's momentum update. Register state
324 # only for parameters that are actually present in this atomic unit;
325 # otherwise a non-owner would retain a device/host slot indefinitely.
326 for param in params:
327 self._ensure_momentum_state(param)
328 return MuonSwapUnit(
329 params=params,
330 slots=self._slots_for_params(params),
331 schedule_key=schedule_key,
332 logical_nbytes=logical_nbytes,
333 hsdp_assign=hsdp_assign,
334 post_update_params=post_update_params,
335 flushes_broadcast=flushes_broadcast,
336 )
338 def _make_hsdp_units(
339 self,
340 group_index: int,
341 hsdp_assign: HSDPGroupAssignment,
342 assignment_index: int,
343 ) -> List[MuonSwapUnit]:
344 """Build the units for one HSDP assignment, in assignment order.
346 Sharded assignments stay atomic because the shard executor consumes the
347 entire assignment. For unsharded assignments, every rank derives all
348 owners' NS batches before selecting its own local batch.
349 """
350 if hsdp_assign.is_shard:
351 return [
352 self._make_unit(
353 (group_index, _HSDP, assignment_index, 0),
354 hsdp_assign.all_params, hsdp_assign.owned_params,
355 hsdp_assign, post_update_params=hsdp_assign.owned_params,
356 flushes_broadcast=True,
357 )
358 ]
359 params_by_owner: Dict[Tuple[int, ...], List[torch.nn.Parameter]] = {}
360 for record in hsdp_assign.all_records:
361 owner = hsdp_assign.owner_by_index.get(record.index)
362 # Frozen records remain in all_records but have no assigned owner.
363 # () is a valid owner when there are no replicate dimensions.
364 if owner is None:
365 continue
366 params_by_owner.setdefault(owner, []).append(record.param)
367 batches_by_owner = {
368 owner: list(self._iter_unshard_batches(params_by_owner[owner]))
369 for owner in sorted(params_by_owner)
370 }
371 batch_count = max((len(batches) for batches in batches_by_owner.values()), default=1)
372 local_batches = batches_by_owner.get(hsdp_assign.replicate_group_ranks, [])
373 units = []
374 for index in range(batch_count):
375 logical_params = [
376 param for batches in batches_by_owner.values() if index < len(batches)
377 for param in batches[index]
378 ]
379 local_params = local_batches[index] if index < len(local_batches) else []
380 post_update_params = hsdp_assign.owned_params if index == batch_count - 1 else ()
381 units.append(self._make_unit(
382 (group_index, _HSDP, assignment_index, index),
383 logical_params, local_params, hsdp_assign,
384 post_update_params=post_update_params,
385 flushes_broadcast=index == batch_count - 1,
386 ))
387 return units
389 def _ensure_momentum_state(self, param: torch.nn.Parameter) -> None:
390 """Create the host-resident momentum slot for one parameter.
392 First-step state is created host-side on purpose: the design requires
393 that a freshly created momentum buffer never becomes device-resident
394 outside its own update unit. Parameters without a gradient keep Muon's
395 original skip semantics and get no slot at all.
397 The device shell published into ``state[param]`` is what
398 ``Muon._update_muon_momentum`` reads during this step. That method only
399 allocates momentum when the key is missing, so a registered slot is
400 written in place and never replaced.
401 """
402 if self._runtime_local_tensor(param.grad).layout != torch.strided:
403 raise ValueError("Muon swap optimizer only supports dense gradients.")
404 state = self.optimizer.state[param]
405 for key in self._configured_state_keys():
406 if (id(param), key) in self._slots:
407 continue
408 device_tensor = self.runtime.make_empty_state_like(param)
409 if not self._runtime_local_tensor(device_tensor).is_contiguous():
410 raise ValueError("Muon swap optimizer requires contiguous momentum state.")
411 state[key] = device_tensor
412 slot = self._make_slot(key, device_tensor)
413 if not slot.swappable:
414 # Too small or otherwise ineligible: keep ordinary device state.
415 device_tensor.zero_()
416 self._slots[(id(param), key)] = slot
417 continue
418 slot.cpu_tensor = self.runtime.make_zero_cpu_tensor_like(param)
419 slot.state = "host"
420 self._slots[(id(param), key)] = slot
421 self.runtime.release_device_storage(slot)
423 def _slots_for_params(self, params: Iterable[torch.nn.Parameter]) -> List[SwapSlot]:
424 """Return the registered momentum slots for ``params``, in order.
426 Only slots the adapter already owns are returned: they carry the swap
427 state (host mirror, released device storage, pending event) that the
428 runtime manipulates, so rebuilding a slot here would drop that state.
429 Momentum state materialized before the wrapper was created is registered
430 separately by :meth:`initial_slots`.
431 """
432 keys = self._configured_state_keys()
433 slots = []
434 seen = set()
435 for param in params:
436 for key in keys:
437 slot = self._slots.get((id(param), key))
438 if slot is None or id(slot) in seen:
439 continue
440 slots.append(slot)
441 seen.add(id(slot))
442 return slots
444 # ------------------------------------------------------------------ execution
445 def iter_update_units(self, step_context: Dict[str, Any]) -> List[MuonSwapUnit]:
446 """Return the units built by :meth:`prepare_step`."""
447 return step_context["units"]
449 def partition_costs(self, units: Sequence[MuonSwapUnit], step_context: Any) -> Sequence[int]:
450 """Return logical momentum bytes without consulting local slots or gradients."""
451 del step_context
452 return [unit.logical_nbytes for unit in units]
454 def step_batch(self, batch: List[MuonSwapUnit], step_context: Dict[str, Any]) -> None:
455 """Run one prefetched batch of Muon update units.
457 The runtime has already waited for this batch's H2D before calling in,
458 so every slot the batch reads is device-resident here.
459 """
460 broadcaster = step_context["broadcaster"]
461 for unit in batch:
462 self._run_unit(unit, broadcaster)
464 def _run_unit(self, unit: MuonSwapUnit, broadcaster: AsyncReplicateBroadcaster) -> None:
465 """Run one atomic update unit with the matching Muon method."""
466 kind = unit.kind
467 if kind not in (_NO_COMM, _HSDP):
468 raise ValueError(f"Unknown Muon swap unit kind {kind!r}.")
469 group = self.optimizer.param_groups[unit.group_index]
470 ns_inputs = self.optimizer._update_muon_momentum(group, unit.params)
471 if kind == _NO_COMM:
472 self._apply_unshard_update(group, ns_inputs)
473 else:
474 hsdp_assign = unit.hsdp_assign
475 if hsdp_assign is None:
476 raise RuntimeError("Muon swap HSDP unit is missing its group assignment.")
477 if hsdp_assign.is_shard:
478 self.optimizer._process_shard_params(
479 group, ns_inputs, [hsdp_assign], unit.group_index,
480 buffer_cache={},
481 )
482 else:
483 self._apply_unshard_update(group, ns_inputs)
484 if self.optimizer.post_update_fn is not None and unit.post_update_params:
485 self.optimizer._run_post_update_fn(group, unit.post_update_params)
486 # Flush at the global assignment boundary, after owner callbacks, even
487 # when this rank has no local updates in the last logical unit.
488 if unit.flushes_broadcast:
489 broadcaster.flush_group(unit.hsdp_assign)
491 def _apply_unshard_update(
492 self,
493 group: Dict[str, Any],
494 ns_inputs: Dict[torch.nn.Parameter, torch.Tensor],
495 ) -> None:
496 """Run the unsharded NS and parameter update for one unit."""
497 if not ns_inputs:
498 return
499 # Muon's own helper re-groups by shape and re-splits into memory-safe
500 # batches; a unit's params are already one such batch, so the split is a
501 # single group and the boundary cannot drift.
502 self.optimizer._process_unshard_params(group, ns_inputs)
504 def finish_step(self, step_context: Any) -> None:
505 """Wait for every parameter broadcast issued during this step."""
506 step_context["broadcaster"].wait_all()
509def build_muon_swap_adapter(optimizer: Any, config: Any, runtime: Any) -> MuonSwapAdapter:
510 """Build the Muon swap adapter for ``optimizer``.
512 Args:
513 optimizer: Muon optimizer instance.
514 config: Resolved ``SwapOptimizerConfig``.
515 runtime: Shared ``PipelineSwapRuntime``.
517 Returns:
518 The Muon swap adapter.
520 Raises:
521 ValueError: If ``optimizer`` is not a Muon optimizer.
522 """
523 if not isinstance(optimizer, Muon):
524 raise ValueError(
525 "Swap muon only supports hyper_parallel.core.optimizer.muon.Muon. "
526 f"Got {type(optimizer)!r}."
527 )
528 return MuonSwapAdapter(optimizer, config, runtime)
531def swap_muon(optimizer: Any, config: Any) -> Any:
532 """Wrap a Muon optimizer with momentum-only state swap.
534 Args:
535 optimizer: Muon optimizer instance.
536 config: Resolved ``SwapOptimizerConfig``. Packed swap preserves Muon's
537 logical units and only coalesces their momentum transfers.
539 Returns:
540 The shared core swap-optimizer wrapper.
541 """
542 return CoreSwapOptimizer(
543 optimizer,
544 config,
545 adapter_factory=build_muon_swap_adapter,
546 )