Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / muon.py: 70%
626 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
16"""Muon optimizer with HSDP shard-group-aware communication."""
18import math
19from collections.abc import Callable, Iterable
20from collections import defaultdict
21from dataclasses import dataclass
22import logging
23from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
25import torch
26import torch.distributed as dist
28from hyper_parallel.core.optimizer.optimizer import AsyncReplicateBroadcaster, BaseDistributedOptimizer
29from hyper_parallel.core.optimizer.dtensor_compat import to_local_if_dtensor
30from hyper_parallel.core.optimizer.sharding_category import (
31 HSDPGroupAssignment,
32 ParamShardMeta,
33 build_owner_by_size,
34)
35from hyper_parallel.core.optimizer.muon_shard import (
36 _debug_param_shard_metadata,
37 build_pad_ns_inputs,
38 build_param_shard_metadata_for_group,
39 chunk_update_by_layout,
40 fused_allgather_dtensor_params,
41)
43logger = logging.getLogger(__name__)
45# Legacy: single-coefficient quintic NS (Keller Jordan / Moonlight).
46# Same (a, b, c) applied every step; typically 5 steps.
47_NS_LEGACY_COEFFS: Tuple[float, float, float] = (3.4445, -4.7750, 2.0315)
49# Asym5: 5-step asymmetric NS (dion). Each step uses different coefficients.
50_NS_ASYM5_COEFFS: Tuple[Tuple[float, float, float], ...] = (
51 (4.0848, -6.8946, 2.9270),
52 (3.9505, -6.3029, 2.6377),
53 (3.7418, -5.5913, 2.3037),
54 (2.8769, -3.1427, 1.2046),
55 (2.8366, -3.0525, 1.2012),
56)
59@dataclass
60class NSInputTransform:
61 """Prepared NS tensors and the callback that restores their updates."""
63 tensors: List[torch.Tensor]
64 restore: Callable[[List[torch.Tensor], torch.Tensor], None]
67@dataclass(frozen=True)
68class MuonPostUpdateContext:
69 """Context supplied to an optional parameter post-update callback."""
71 param_fqn: Optional[str]
72 logical_shape: Tuple[int, ...]
73 lr: float
74 weight_decay: float
75 step: int
78_MUON_ADVANCED_DEFAULTS: Dict[str, Any] = {
79 "ns_coefficients": None,
80 "ns_epsilon": 1e-10,
81 "zeropower_fn": None,
82 "momentum_update_fn": None,
83 "reshape_fn": None,
84 "ns_transform_fn": None,
85 "post_update_fn": None,
86 "zero_rms_scale_mode": "zero",
87 "apply_lr_in_update": False,
88}
91@dataclass(frozen=True)
92class _MuonAdvancedOptions:
93 """Validated low-frequency Muon options."""
95 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]]
96 ns_epsilon: float
97 zeropower_fn: Optional[Callable[..., torch.Tensor]]
98 momentum_update_fn: Optional[Callable[..., torch.Tensor]]
99 reshape_fn: Optional[Callable[..., Any]]
100 ns_transform_fn: Optional[Callable[..., Optional[NSInputTransform]]]
101 post_update_fn: Optional[Callable[..., None]]
102 zero_rms_scale_mode: str
103 apply_lr_in_update: bool
106def _resolve_muon_advanced_options(options: Dict[str, Any]) -> _MuonAdvancedOptions:
107 """Validate and fill low-frequency Muon keyword options."""
108 unknown = options.keys() - _MUON_ADVANCED_DEFAULTS.keys()
109 if unknown:
110 raise TypeError(f"Muon got unexpected keyword arguments: {', '.join(sorted(unknown))}")
111 resolved = dict(_MUON_ADVANCED_DEFAULTS)
112 resolved.update(options)
113 return _MuonAdvancedOptions(**resolved)
116_TransformNSState = Dict[str, Any]
119def _resolve_ns_coefficients(
120 steps: int,
121 ns_variant: str = "asym5",
122 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]] = None,
123) -> Tuple[Tuple[float, float, float], ...]:
124 """Resolve the coefficient prefix shared by native NS and backend construction."""
125 if ns_variant == "legacy":
126 return (_NS_LEGACY_COEFFS,) * steps
127 if ns_variant == "asym5":
128 return _NS_ASYM5_COEFFS[:steps]
129 if ns_variant == "custom":
130 if ns_coefficients is None:
131 raise ValueError("ns_coefficients must be provided when ns_variant='custom'")
132 if steps > len(ns_coefficients):
133 raise ValueError(
134 f"ns_steps={steps} exceeds the {len(ns_coefficients)} provided coefficient groups"
135 )
136 return tuple(tuple(row) for row in ns_coefficients[:steps])
137 raise ValueError(f"ns_variant must be 'legacy', 'asym5', or 'custom', got {ns_variant!r}")
140def zeropower_via_newtonschulz5(
141 ns_inputs: torch.Tensor,
142 steps: int,
143 ns_variant: str = "asym5",
144 epsilon: float = 1e-10,
145 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]] = None,
146) -> torch.Tensor:
147 """Newton-Schulz orthogonalization with preallocated matmul buffers.
149 Args:
150 ns_inputs: Matrix or matrix batch to orthogonalize.
151 steps: Requested iteration count; asym5 uses at most five iterations.
152 ns_variant: Built-in legacy/asym5 schedule or an explicit custom schedule.
153 epsilon: Normalization epsilon added to the input norm.
154 ns_coefficients: Coefficient triples required by the custom variant.
156 Returns:
157 Orthogonalized updates with the input shape, dtype and device.
158 """
159 mat_x = ns_inputs
160 transposed = ns_inputs.size(-2) > ns_inputs.size(-1)
161 if transposed:
162 mat_x = mat_x.mT
164 # Normalize input before Newton-Schulz iteration.
165 mat_x = mat_x / (mat_x.norm(dim=(-2, -1), keepdim=True) + epsilon)
167 n_size = mat_x.size(-2)
168 buf_a = torch.empty(mat_x.shape[:-2] + (n_size, n_size), dtype=mat_x.dtype, device=mat_x.device)
169 buf_b = torch.empty_like(buf_a)
170 buf_c = torch.empty(mat_x.shape, dtype=mat_x.dtype, device=mat_x.device)
172 step_coeffs = _resolve_ns_coefficients(steps, ns_variant, ns_coefficients)
174 for coeff_a, coeff_b, coeff_c in step_coeffs:
175 torch.matmul(mat_x, mat_x.mT, out=buf_a)
176 torch.matmul(buf_a, buf_a, out=buf_b)
177 buf_a.mul_(coeff_b)
178 buf_b.mul_(coeff_c)
179 buf_a.add_(buf_b)
180 torch.matmul(buf_a, mat_x, out=buf_c)
181 mat_x.mul_(coeff_a).add_(buf_c)
183 del buf_a, buf_b, buf_c
185 if transposed:
186 mat_x = mat_x.mT
188 return mat_x
191def compute_muon_slice_scale(
192 slice_tensor: torch.Tensor,
193 matched_adamw_rms: float,
194 zero_rms_scale_mode: str = "zero",
195) -> float:
196 """Compute Muon scale from the logical matrix dims of a reshaped slice."""
197 if not matched_adamw_rms:
198 return 1.0 if zero_rms_scale_mode == "use_lr" else 0.0
199 shape = tuple(slice_tensor.shape)
200 if len(shape) == 3 and shape[1] == 1:
201 logical_dims = (shape[0], shape[2])
202 else:
203 logical_dims = shape[-2:]
204 return math.sqrt(max(logical_dims)) * matched_adamw_rms
207class Muon(BaseDistributedOptimizer):
208 """Muon optimizer with HSDP shard-group-aware Newton-Schulz orthogonalization.
210 Implements the Muon optimizer which uses Newton-Schulz iteration for matrix
211 orthogonalization of gradient updates, with HSDP-aware communication for
212 sharded parameters.
213 """
215 ADDITIONAL_CONFIG_KEYS = set(_MUON_ADVANCED_DEFAULTS)
217 def __init__(
218 self,
219 params: Any,
220 lr: float = 2e-2,
221 weight_decay: float = 0.1,
222 matched_adamw_rms: float = 0.2,
223 momentum: Union[float, List[float], Tuple[float, ...]] = 0.95,
224 nesterov: bool = True,
225 ns_steps: int = 5,
226 ns_variant: str = "asym5",
227 hsdp_replica_count: Optional[Union[int, Tuple[int, ...]]] = None,
228 momentum_mode: str = "legacy",
229 **advanced_options: Any,
230 ) -> None:
231 """Initialize Muon and build parameter-identity runtime caches.
233 Args:
234 params: Parameters or parameter groups optimized by Muon.
235 lr: Learning rate.
236 weight_decay: Decoupled weight-decay coefficient.
237 matched_adamw_rms: RMS scale matched to an AdamW update.
238 momentum: Momentum coefficient or coefficient pair.
239 nesterov: Whether to use Nesterov momentum.
240 ns_steps: Newton-Schulz iteration count.
241 ns_variant: Newton-Schulz coefficient variant.
242 ns_coefficients: Optional custom Newton-Schulz coefficients.
243 ns_epsilon: Numerical stability term used during normalization.
244 zeropower_fn: Optional ``fn(local_tensor, steps=ns_steps)`` override.
245 Return a writable tensor with the same shape, dtype and device.
246 The callback owns its coefficient schedule and epsilon; these
247 group options are not forwarded. Recreate matching callback
248 configuration when restoring an optimizer checkpoint.
249 momentum_update_fn: Optional momentum update implementation override.
250 reshape_fn: Optional logical-matrix reshape function.
251 ns_transform_fn: Optional Newton-Schulz input transformation.
252 post_update_fn: Optional callback applied after a parameter update.
253 zero_rms_scale_mode: Scaling behavior when matched AdamW RMS is zero.
254 apply_lr_in_update: Whether the update callback applies the learning rate.
255 hsdp_replica_count: Optional optimizer-state replica group size.
256 momentum_mode: Momentum formula, either "legacy" or "vanilla".
257 Both modes support distinct update and buffer coefficients.
258 """
259 advanced = _resolve_muon_advanced_options(advanced_options)
260 if momentum_mode not in ("legacy", "vanilla"):
261 raise ValueError(
262 f"momentum_mode must be 'legacy' or 'vanilla', got {momentum_mode!r}"
263 )
264 if ns_variant not in ("legacy", "asym5", "custom"):
265 raise ValueError(
266 f"ns_variant must be 'legacy', 'asym5', or 'custom', got {ns_variant!r}"
267 )
268 if advanced.ns_epsilon < 0.0 or not math.isfinite(advanced.ns_epsilon):
269 raise ValueError(f"ns_epsilon must be a finite non-negative value, got {advanced.ns_epsilon}")
270 if advanced.zero_rms_scale_mode not in ("zero", "use_lr"):
271 raise ValueError(
272 "zero_rms_scale_mode must be 'zero' or 'use_lr', "
273 f"got {advanced.zero_rms_scale_mode!r}"
274 )
275 normalized_coefficients = self._validate_ns_coefficients(
276 ns_variant, ns_steps, advanced.ns_coefficients
277 )
278 if not isinstance(momentum, (list, tuple)):
279 momentum = [momentum]
280 if len(momentum) == 1:
281 momentum = (momentum[0], momentum[0])
282 defaults = {
283 "lr": lr,
284 "weight_decay": weight_decay,
285 "matched_adamw_rms": matched_adamw_rms,
286 "momentum": momentum,
287 "nesterov": nesterov,
288 "ns_steps": ns_steps,
289 "ns_variant": ns_variant,
290 "ns_coefficients": normalized_coefficients,
291 "ns_epsilon": advanced.ns_epsilon,
292 "zero_rms_scale_mode": advanced.zero_rms_scale_mode,
293 "apply_lr_in_update": advanced.apply_lr_in_update,
294 }
295 super().__init__(params, defaults, is_muon=True, hsdp_replica_count=hsdp_replica_count)
296 self.reshape_fn = advanced.reshape_fn
297 self.zeropower_fn = advanced.zeropower_fn
298 self.momentum_update_fn = advanced.momentum_update_fn
299 self._update_muon_momentum_mode = momentum_mode
300 self.ns_transform_fn = advanced.ns_transform_fn
301 self.post_update_fn = advanced.post_update_fn
302 self.reset_optimizer_parameters()
304 def reset_optimizer_parameters(self) -> None:
305 """Rebuild all parameter-identity caches from current param groups."""
306 self._group_dtensor_by_mesh()
307 self._build_param_shard_metadata()
308 deduced_count = self._auto_deduce_replica_count()
309 if deduced_count is None:
310 self.hsdp_replica_count = None
311 elif self.hsdp_replica_count is None:
312 self.hsdp_replica_count = deduced_count
313 self._split_replicate_groups()
314 self._build_hsdp_batch()
315 self._build_param_broadcast_info()
316 self._classify_parameters_for_step()
318 @staticmethod
319 def _validate_ns_coefficients(
320 ns_variant: str,
321 ns_steps: int,
322 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]],
323 ) -> Optional[Tuple[Tuple[float, float, float], ...]]:
324 """Validate and normalize externally supplied Newton-Schulz coefficients."""
325 if ns_variant != "custom":
326 if ns_coefficients is not None:
327 raise ValueError("ns_coefficients can only be provided when ns_variant='custom'")
328 return None
329 if not ns_coefficients:
330 raise ValueError("ns_coefficients must be provided when ns_variant='custom'")
331 if ns_steps > len(ns_coefficients):
332 raise ValueError(
333 f"ns_steps={ns_steps} exceeds the {len(ns_coefficients)} provided coefficient groups"
334 )
336 normalized = []
337 for index, coefficients in enumerate(ns_coefficients):
338 if len(coefficients) != 3:
339 raise ValueError(
340 f"ns_coefficients[{index}] must contain exactly three values, got {coefficients!r}"
341 )
342 normalized_coefficients = tuple(float(value) for value in coefficients)
343 if not all(math.isfinite(value) for value in normalized_coefficients):
344 raise ValueError(f"ns_coefficients[{index}] must contain only finite values")
345 normalized.append(normalized_coefficients)
346 return tuple(normalized)
348 def __str__(self) -> str:
349 """Return the optimizer representation."""
350 return super().__repr__()
352 __repr__ = __str__
354 @torch.no_grad()
355 def step(
356 self,
357 closure: Optional[Callable[[], Any]] = None,
358 ) -> Optional[float]:
359 """Perform a single optimization step.
361 De-duplication is controlled by the caller: ``param_to_ns_input`` should already contain only the owned
362 params (via ``hsdp_assign.owned_params``). The caller is responsible for broadcasting the updated params to
363 replica peers via ``AsyncReplicateBroadcaster.flush_group``.
365 Args:
366 closure: Optional callable that reevaluates the model and returns loss.
368 Returns:
369 The closure loss when provided, otherwise ``None``.
370 """
371 loss = None
372 if closure is not None:
373 with torch.enable_grad():
374 loss = closure()
375 broadcaster = AsyncReplicateBroadcaster(self)
377 num_groups = len(self.param_groups)
379 for group in self.param_groups:
380 group['step'] = (group.get('step') or 0) + 1
382 # Compute momentum only for no-comm params upfront.
383 no_comm_ns: Dict[int, Dict] = {}
384 for group_idx in range(num_groups):
385 info = self._hsdp_assignment_batches.get(group_idx)
386 if not info:
387 # No HSDP info — all params are no_comm.
388 unshard_params = self.unshard_params_by_group.get(group_idx, [])
389 no_comm_ns[group_idx] = self._update_muon_momentum(
390 self.param_groups[group_idx], unshard_params
391 )
392 else:
393 no_comm_params = info.get("no_comm", [])
394 if no_comm_params:
395 no_comm_ns[group_idx] = self._update_muon_momentum(
396 self.param_groups[group_idx], no_comm_params
397 )
398 else:
399 no_comm_ns[group_idx] = {}
401 # Process no-comm params.
402 for group_idx in range(num_groups):
403 if no_comm_ns[group_idx]:
404 group = self.param_groups[group_idx]
405 self._process_unshard_params(group, no_comm_ns[group_idx])
406 if self.post_update_fn is not None:
407 self._run_post_update_fn(group, no_comm_ns[group_idx].keys())
409 # Flatten nested batch into a linear schedule.
410 group_linear_batches: Dict[int, List[HSDPGroupAssignment]] = {}
411 max_num_batches = 0
412 for group_idx in range(num_groups):
413 info = self._hsdp_assignment_batches.get(group_idx)
414 linear_batches = []
415 if info:
416 for bg in info.get("batch_groups", []):
417 linear_batches.extend(bg.get("sub_batches", []))
418 group_linear_batches[group_idx] = linear_batches
419 max_num_batches = max(max_num_batches, len(linear_batches))
421 # Process batches: compute momentum per-batch for owned params only (HSDP de-duplication).
422 for batch_idx in range(max_num_batches):
423 for group_idx in range(num_groups):
424 linear_batches = group_linear_batches[group_idx]
425 if batch_idx >= len(linear_batches):
426 continue
428 hsdp_assign = linear_batches[batch_idx]
429 group = self.param_groups[group_idx]
431 # Compute momentum for this assignment's owned params only.
432 ns_inputs = self._update_muon_momentum(group, hsdp_assign.owned_params)
433 if not hsdp_assign.is_shard:
434 if ns_inputs:
435 self._process_unshard_params(group, ns_inputs)
436 else:
437 self._process_shard_params(
438 group, ns_inputs, [hsdp_assign], group_idx,
439 buffer_cache={},
440 )
441 if self.post_update_fn is not None:
442 self._run_post_update_fn(group, ns_inputs.keys())
444 # Flush broadcasts after each assignment.
445 broadcaster.flush_group(hsdp_assign)
447 broadcaster.wait_all()
448 return loss
450 def _run_post_update_fn(
451 self,
452 group: Dict[str, Any],
453 params: Iterable[torch.nn.Parameter],
454 ) -> None:
455 """Run the optional model callback on updated owner parameters before broadcast."""
456 if self.post_update_fn is None:
457 return
458 context_values = {
459 "lr": group["lr"],
460 "weight_decay": group["weight_decay"],
461 "step": group["step"],
462 }
463 for param in params:
464 local_tensor = to_local_if_dtensor(param.data)
465 context = MuonPostUpdateContext(
466 param_fqn=getattr(param, "model_name", None),
467 logical_shape=tuple(param.shape),
468 **context_values,
469 )
470 self.post_update_fn(param, local_tensor, context)
472 def _classify_parameters_for_step(self) -> None:
473 """Classify params by whether the last two dims are sharded.
475 unshard: run Newton-Schulz locally.
476 shard: all-gather before Newton-Schulz.
478 Reads from self._hsdp_assignment_batches which is organized by batch.
479 """
480 self.unshard_params_by_group: Dict[int, List] = {}
481 self.shard_params_by_group: Dict[int, List] = {}
482 self.shard_assignments_by_group: Dict[int, List[HSDPGroupAssignment]] = {}
483 # Per group: record.index -> shard coord that computes NS.
484 self._shard_compute_coord: Dict[int, Dict[int, Tuple[int, ...]]] = {}
486 for group_idx, group in enumerate(self.param_groups):
487 assignment_info = self._hsdp_assignment_batches.get(group_idx)
489 unshard_params = []
490 shard_hsdp_assignments: List[HSDPGroupAssignment] = []
492 if assignment_info:
493 unshard_params.extend(assignment_info["no_comm"])
494 for bg in assignment_info["batch_groups"]:
495 for hsdp_assign in bg["sub_batches"]:
496 if hsdp_assign.is_shard:
497 shard_hsdp_assignments.append(hsdp_assign)
498 else:
499 unshard_params.extend(hsdp_assign.owned_params)
500 else:
501 unshard_params.extend(group["params"])
503 # Shard params only include locally owned params; replica ownership is enforced at apply time.
504 shard_params = [p for a in shard_hsdp_assignments for p in a.owned_params]
506 # Greedily assign NS compute across shard ranks.
507 self._shard_compute_coord[group_idx] = {}
508 for hsdp_assign in shard_hsdp_assignments:
509 shard_sizes, _, _, _ = self._get_shard_info(hsdp_assign)
510 compute_by_index = build_owner_by_size(
511 records=hsdp_assign.owned_records,
512 replicate_sizes=shard_sizes,
513 )
514 self._shard_compute_coord[group_idx].update(compute_by_index)
516 self.unshard_params_by_group[group_idx] = unshard_params
517 self.shard_assignments_by_group[group_idx] = shard_hsdp_assignments
518 self.shard_params_by_group[group_idx] = shard_params
520 def _update_muon_momentum(
521 self,
522 group: Dict[str, Any],
523 params: List[torch.Tensor],
524 ) -> Dict[torch.nn.Parameter, torch.Tensor]:
525 """Compute first-order momentum and return bfloat16 NS inputs."""
526 momentum = group["momentum"]
527 if not isinstance(momentum, (list, tuple)):
528 momentum = [momentum]
529 if len(momentum) == 1:
530 momentum = (momentum[0], momentum[0])
531 momentum1, momentum2 = momentum
532 # Pre-filter params with valid grads and ensure momentum buffers exist
533 valid_params = []
534 grads = []
535 bufs = []
536 for p in params:
537 g = p.grad
538 if g is None:
539 continue
540 if "momentum_buffer" not in self.state[p]:
541 self.state[p]["momentum_buffer"] = torch.zeros_like(g)
542 valid_params.append(p)
543 grads.append(g)
544 bufs.append(self.state[p]["momentum_buffer"])
546 if not valid_params:
547 return {}
549 grads = [to_local_if_dtensor(grad) for grad in grads]
550 bufs = [to_local_if_dtensor(buffer) for buffer in bufs]
551 if self.momentum_update_fn is not None:
552 custom_updates = list(self.momentum_update_fn(grads, bufs, momentum1, momentum2, group["nesterov"]))
553 if len(custom_updates) != len(valid_params):
554 raise ValueError(
555 "momentum_update_fn must return one update tensor per input gradient, "
556 f"got {len(custom_updates)} updates for {len(valid_params)} gradients"
557 )
558 custom_updates = [update.to(torch.bfloat16) for update in custom_updates]
559 param_updates = dict(zip(valid_params, custom_updates))
560 return param_updates
562 if self._update_muon_momentum_mode == "vanilla":
563 # Match vanilla implementation in Muon paper & PyTorch implementation:
564 # m_for_update = (1 - momentum1) * grad + momentum1 * m_old
565 # m_new = (1 - momentum2) * grad + momentum2 * m_old
566 # Fold the Nesterov interpolation into one lerp with momentum1 squared.
567 # pylint: disable=protected-access
568 update_momentum = momentum1 ** 2 if group["nesterov"] else momentum1
569 local_us = torch._foreach_lerp(grads, bufs, weight=update_momentum)
570 torch._foreach_lerp_(bufs, grads, weight=1 - momentum2)
571 elif self._update_muon_momentum_mode == "legacy":
572 # Match muon_update_core():
573 # m_for_update = grad + momentum1 * m_old
574 # m_new = grad + momentum2 * m_old
575 # pylint: disable=protected-access
576 local_us = torch._foreach_add(grads, bufs, alpha=momentum1)
578 torch._foreach_mul_(bufs, momentum2)
579 torch._foreach_add_(bufs, grads)
581 if group["nesterov"]:
582 torch._foreach_mul_(local_us, momentum1)
583 torch._foreach_add_(local_us, grads)
584 else:
585 raise ValueError(
586 "Muon._update_muon_momentum_mode must be 'vanilla' or 'legacy', "
587 f"got {self._update_muon_momentum_mode!r}"
588 )
590 if local_us[0].dtype != torch.bfloat16:
591 local_us = [update.to(torch.bfloat16) for update in local_us]
593 param_updates = dict(zip(valid_params, local_us))
594 return param_updates
596 def _process_unshard_params(
597 self,
598 group: Dict[str, Any],
599 param_to_ns_input: Dict[torch.nn.Parameter, torch.Tensor],
600 ) -> None:
601 """Process un-sharded params: shape-group -> memory-batch -> NS -> local update."""
602 lr = group["lr"]
603 weight_decay = group["weight_decay"]
605 # empty params are not processed in muon, for uneven shard
606 param_to_ns_input = {
607 p: ns_input for p, ns_input in param_to_ns_input.items() if ns_input.numel() > 0
608 }
609 if not param_to_ns_input:
610 return
612 shape_groups = self._group_by_shape(list(param_to_ns_input.keys()))
613 for _, p_list in shape_groups.items():
614 safe_batches = self._split_into_memory_safe_batches(p_list, shard_size=1)
616 for sub_batch in safe_batches:
617 updates_dict = self._compute_batched_ns_updates(
618 sub_batch, param_to_ns_input, group, no_shard=True
619 )
621 local_params = [to_local_if_dtensor(p.data) for p in sub_batch]
622 local_updates = [updates_dict[p].view(lp.shape) for p, lp in zip(sub_batch, local_params)]
624 if weight_decay != 0.0:
625 # pylint: disable=protected-access
626 torch._foreach_mul_(local_params, 1 - lr * weight_decay)
627 # pylint: disable=protected-access
628 apply_alpha = 1.0 if group["apply_lr_in_update"] else -lr
629 torch._foreach_add_(local_params, local_updates, alpha=apply_alpha)
631 def _gather_and_compute_shard_updates(
632 self,
633 valid_params: List[torch.nn.Parameter],
634 param_to_ns_input: Dict[torch.nn.Parameter, torch.Tensor],
635 hsdp_assign: HSDPGroupAssignment,
636 shard_compute_coord: Dict[int, Tuple[int, ...]],
637 group: Dict[str, Any],
638 buffer_cache: Optional[Dict],
639 ) -> Tuple[
640 Dict[torch.nn.Parameter, torch.Tensor],
641 Dict[torch.nn.Parameter, Tuple[int, ...]],
642 ]:
643 """Gather NS inputs and compute updates for locally-assigned shard params.
645 Returns:
646 (my_updates, param_compute_coord) for this HSDP assignment.
647 """
648 shard_sizes, local_coords, shard_pgs, total_shard_size = self._get_shard_info(hsdp_assign)
649 param_to_index = {record.param: record.index for record in hsdp_assign.owned_records}
651 my_params: List[torch.nn.Parameter] = []
652 my_param_ids: set = set()
653 param_compute_coord: Dict[torch.nn.Parameter, Tuple[int, ...]] = {}
654 my_indices: set = set()
656 for idx, p in enumerate(valid_params):
657 p_index = param_to_index[p]
658 compute_coord = shard_compute_coord.get(p_index, (0,) * len(shard_sizes))
659 param_compute_coord[p] = compute_coord
660 if compute_coord == local_coords:
661 my_params.append(p)
662 my_param_ids.add(id(p))
663 my_indices.add(idx)
665 # Fused all-gather full NS inputs; keep only tensors computed locally.
666 gathered_inputs: Dict[torch.nn.Parameter, torch.Tensor] = {}
667 local_inputs = build_pad_ns_inputs(
668 valid_params,
669 param_to_ns_input,
670 self._param_shard_metadata,
671 )
672 param_shard_metadata = [self._param_shard_metadata.get(p) for p in valid_params]
673 gathered_list = fused_allgather_dtensor_params(
674 local_inputs, shard_pgs, hsdp_assign.layout_spec,
675 param_shard_metadata=param_shard_metadata,
676 buffer_cache=buffer_cache,
677 keep_indices=my_indices,
678 )
679 for p, full_inp in zip(valid_params, gathered_list):
680 if id(p) in my_param_ids:
681 gathered_inputs[p] = full_inp
682 local_inputs.clear()
683 gathered_list.clear()
685 # Compute NS updates in shape groups and memory-safe batches.
686 my_updates: Dict[torch.nn.Parameter, torch.Tensor] = {}
687 if my_params:
688 shape_groups = self._group_by_shape(my_params)
689 for _, p_list in shape_groups.items():
690 safe_batches = self._split_into_memory_safe_batches(p_list, shard_size=total_shard_size)
691 for sub_batch in safe_batches:
692 updates_dict = self._compute_batched_ns_updates(
693 sub_batch, gathered_inputs, group
694 )
695 for p in sub_batch:
696 my_updates[p] = updates_dict[p].contiguous()
697 del updates_dict
698 gathered_inputs.clear()
700 return my_updates, param_compute_coord
702 def _process_shard_params(
703 self,
704 group: Dict[str, Any],
705 param_to_ns_input: Dict[torch.nn.Parameter, torch.Tensor],
706 hsdp_assignments: List[HSDPGroupAssignment],
707 group_idx: int,
708 buffer_cache: Optional[Dict] = None,
709 ) -> None:
710 """Process sharded params with greedy shard-group compute assignment."""
711 shard_compute_coord = self._shard_compute_coord.get(group_idx, {})
713 for hsdp_assign in hsdp_assignments:
714 owned_params = hsdp_assign.owned_params
715 # Communication membership is static per assignment. Ranks with empty
716 # local shards still participate via build_pad_ns_inputs().
717 valid_params = owned_params
719 shard_sizes, _, shard_pgs, total_shard_size = self._get_shard_info(hsdp_assign)
721 # uneven shard check
722 shard_pgs_rank = []
723 for pg in shard_pgs:
724 pg_rank = torch.distributed.get_process_group_ranks(pg)
725 shard_pgs_rank.append(pg_rank)
727 for p in valid_params:
728 if tuple(p.shape)[0] % shard_sizes[0] != 0:
729 logger.debug_rank0(
730 "[hyper-optimizer uneven-shard] p %s placement %s device_mesh %s "
731 "local_shape %s global_shape %s pg_rank %s shard_sizes %s",
732 p.model_name,
733 p.placements,
734 p.device_mesh,
735 p.to_local().shape,
736 p.shape,
737 shard_pgs_rank,
738 shard_sizes,
739 )
741 if valid_params:
742 safe_batches = self._split_into_memory_safe_batches(
743 valid_params, shard_size=total_shard_size, min_batch_size=total_shard_size
744 )
745 for sub_batch in safe_batches:
746 my_updates, param_compute_coord = self._gather_and_compute_shard_updates(
747 sub_batch, param_to_ns_input, hsdp_assign,
748 shard_compute_coord, group, buffer_cache,
749 )
751 self._fused_broadcast_and_apply(
752 sub_batch, my_updates, param_compute_coord,
753 group, hsdp_assign,
754 )
756 def _group_by_shape(
757 self,
758 params: List[torch.nn.Parameter],
759 ) -> Dict[tuple, List[torch.nn.Parameter]]:
760 """Group parameters by their last-2-dim shape (A, B) for batched NS.
762 [1024, 1024], [1024, 1, 1024], and [3, 1024, 1024] all map to
763 key (1024, 1024) for maximum batch merging.
764 """
765 groups = defaultdict(list)
767 for p in params:
768 core_shape = self._shape_to_core_shape(tuple(p.shape))
769 groups[core_shape].append(p)
771 return groups
773 @staticmethod
774 def _shape_to_core_shape(shape: Tuple[int, ...]) -> Tuple[int, int]:
775 if len(shape) == 2:
776 return (shape[0], shape[1])
777 if len(shape) == 3 and shape[1] == 1:
778 return (shape[0], shape[2])
779 if len(shape) >= 3:
780 return (shape[-2], shape[-1])
781 raise ValueError('1D parameters are not supported in Muon')
783 def _reshape_ns_input(
784 self,
785 param: torch.nn.Parameter,
786 ns_input: torch.Tensor,
787 ) -> Tuple[torch.Tensor, List[torch.Tensor]]:
788 """Return the contiguous NS input and any reshape views used for NS."""
789 working_input = ns_input if ns_input.is_contiguous() else ns_input.contiguous()
790 if self.reshape_fn is None:
791 return working_input, [working_input]
793 param_fqn = param.model_name
794 if param_fqn is None:
795 return working_input, [working_input]
797 reshaped_inputs = list(self.reshape_fn(param_fqn, working_input))
798 if not reshaped_inputs:
799 return working_input, [working_input]
801 if reshaped_inputs[0].shape != working_input.shape:
802 logger.info_rank0(
803 "Reshape %s from %s to %s",
804 param_fqn,
805 working_input.shape,
806 reshaped_inputs[0].shape,
807 )
809 for reshaped_input in reshaped_inputs:
810 if reshaped_input.untyped_storage().data_ptr() != working_input.untyped_storage().data_ptr():
811 raise ValueError("reshape_fn must return views that share storage with the working NS input tensor")
813 return working_input, reshaped_inputs
815 def _prepare_ns_transform(
816 self,
817 param: torch.nn.Parameter,
818 ns_input: torch.Tensor,
819 ) -> Tuple[torch.Tensor, NSInputTransform]:
820 """Route one parameter through custom, zero-copy, or identity NS preparation."""
821 working_input = ns_input if ns_input.is_contiguous() else ns_input.contiguous()
822 param_fqn = getattr(param, "model_name", None)
823 if self.ns_transform_fn is not None and param_fqn is not None:
824 transform = self.ns_transform_fn(param_fqn, working_input)
825 if transform is not None:
826 self._validate_ns_transform(transform, working_input)
827 return working_input, transform
829 _, reshaped_inputs = self._reshape_ns_input(param, working_input)
831 def restore_view_updates(updates: List[torch.Tensor], output: torch.Tensor) -> None:
832 """Write reshaped NS updates back into the views and the working input."""
833 for reshaped_input, update in zip(reshaped_inputs, updates):
834 reshaped_input.copy_(update.contiguous().view_as(reshaped_input))
835 if output.untyped_storage().data_ptr() != working_input.untyped_storage().data_ptr():
836 output.copy_(working_input)
838 return working_input, NSInputTransform(tensors=reshaped_inputs, restore=restore_view_updates)
840 @staticmethod
841 def _validate_ns_transform(transform: NSInputTransform, working_input: torch.Tensor) -> None:
842 """Validate a model-provided reversible NS transform."""
843 if not isinstance(transform, NSInputTransform):
844 raise ValueError(
845 "ns_transform_fn must return NSInputTransform or None, "
846 f"got {type(transform).__name__}"
847 )
848 if not transform.tensors:
849 raise ValueError("NSInputTransform.tensors must not be empty")
850 for index, tensor in enumerate(transform.tensors):
851 if not isinstance(tensor, torch.Tensor):
852 raise ValueError(f"NSInputTransform.tensors[{index}] must be a torch.Tensor")
853 if tensor.device != working_input.device:
854 raise ValueError("NSInputTransform tensors must be on the same device as the NS input")
855 if tensor.dtype != working_input.dtype:
856 raise ValueError("NSInputTransform tensors must have the same dtype as the NS input")
857 if tensor.dim() < 2:
858 raise ValueError("NSInputTransform tensors must have at least two dimensions")
859 if not callable(transform.restore):
860 raise ValueError("NSInputTransform.restore must be callable")
862 def _compute_batched_ns_outputs_for_tensors(
863 self,
864 tensor_list: List[torch.Tensor],
865 ns_steps: int,
866 ns_variant: str = "asym5",
867 ns_coefficients: Optional[Sequence[Tuple[float, float, float]]] = None,
868 ns_epsilon: float = 1e-10,
869 ) -> List[torch.Tensor]:
870 """Run batched NS on mixed-shape tensors and restore their original shapes."""
872 if not tensor_list:
873 return []
875 state = {"inputs_3d": [], "slice_sizes": [], "shapes_info": [], "restored_updates": [], "current_idx": 0}
877 for tensor in tensor_list:
878 origin_shape = tuple(tensor.shape)
879 is_conv = False
881 if len(origin_shape) == 2:
882 inp_3d = tensor.unsqueeze(0)
883 n_dim = 1
884 elif len(origin_shape) == 3 and origin_shape[1] == 1:
885 inp_3d = tensor.squeeze(1).unsqueeze(0)
886 is_conv = True
887 n_dim = 1
888 elif len(origin_shape) == 3:
889 inp_3d = tensor
890 n_dim = origin_shape[0]
891 else:
892 inp_3d = tensor.reshape(-1, origin_shape[-2], origin_shape[-1])
893 n_dim = inp_3d.shape[0]
895 state["inputs_3d"].append(inp_3d)
896 state["slice_sizes"].append(n_dim)
897 state["shapes_info"].append((origin_shape, is_conv))
899 merged_input = torch.cat(state["inputs_3d"], dim=0)
901 if merged_input.shape[0] == 1:
902 merged_input = merged_input.squeeze(0)
904 if self.zeropower_fn is None:
905 merged_update = zeropower_via_newtonschulz5(
906 merged_input,
907 steps=ns_steps,
908 ns_variant=ns_variant,
909 epsilon=ns_epsilon,
910 ns_coefficients=ns_coefficients,
911 )
912 else:
913 merged_update = self.zeropower_fn(merged_input, steps=ns_steps)
915 if merged_input.dim() == 2:
916 merged_update = merged_update.unsqueeze(0)
917 del merged_input
919 for n_dim, (origin_shape, is_conv) in zip(state["slice_sizes"], state["shapes_info"]):
920 update = merged_update[state["current_idx"]:state["current_idx"] + n_dim]
921 state["current_idx"] += n_dim
923 if is_conv:
924 update = update.squeeze(0).unsqueeze(1)
925 elif len(origin_shape) == 2:
926 update = update.squeeze(0)
927 elif len(origin_shape) >= 4:
928 update = update.reshape(origin_shape)
930 state["restored_updates"].append(update)
932 del merged_update
933 return state["restored_updates"]
935 def _split_into_memory_safe_batches(
936 self,
937 p_list: List[torch.nn.Parameter],
938 shard_size: int = 1,
939 min_batch_size: int = 1,
940 ) -> List[List[torch.nn.Parameter]]:
941 """Split parameters into memory-safe batches to prevent OOM during NS.
943 The per-batch element limit is scaled down by shard_size to account
944 for the memory amplification from allgather. Sharded paths prioritize
945 filling the compute-owner ranks before applying this soft limit.
946 """
947 max_numel_per_batch = 512 * 1024 * 1024 // shard_size
948 min_batch_size = max(1, min_batch_size)
950 batches = []
951 current_batch = []
952 current_count = 0
954 for p in p_list:
955 p_count = p.numel()
956 if (
957 len(current_batch) >= min_batch_size
958 and current_count + p_count > max_numel_per_batch
959 ):
960 batches.append(current_batch)
961 current_batch = [p]
962 current_count = p_count
963 else:
964 current_batch.append(p)
965 current_count += p_count
967 if current_batch:
968 batches.append(current_batch)
970 return batches
972 def _compute_batched_ns_updates(
973 self,
974 p_list: List[torch.nn.Parameter],
975 ns_inputs: Dict[torch.nn.Parameter, torch.Tensor],
976 group: Dict[str, Any],
977 no_shard: bool = False
978 ) -> Dict[torch.nn.Parameter, torch.Tensor]:
979 """Route batched NS through the native or reversible-transform path."""
980 if self.ns_transform_fn is None:
981 return self._compute_batched_ns_updates_fast(p_list, ns_inputs, group, no_shard)
982 return self._compute_batched_ns_updates_with_transform(p_list, ns_inputs, group, no_shard)
984 def _compute_batched_ns_updates_fast(
985 self,
986 p_list: List[torch.nn.Parameter],
987 ns_inputs: Dict[torch.nn.Parameter, torch.Tensor],
988 group: Dict[str, Any],
989 no_shard: bool = False,
990 ) -> Dict[torch.nn.Parameter, torch.Tensor]:
991 """Compute native reshape/view NS updates without transform bookkeeping."""
992 if not p_list:
993 return {}
994 state = {"reshape_groups": defaultdict(list), "origin_shapes": {}, "working_inputs": {}, "restored": {}}
995 for param in p_list:
996 local_shape = getattr(param, "local_shape", None)
997 if local_shape is None:
998 local_shape = to_local_if_dtensor(param.data).shape
999 state["origin_shapes"][param] = tuple(local_shape) if no_shard else tuple(param.shape)
1000 working_input, reshaped_inputs = self._reshape_ns_input(
1001 param, ns_inputs[param].view(state["origin_shapes"][param])
1002 )
1003 state["working_inputs"][param] = working_input
1004 for reshaped_input in reshaped_inputs:
1005 state["reshape_groups"][self._shape_to_core_shape(tuple(reshaped_input.shape))].append(
1006 reshaped_input
1007 )
1008 for tensor_list in state["reshape_groups"].values():
1009 updates = self._compute_batched_ns_outputs_for_tensors(
1010 tensor_list, group["ns_steps"], group["ns_variant"], group["ns_coefficients"], group["ns_epsilon"]
1011 )
1012 for reshaped_input, update in zip(tensor_list, updates):
1013 scale = compute_muon_slice_scale(
1014 update, group["matched_adamw_rms"], zero_rms_scale_mode=group["zero_rms_scale_mode"]
1015 )
1016 if group["apply_lr_in_update"]:
1017 scale *= -group["lr"]
1018 update.mul_(scale)
1019 reshaped_input.copy_(update.contiguous().view_as(reshaped_input))
1021 for param in p_list:
1022 update = ns_inputs[param].view(state["origin_shapes"][param])
1023 working_input = state["working_inputs"][param]
1024 if working_input.untyped_storage().data_ptr() != update.untyped_storage().data_ptr():
1025 update.copy_(working_input)
1026 state["restored"][param] = update
1027 return state["restored"]
1029 def _compute_batched_ns_updates_with_transform(
1030 self,
1031 p_list: List[torch.nn.Parameter],
1032 ns_inputs: Dict[torch.nn.Parameter, torch.Tensor],
1033 group: Dict[str, Any],
1034 no_shard: bool = False,
1035 ) -> Dict[torch.nn.Parameter, torch.Tensor]:
1036 """Batched Newton-Schulz update for mixed 2D / Conv3D / 3D parameters.
1038 Normalizes all inputs to 3D, concatenates along dim 0, runs a single
1039 NS iteration, then slices results back to original shapes.
1041 """
1043 if not p_list:
1044 return {}
1046 state: _TransformNSState = {
1047 "reshape_groups": defaultdict(list),
1048 "origin_shapes": {},
1049 "working_inputs": {},
1050 "transforms": {},
1051 "transform_updates": [],
1052 "restored": {},
1053 }
1055 for p in p_list:
1056 state["origin_shapes"][p] = (
1057 tuple(to_local_if_dtensor(p.data).shape) if no_shard else tuple(p.shape)
1058 )
1059 prepared = self._prepare_ns_transform(p, ns_inputs[p].view(state["origin_shapes"][p]))
1060 state["working_inputs"][p] = prepared[0]
1061 state["transforms"][p] = prepared[1]
1062 state["transform_updates"].append([None] * len(prepared[1].tensors))
1064 for tensor_index, transformed_input in enumerate(prepared[1].tensors):
1065 state["reshape_groups"][self._shape_to_core_shape(tuple(transformed_input.shape))].append(
1066 (transformed_input, len(state["transform_updates"]) - 1, tensor_index)
1067 )
1069 for tensor_records in state["reshape_groups"].values():
1070 reshaped_updates = self._compute_batched_ns_outputs_for_tensors(
1071 [record[0] for record in tensor_records],
1072 group["ns_steps"],
1073 ns_variant=group["ns_variant"],
1074 ns_coefficients=group["ns_coefficients"],
1075 ns_epsilon=group["ns_epsilon"],
1076 )
1078 # Scale updates.
1079 for record, reshaped_update in zip(tensor_records, reshaped_updates):
1080 slice_scale = compute_muon_slice_scale(
1081 reshaped_update,
1082 group["matched_adamw_rms"],
1083 zero_rms_scale_mode=group["zero_rms_scale_mode"],
1084 )
1086 if group["apply_lr_in_update"]:
1087 slice_scale *= -group["lr"]
1088 reshaped_update.mul_(slice_scale)
1089 state["transform_updates"][record[1]][record[2]] = reshaped_update
1091 for p, reshaped_updates in zip(p_list, state["transform_updates"]):
1093 if any(update is None for update in reshaped_updates):
1094 raise RuntimeError("Missing Newton-Schulz update for a transformed input")
1095 state["transforms"][p].restore(reshaped_updates, state["working_inputs"][p])
1097 for p in p_list:
1098 reshaped_update = ns_inputs[p].view(state["origin_shapes"][p])
1100 if (state["working_inputs"][p].untyped_storage().data_ptr()
1101 != reshaped_update.untyped_storage().data_ptr()):
1102 reshaped_update.copy_(state["working_inputs"][p])
1103 state["restored"][p] = reshaped_update
1105 return state["restored"]
1107 def _fused_broadcast_and_apply(
1108 self,
1109 valid_params: List[torch.nn.Parameter],
1110 my_updates: Dict[torch.nn.Parameter, torch.Tensor],
1111 param_compute_coord: Dict[torch.nn.Parameter, Tuple[int, ...]],
1112 group: Dict[str, Any],
1113 hsdp_assign: HSDPGroupAssignment,
1114 ) -> None:
1115 """Fused broadcast and apply for shard-group updates."""
1116 lr = group["lr"]
1117 weight_decay = group["weight_decay"]
1118 shard_sizes, local_coords, shard_pgs, total_shard_size = self._get_shard_info(hsdp_assign)
1119 device = to_local_if_dtensor(valid_params[0].data).device
1121 coord_groups: Dict[Tuple[int, ...], List[torch.nn.Parameter]] = defaultdict(list)
1122 for p in valid_params:
1123 coord_groups[param_compute_coord[p]].append(p)
1125 all_local_params: List[torch.Tensor] = []
1126 all_update_shards: List[torch.Tensor] = []
1128 alignment_bytes = 512
1129 element_size = torch.empty(0, dtype=torch.bfloat16, device=device).element_size()
1130 alignment_elements = max(1, alignment_bytes // element_size)
1132 pack_buffers: Dict[Tuple[int, ...], torch.Tensor] = {}
1133 coord_param_offsets: Dict[Tuple[int, ...], List[Tuple[int, int, int]]] = {}
1135 for coord, coord_params in coord_groups.items():
1136 is_compute_rank = coord == local_coords
1137 param_offsets: List[Tuple[int, int, int]] = []
1138 total_padded_numel = 0
1140 for p in coord_params:
1141 actual_numel = p.numel()
1142 padded_numel = ((actual_numel + alignment_elements - 1) // alignment_elements) * alignment_elements
1143 param_offsets.append((total_padded_numel, actual_numel, padded_numel))
1144 total_padded_numel += padded_numel
1146 coord_param_offsets[coord] = param_offsets
1147 pack_buffer = torch.empty(total_padded_numel, dtype=torch.bfloat16, device=device)
1149 if is_compute_rank:
1150 for p, (offset, actual_numel, padded_numel) in zip(coord_params, param_offsets):
1151 update = my_updates[p]
1152 pack_buffer[offset:offset + actual_numel].copy_(update.reshape(-1))
1153 if padded_numel > actual_numel:
1154 pack_buffer[offset + actual_numel:offset + padded_numel].zero_()
1156 pack_buffers[coord] = pack_buffer
1158 if total_shard_size > 1:
1159 self._batched_relay_broadcast(
1160 pack_buffers, shard_pgs, shard_sizes, local_coords
1161 )
1163 layout_spec = hsdp_assign.layout_spec
1164 for coord, coord_params in coord_groups.items():
1165 pack_buffer = pack_buffers[coord]
1166 param_offsets = coord_param_offsets[coord]
1168 for p, (offset, actual_numel, _) in zip(coord_params, param_offsets):
1169 full_update = pack_buffer[offset:offset + actual_numel].view(tuple(p.shape))
1170 update_to_apply = chunk_update_by_layout(
1171 full_update,
1172 p,
1173 layout_spec,
1174 self._param_shard_metadata.get(p),
1175 )
1177 local_param = to_local_if_dtensor(p.data)
1178 all_local_params.append(local_param)
1179 all_update_shards.append(update_to_apply.view(local_param.shape))
1181 if not all_local_params:
1182 return
1184 if weight_decay != 0.0:
1185 coeff = 1.0 - lr * weight_decay
1186 # pylint: disable=protected-access
1187 torch._foreach_mul_(all_local_params, coeff)
1189 # Slice-wise Muon scaling has already been applied during NS postprocess.
1190 # pylint: disable=protected-access
1191 apply_alpha = 1.0 if group["apply_lr_in_update"] else -lr
1192 torch._foreach_add_(all_local_params, all_update_shards, alpha=apply_alpha)
1194 def _build_param_shard_metadata(self) -> None:
1195 """Build shard metadata once during optimizer init."""
1196 self._param_shard_metadata: Dict[torch.nn.Parameter, ParamShardMeta] = {}
1198 for _, hsdp_groups in self._hsdp_grouping.values():
1199 for hsdp_group in hsdp_groups:
1200 if hsdp_group.layout_spec is None or not hsdp_group.layout_spec.shard_axes:
1201 continue
1202 group_param_to_meta = build_param_shard_metadata_for_group(hsdp_group)
1203 for param, shard_meta in group_param_to_meta.items():
1204 self._param_shard_metadata[param] = shard_meta
1205 _debug_param_shard_metadata(hsdp_group, group_param_to_meta)
1207 @staticmethod
1208 def _get_shard_info(
1209 hsdp_assign: HSDPGroupAssignment,
1210 ) -> Tuple[Tuple[int, ...], Tuple[int, ...], Tuple[dist.ProcessGroup, ...], int]:
1211 """Extract shard topology from HSDPGroupAssignment.
1213 Returns:
1214 shard_sizes: Size of each shard mesh dimension.
1215 local_coords: Current rank's coordinate in each shard mesh dim.
1216 shard_pgs: ProcessGroup for each shard mesh dim.
1217 total_shard_size: Product of all shard_sizes.
1218 """
1219 shard_pgs = hsdp_assign.shard_pgs
1220 shard_sizes = tuple(
1221 dist.get_world_size(pg) if pg is not None else 1
1222 for pg in shard_pgs
1223 )
1224 local_coords = tuple(
1225 dist.get_rank(pg) if pg is not None else 0
1226 for pg in shard_pgs
1227 )
1228 total_shard_size = 1
1229 for s in shard_sizes:
1230 total_shard_size *= s
1232 return shard_sizes, local_coords, shard_pgs, total_shard_size
1234 @staticmethod
1235 def _batched_relay_broadcast(
1236 tensor_dict: Dict[Tuple[int, ...], torch.Tensor],
1237 shard_pgs: Tuple[dist.ProcessGroup, ...],
1238 shard_sizes: Tuple[int, ...],
1239 local_coords: Tuple[int, ...],
1240 ) -> None:
1241 """
1242 Batched asynchronous multi-dimensional relay broadcast.
1243 By operating asynchronously within each dimension, we eliminate CPU overhead bubbles
1244 while strictly preserving the multidimensional relay dependency.
1245 """
1246 for dim_idx, pg in enumerate(shard_pgs):
1247 if pg is None or shard_sizes[dim_idx] <= 1:
1248 continue
1250 work_handles = []
1252 for coord, tensor in tensor_dict.items():
1253 aligned = all(
1254 local_coords[sub_dim] == coord[sub_dim]
1255 for sub_dim in range(dim_idx + 1, len(shard_pgs))
1256 )
1257 if not aligned:
1258 continue
1260 src_rank_in_pg = coord[dim_idx]
1261 global_src_rank = dist.get_global_rank(pg, src_rank_in_pg)
1263 work = dist.broadcast(tensor, src=global_src_rank, group=pg, async_op=True)
1264 if work is not None:
1265 work_handles.append(work)
1267 for work in work_handles:
1268 work.wait()