Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / gram_newton_schulz.py: 99%
77 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Optional Gram Newton-Schulz adapter for Muon's zeropower callback."""
17import copy
18import importlib
19import math
20from collections.abc import Callable, Mapping, Sequence
21from typing import Any, Optional
23import torch
26def _load_gram_newton_schulz() -> Callable[..., Any]:
27 """Load the optional package only when its backend is explicitly requested."""
28 try:
29 # CUDA packages must stay off the default CPU/NPU import path.
30 package = importlib.import_module("gram_newton_schulz")
31 except (ImportError, OSError) as error:
32 raise ImportError(
33 "Could not load gram-newton-schulz. Install a compatible version following "
34 "hyper_parallel/components/functional/README.md. "
35 f"Original error: {error}"
36 ) from error
37 implementation = getattr(package, "GramNewtonSchulz", None)
38 if not callable(implementation):
39 raise ImportError("gram-newton-schulz must export a callable GramNewtonSchulz")
40 return implementation
43def _normalize_coefficients(coefficients: Sequence[Sequence[float]]) -> tuple[tuple[float, float, float], ...]:
44 """Own an immutable, finite coefficient schedule independently of the caller."""
45 normalized = []
46 for index, row in enumerate(coefficients):
47 if len(row) != 3:
48 raise ValueError(f"ns_coefficients[{index}] must contain exactly three values")
49 values = tuple(float(value) for value in row)
50 if not all(math.isfinite(value) for value in values):
51 raise ValueError(f"ns_coefficients[{index}] must contain only finite values")
52 normalized.append(values)
53 if not normalized:
54 raise ValueError("ns_coefficients must not be empty")
55 return tuple(normalized)
58class _GramNewtonSchulzAdapter:
59 """Bind an NS schedule and cache upstream instances per device and step count."""
61 def __init__(
62 self,
63 implementation: Callable[..., Any],
64 coefficients: tuple[tuple[float, float, float], ...],
65 epsilon: float,
66 reset_iterations: tuple[int, ...],
67 use_kernels: Optional[bool],
68 compile_kwargs: Optional[Mapping[str, Any]],
69 ) -> None:
70 self._implementation = implementation
71 self._coefficients = coefficients
72 self._epsilon = epsilon
73 self._reset_iterations = reset_iterations
74 self._use_kernels = use_kernels
75 self._compile_kwargs = copy.deepcopy(dict(compile_kwargs)) if compile_kwargs is not None else None
76 self._instances: dict[tuple[torch.device, int], Any] = {}
78 def _resolve_use_kernels(self, device: torch.device) -> bool:
79 """Choose kernels from the input device, preserving explicit overrides."""
80 if device.type != "cuda":
81 if self._use_kernels is True:
82 raise ValueError("Gram Newton-Schulz custom kernels require a Hopper or Blackwell CUDA device")
83 if device.type != "cpu":
84 raise ValueError("Gram Newton-Schulz's Torch path supports CPU/CUDA; NPU is not qualified")
85 return False
86 if self._use_kernels is False:
87 return False
88 major, _ = torch.cuda.get_device_capability(device)
89 supported = major in (9, 10)
90 if self._use_kernels is True and not supported:
91 raise ValueError(
92 f"Gram Newton-Schulz custom kernels require Hopper/Blackwell (SM 9.x/10.x), got SM {major}.x"
93 )
94 return supported
96 @torch.no_grad()
97 def __call__(self, ns_inputs: torch.Tensor, steps: int) -> torch.Tensor:
98 """Orthogonalize a local 2D matrix or 3D batch without retaining inputs."""
99 if not isinstance(steps, int) or isinstance(steps, bool) or not 1 <= steps <= len(self._coefficients):
100 raise ValueError(f"steps must be an integer in [1, {len(self._coefficients)}], got {steps!r}")
101 if ns_inputs.ndim not in (2, 3):
102 raise ValueError(f"Gram Newton-Schulz expects a 2D matrix or 3D batch, got {ns_inputs.ndim}D")
103 if ns_inputs.dtype not in (torch.float16, torch.bfloat16, torch.float32):
104 raise ValueError(f"Gram Newton-Schulz expects float16, bfloat16 or float32, got {ns_inputs.dtype}")
105 if ns_inputs.numel() == 0:
106 self._resolve_use_kernels(ns_inputs.device)
107 return torch.empty_like(ns_inputs)
108 key = (ns_inputs.device, steps)
109 if key not in self._instances:
110 self._instances[key] = self._implementation(
111 ns_epsilon=self._epsilon,
112 ns_use_kernels=self._resolve_use_kernels(ns_inputs.device),
113 ns_coefficients=[list(row) for row in self._coefficients[:steps]],
114 gram_newton_schulz_reset_iterations=[index for index in self._reset_iterations if index < steps],
115 compile_kwargs=copy.deepcopy(self._compile_kwargs),
116 )
117 return self._instances[key](ns_inputs.contiguous())
120def make_gram_newton_schulz(
121 ns_coefficients: Sequence[Sequence[float]],
122 *,
123 epsilon: float = 1e-10,
124 reset_iterations: Sequence[int] = (2,),
125 use_kernels: Optional[bool] = None,
126 compile_kwargs: Optional[Mapping[str, Any]] = None,
127) -> Callable[..., torch.Tensor]:
128 """Create a Gram Newton-Schulz callback accepted by ``Muon(zeropower_fn=...)``.
130 Args:
131 ns_coefficients: Explicit coefficient schedule. The callback uses its first
132 ``steps`` entries. Pass the same schedule to Muon with ``ns_variant='custom'``.
133 epsilon: Finite positive normalization epsilon; share it with Muon's ``ns_epsilon``.
134 reset_iterations: Positive iteration counts after which to restart. Counts
135 greater than or equal to the requested step count are ignored.
136 use_kernels: None (default) selects kernels for Hopper/Blackwell CUDA inputs
137 and the upstream Torch path for CPU/other CUDA inputs. True requires
138 supported CUDA hardware; False forces the Torch path on CPU/CUDA.
139 compile_kwargs: Options forwarded to upstream ``torch.compile``. None disables
140 compilation; pass a mapping to enable it (including an empty mapping).
142 Returns:
143 Callable with signature ``callback(ns_inputs, steps)`` that preserves input
144 shape, dtype and device. Instances are cached per device and step count.
146 Raises:
147 ImportError: The optional package or its public entry point is unavailable.
148 ValueError: Configuration, input, step count or device is unsupported.
150 Note:
151 Coefficients and epsilon are bound here: Muon's existing callback only
152 forwards ``steps``. All groups must share the bound schedule and epsilon.
153 Recreate the callback with matching configuration when restoring a checkpoint.
154 Upstream uses FP32 normalization and FP16 iteration, including for BF16 input.
155 Automatic selection uses device capability, not exception recovery. Import,
156 compilation and execution errors propagate. NPU is not qualified.
157 """
158 coefficients = _normalize_coefficients(ns_coefficients)
159 if not math.isfinite(epsilon) or epsilon <= 0:
160 raise ValueError(f"epsilon must be finite and positive, got {epsilon!r}")
161 resets = tuple(reset_iterations)
162 if any(not isinstance(index, int) or isinstance(index, bool) or index <= 0 for index in resets):
163 raise ValueError("reset_iterations must contain positive integer iteration counts")
164 if use_kernels is not None and not isinstance(use_kernels, bool):
165 raise ValueError("use_kernels must be a bool or None")
166 if compile_kwargs is not None and not isinstance(compile_kwargs, Mapping):
167 raise ValueError("compile_kwargs must be a mapping or None")
168 return _GramNewtonSchulzAdapter(
169 _load_gram_newton_schulz(), coefficients, float(epsilon), tuple(sorted(set(resets))),
170 use_kernels, compile_kwargs,
171 )