Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / __init__.py: 98%
90 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""HyperParallel optimizer module."""
16from importlib import import_module as _import_module # pylint: disable=invalid-name
18import inspect
19import logging
20from typing import Any, Callable, Dict, List, Optional
22from hyper_parallel.core.optimizer.swap_optimizer import (
23 SwapOptimizer,
24 SwapOptimizerConfig,
25 swap_optimizer,
26)
28logger = logging.getLogger(__name__)
29logger.setLevel(logging.INFO)
31# Optimizer implementations import torch at module load. Keep them off the
32# eager path so importing the package does not pull in torch.
33_LAZY_EXPORTS = {
34 "AdamW": ".adamw",
35 "Muon": ".muon",
36 "Sinkhorn": ".sinkhorn",
37 "ChainedOptimizer": ".optimizer",
38 "detect_dtensor_backend": ".dtensor_compat",
39}
42def __getattr__(name): # pylint: disable=invalid-name
43 """Lazily import torch-only optimizer symbols."""
44 if name not in _LAZY_EXPORTS:
45 raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
46 module = _import_module(_LAZY_EXPORTS[name], __name__)
47 value = getattr(module, name)
48 globals()[name] = value
49 return value
52def __dir__(): # pylint: disable=invalid-name
53 """Include lazy torch-only optimizer exports in ``dir()``."""
54 return sorted(set(globals()) | set(_LAZY_EXPORTS))
57def _load_torch_optimizer_runtime():
58 """Import torch-only optimizer helpers used by the factory APIs."""
59 # pylint: disable=import-outside-toplevel,unused-import
60 import hyper_parallel.core.optimizer.utils # noqa: F401 - install rank0 logging helpers
62 from hyper_parallel.core.optimizer.adamw import AdamW
63 from hyper_parallel.core.optimizer.dtensor_compat import detect_dtensor_backend
64 from hyper_parallel.core.optimizer.muon import Muon, _resolve_ns_coefficients
65 from hyper_parallel.core.optimizer.optimizer import ChainedOptimizer
67 return AdamW, Muon, ChainedOptimizer, detect_dtensor_backend, _resolve_ns_coefficients
70def _effective_optimizer_config(
71 optimizer_class: Any,
72 configured_values: Dict[str, Any],
73 runtime_optimizer: Any,
74) -> Dict[str, Any]:
75 """Merge constructor defaults, user values, and resolved runtime defaults."""
76 signature = inspect.signature(optimizer_class.__init__)
77 effective_config = {}
78 for name, parameter in signature.parameters.items():
79 if name not in {"self", "params"} and parameter.default is not inspect.Parameter.empty:
80 effective_config[name] = parameter.default
81 effective_config.update(configured_values)
82 effective_config.update(runtime_optimizer.defaults)
83 if hasattr(runtime_optimizer, "hsdp_replica_count"):
84 effective_config["hsdp_replica_count"] = runtime_optimizer.hsdp_replica_count
85 return effective_config
88def _filter_optimizer_config(
89 optimizer_name: str,
90 optimizer_class: Any,
91 configured_values: Optional[Dict[str, Any]],
92) -> Dict[str, Any]:
93 """Normalize prefixed YAML keys and remove unsupported constructor args."""
94 prefix = f"{optimizer_name}_"
95 normalized_config = {
96 key[len(prefix):] if key.startswith(prefix) else key: value
97 for key, value in (configured_values or {}).items()
98 }
99 allowed_keys = (
100 inspect.signature(optimizer_class.__init__).parameters.keys()
101 - {"self", "params"}
102 )
103 allowed_keys = set(allowed_keys) | set(getattr(optimizer_class, "ADDITIONAL_CONFIG_KEYS", ()))
104 filtered_config = {
105 key: value
106 for key, value in normalized_config.items()
107 if key in allowed_keys
108 }
109 excluded_keys = normalized_config.keys() - allowed_keys
110 if excluded_keys:
111 logger.info_rank0(
112 "Excluded %s config: %s",
113 optimizer_name,
114 list(excluded_keys),
115 )
116 return filtered_config
119def _build_configured_optimizer(
120 optimizer_name: str,
121 optimizer_class: Any,
122 param_groups: Any,
123 configured_values: Dict[str, Any],
124) -> Any:
125 """Construct one leaf optimizer and log its effective configuration."""
126 optimizer = optimizer_class(param_groups, **configured_values)
127 logger.info_rank0(
128 f"Effective {optimizer_name} config: %s",
129 _effective_optimizer_config(
130 optimizer_class,
131 configured_values,
132 optimizer,
133 ),
134 )
135 return optimizer
138def _configure_muon_gram_ns(optimizer: Any, resolve_coefficients: Callable[..., Any], backend: str) -> None:
139 """Bind the optional Gram callback from fully resolved Muon parameter groups."""
140 if optimizer.zeropower_fn is not None:
141 raise ValueError(f"muon_ns_backend={backend!r} cannot be combined with an explicit zeropower_fn")
142 schedules = []
143 for group in optimizer.param_groups:
144 steps = group["ns_steps"]
145 if not isinstance(steps, int) or isinstance(steps, bool) or steps <= 0:
146 raise ValueError(f"Gram Muon ns_steps must be a positive integer, got {steps!r}")
147 schedule = resolve_coefficients(steps, group["ns_variant"], group["ns_coefficients"])
148 if steps > len(schedule):
149 raise ValueError(f"Gram Muon ns_steps={steps} exceeds the {len(schedule)} coefficient groups")
150 schedules.append(schedule)
151 coefficients = max(schedules, key=len)
152 epsilon = optimizer.param_groups[0]["ns_epsilon"]
153 for group, schedule in zip(optimizer.param_groups, schedules):
154 if schedule != coefficients[:len(schedule)] or group["ns_epsilon"] != epsilon:
155 raise ValueError(
156 f"muon_ns_backend={backend!r} requires compatible coefficient prefixes and the same "
157 "ns_epsilon across Muon parameter groups"
158 )
159 # Optional CUDA dependencies stay off the native factory and optimizer import paths.
160 gram_module = _import_module("hyper_parallel.components.functional.gram_newton_schulz")
161 optimizer.zeropower_fn = gram_module.make_gram_newton_schulz(
162 coefficients, epsilon=epsilon, use_kernels=False if backend == "gram" else None,
163 )
166def get_hyper_optimizer(
167 model: Any,
168 muon_params: List[Dict[str, Any]],
169 adamw_params: List[Dict[str, Any]],
170 muon_kwargs: Optional[Dict[str, Any]] = None,
171 adamw_kwargs: Optional[Dict[str, Any]] = None,
172 *,
173 muon_ns_backend: str = "native",
174 sinkhorn_params: Optional[List[Dict[str, Any]]] = None,
175 sinkhorn_kwargs: Optional[Dict[str, Any]] = None,
176) -> Any:
177 """Create a chained Muon + AdamW optimizer with optional Sinkhorn groups.
179 Args:
180 model: The neural network model.
181 muon_params: Param groups for Muon. Empty list disables Muon.
182 adamw_params: Param groups for AdamW. Empty list disables AdamW.
183 muon_kwargs: Dedicated configurations dict for Muon.
184 adamw_kwargs: Dedicated configurations dict for AdamW.
185 muon_ns_backend: ``"native"`` (default) preserves the configured Muon callback.
186 ``"gram"`` uses the upstream Torch path on CPU/CUDA. ``"gram_quack"``
187 enables Quack on Hopper/Blackwell CUDA and falls back to Torch on CPU
188 or other CUDA devices. Both bind the effective Muon group settings,
189 require compatible coefficient prefixes and one epsilon across groups,
190 and reject a manual ``zeropower_fn``. Ignored without Muon parameters.
191 sinkhorn_params: Optional Sinkhorn groups. Empty or None disables Sinkhorn.
192 sinkhorn_kwargs: Dedicated configurations dict for Sinkhorn.
194 Example:
195 from hyper_parallel.core.optimizer import get_hyper_optimizer
197 _adamw_legacy = {
198 'adamw_lr': 1e-3,
199 'adamw_weight_decay': 1e-2,
200 'adamw_betas': (0.9, 0.95),
201 'adamw_eps': 1e-8,
202 'fused': True
203 }
204 _muon_legacy = {
205 'muon_lr': 2e-2,
206 'muon_weight_decay': 0.1,
207 'muon_momentum': 0.95,
208 'muon_ns_steps': 5,
209 'muon_ns_variant': 'asym5',
210 'muon_nesterov': True,
211 'muon_hsdp_replica_count': 2
212 }
214 optimizer = get_hyper_optimizer(
215 model=model,
216 muon_params=muon_groups,
217 adamw_params=adamw_groups,
218 adamw_kwargs=_adamw_legacy,
219 muon_kwargs=_muon_legacy,
220 )
222 optimizer.step()
223 """
224 if muon_ns_backend not in ("native", "gram", "gram_quack"):
225 raise ValueError(f"muon_ns_backend must be 'native', 'gram', or 'gram_quack', got {muon_ns_backend!r}")
226 AdamW, Muon, ChainedOptimizer, detect_dtensor_backend, resolve_coefficients = _load_torch_optimizer_runtime()
228 filtered_adamw_config = _filter_optimizer_config("adamw", AdamW, adamw_kwargs)
229 filtered_muon_config = _filter_optimizer_config("muon", Muon, muon_kwargs)
230 optimizers = {}
231 detect_dtensor_backend(adamw_params or sinkhorn_params or [], muon_params)
233 if adamw_params:
234 optimizers["adamw"] = _build_configured_optimizer(
235 "adamw",
236 AdamW,
237 adamw_params,
238 filtered_adamw_config,
239 )
241 if muon_params:
242 optimizers["muon"] = _build_configured_optimizer(
243 "muon",
244 Muon,
245 muon_params,
246 filtered_muon_config,
247 )
248 if muon_ns_backend != "native":
249 _configure_muon_gram_ns(optimizers["muon"], resolve_coefficients, muon_ns_backend)
250 logger.info_rank0("Muon NS backend selection: %s", muon_ns_backend)
252 if sinkhorn_params:
253 sinkhorn_class = __getattr__("Sinkhorn")
254 filtered_sinkhorn_config = _filter_optimizer_config("sinkhorn", sinkhorn_class, sinkhorn_kwargs)
255 optimizers["sinkhorn"] = _build_configured_optimizer(
256 "sinkhorn", sinkhorn_class, sinkhorn_params, filtered_sinkhorn_config,
257 )
259 flatten = len(optimizers) > 1
261 return ChainedOptimizer(
262 model,
263 optimizers=optimizers,
264 flatten=flatten,
265 )
268__all__ = [
269 'SwapOptimizer',
270 'SwapOptimizerConfig',
271 'get_hyper_optimizer',
272 'swap_optimizer',
273]