Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / __init__.py: 98%

90 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""HyperParallel optimizer module.""" 

16from importlib import import_module as _import_module # pylint: disable=invalid-name 

17 

18import inspect 

19import logging 

20from typing import Any, Callable, Dict, List, Optional 

21 

22from hyper_parallel.core.optimizer.swap_optimizer import ( 

23 SwapOptimizer, 

24 SwapOptimizerConfig, 

25 swap_optimizer, 

26) 

27 

28logger = logging.getLogger(__name__) 

29logger.setLevel(logging.INFO) 

30 

31# Optimizer implementations import torch at module load. Keep them off the 

32# eager path so importing the package does not pull in torch. 

33_LAZY_EXPORTS = { 

34 "AdamW": ".adamw", 

35 "Muon": ".muon", 

36 "Sinkhorn": ".sinkhorn", 

37 "ChainedOptimizer": ".optimizer", 

38 "detect_dtensor_backend": ".dtensor_compat", 

39} 

40 

41 

42def __getattr__(name): # pylint: disable=invalid-name 

43 """Lazily import torch-only optimizer symbols.""" 

44 if name not in _LAZY_EXPORTS: 

45 raise AttributeError(f"module {__name__!r} has no attribute {name!r}") 

46 module = _import_module(_LAZY_EXPORTS[name], __name__) 

47 value = getattr(module, name) 

48 globals()[name] = value 

49 return value 

50 

51 

52def __dir__(): # pylint: disable=invalid-name 

53 """Include lazy torch-only optimizer exports in ``dir()``.""" 

54 return sorted(set(globals()) | set(_LAZY_EXPORTS)) 

55 

56 

57def _load_torch_optimizer_runtime(): 

58 """Import torch-only optimizer helpers used by the factory APIs.""" 

59 # pylint: disable=import-outside-toplevel,unused-import 

60 import hyper_parallel.core.optimizer.utils # noqa: F401 - install rank0 logging helpers 

61 

62 from hyper_parallel.core.optimizer.adamw import AdamW 

63 from hyper_parallel.core.optimizer.dtensor_compat import detect_dtensor_backend 

64 from hyper_parallel.core.optimizer.muon import Muon, _resolve_ns_coefficients 

65 from hyper_parallel.core.optimizer.optimizer import ChainedOptimizer 

66 

67 return AdamW, Muon, ChainedOptimizer, detect_dtensor_backend, _resolve_ns_coefficients 

68 

69 

70def _effective_optimizer_config( 

71 optimizer_class: Any, 

72 configured_values: Dict[str, Any], 

73 runtime_optimizer: Any, 

74) -> Dict[str, Any]: 

75 """Merge constructor defaults, user values, and resolved runtime defaults.""" 

76 signature = inspect.signature(optimizer_class.__init__) 

77 effective_config = {} 

78 for name, parameter in signature.parameters.items(): 

79 if name not in {"self", "params"} and parameter.default is not inspect.Parameter.empty: 

80 effective_config[name] = parameter.default 

81 effective_config.update(configured_values) 

82 effective_config.update(runtime_optimizer.defaults) 

83 if hasattr(runtime_optimizer, "hsdp_replica_count"): 

84 effective_config["hsdp_replica_count"] = runtime_optimizer.hsdp_replica_count 

85 return effective_config 

86 

87 

88def _filter_optimizer_config( 

89 optimizer_name: str, 

90 optimizer_class: Any, 

91 configured_values: Optional[Dict[str, Any]], 

92) -> Dict[str, Any]: 

93 """Normalize prefixed YAML keys and remove unsupported constructor args.""" 

94 prefix = f"{optimizer_name}_" 

95 normalized_config = { 

96 key[len(prefix):] if key.startswith(prefix) else key: value 

97 for key, value in (configured_values or {}).items() 

98 } 

99 allowed_keys = ( 

100 inspect.signature(optimizer_class.__init__).parameters.keys() 

101 - {"self", "params"} 

102 ) 

103 allowed_keys = set(allowed_keys) | set(getattr(optimizer_class, "ADDITIONAL_CONFIG_KEYS", ())) 

104 filtered_config = { 

105 key: value 

106 for key, value in normalized_config.items() 

107 if key in allowed_keys 

108 } 

109 excluded_keys = normalized_config.keys() - allowed_keys 

110 if excluded_keys: 

111 logger.info_rank0( 

112 "Excluded %s config: %s", 

113 optimizer_name, 

114 list(excluded_keys), 

115 ) 

116 return filtered_config 

117 

118 

119def _build_configured_optimizer( 

120 optimizer_name: str, 

121 optimizer_class: Any, 

122 param_groups: Any, 

123 configured_values: Dict[str, Any], 

124) -> Any: 

125 """Construct one leaf optimizer and log its effective configuration.""" 

126 optimizer = optimizer_class(param_groups, **configured_values) 

127 logger.info_rank0( 

128 f"Effective {optimizer_name} config: %s", 

129 _effective_optimizer_config( 

130 optimizer_class, 

131 configured_values, 

132 optimizer, 

133 ), 

134 ) 

135 return optimizer 

136 

137 

138def _configure_muon_gram_ns(optimizer: Any, resolve_coefficients: Callable[..., Any], backend: str) -> None: 

139 """Bind the optional Gram callback from fully resolved Muon parameter groups.""" 

140 if optimizer.zeropower_fn is not None: 

141 raise ValueError(f"muon_ns_backend={backend!r} cannot be combined with an explicit zeropower_fn") 

142 schedules = [] 

143 for group in optimizer.param_groups: 

144 steps = group["ns_steps"] 

145 if not isinstance(steps, int) or isinstance(steps, bool) or steps <= 0: 

146 raise ValueError(f"Gram Muon ns_steps must be a positive integer, got {steps!r}") 

147 schedule = resolve_coefficients(steps, group["ns_variant"], group["ns_coefficients"]) 

148 if steps > len(schedule): 

149 raise ValueError(f"Gram Muon ns_steps={steps} exceeds the {len(schedule)} coefficient groups") 

150 schedules.append(schedule) 

151 coefficients = max(schedules, key=len) 

152 epsilon = optimizer.param_groups[0]["ns_epsilon"] 

153 for group, schedule in zip(optimizer.param_groups, schedules): 

154 if schedule != coefficients[:len(schedule)] or group["ns_epsilon"] != epsilon: 

155 raise ValueError( 

156 f"muon_ns_backend={backend!r} requires compatible coefficient prefixes and the same " 

157 "ns_epsilon across Muon parameter groups" 

158 ) 

159 # Optional CUDA dependencies stay off the native factory and optimizer import paths. 

160 gram_module = _import_module("hyper_parallel.components.functional.gram_newton_schulz") 

161 optimizer.zeropower_fn = gram_module.make_gram_newton_schulz( 

162 coefficients, epsilon=epsilon, use_kernels=False if backend == "gram" else None, 

163 ) 

164 

165 

166def get_hyper_optimizer( 

167 model: Any, 

168 muon_params: List[Dict[str, Any]], 

169 adamw_params: List[Dict[str, Any]], 

170 muon_kwargs: Optional[Dict[str, Any]] = None, 

171 adamw_kwargs: Optional[Dict[str, Any]] = None, 

172 *, 

173 muon_ns_backend: str = "native", 

174 sinkhorn_params: Optional[List[Dict[str, Any]]] = None, 

175 sinkhorn_kwargs: Optional[Dict[str, Any]] = None, 

176) -> Any: 

177 """Create a chained Muon + AdamW optimizer with optional Sinkhorn groups. 

178 

179 Args: 

180 model: The neural network model. 

181 muon_params: Param groups for Muon. Empty list disables Muon. 

182 adamw_params: Param groups for AdamW. Empty list disables AdamW. 

183 muon_kwargs: Dedicated configurations dict for Muon. 

184 adamw_kwargs: Dedicated configurations dict for AdamW. 

185 muon_ns_backend: ``"native"`` (default) preserves the configured Muon callback. 

186 ``"gram"`` uses the upstream Torch path on CPU/CUDA. ``"gram_quack"`` 

187 enables Quack on Hopper/Blackwell CUDA and falls back to Torch on CPU 

188 or other CUDA devices. Both bind the effective Muon group settings, 

189 require compatible coefficient prefixes and one epsilon across groups, 

190 and reject a manual ``zeropower_fn``. Ignored without Muon parameters. 

191 sinkhorn_params: Optional Sinkhorn groups. Empty or None disables Sinkhorn. 

192 sinkhorn_kwargs: Dedicated configurations dict for Sinkhorn. 

193 

194 Example: 

195 from hyper_parallel.core.optimizer import get_hyper_optimizer 

196 

197 _adamw_legacy = { 

198 'adamw_lr': 1e-3, 

199 'adamw_weight_decay': 1e-2, 

200 'adamw_betas': (0.9, 0.95), 

201 'adamw_eps': 1e-8, 

202 'fused': True 

203 } 

204 _muon_legacy = { 

205 'muon_lr': 2e-2, 

206 'muon_weight_decay': 0.1, 

207 'muon_momentum': 0.95, 

208 'muon_ns_steps': 5, 

209 'muon_ns_variant': 'asym5', 

210 'muon_nesterov': True, 

211 'muon_hsdp_replica_count': 2 

212 } 

213 

214 optimizer = get_hyper_optimizer( 

215 model=model, 

216 muon_params=muon_groups, 

217 adamw_params=adamw_groups, 

218 adamw_kwargs=_adamw_legacy, 

219 muon_kwargs=_muon_legacy, 

220 ) 

221 

222 optimizer.step() 

223 """ 

224 if muon_ns_backend not in ("native", "gram", "gram_quack"): 

225 raise ValueError(f"muon_ns_backend must be 'native', 'gram', or 'gram_quack', got {muon_ns_backend!r}") 

226 AdamW, Muon, ChainedOptimizer, detect_dtensor_backend, resolve_coefficients = _load_torch_optimizer_runtime() 

227 

228 filtered_adamw_config = _filter_optimizer_config("adamw", AdamW, adamw_kwargs) 

229 filtered_muon_config = _filter_optimizer_config("muon", Muon, muon_kwargs) 

230 optimizers = {} 

231 detect_dtensor_backend(adamw_params or sinkhorn_params or [], muon_params) 

232 

233 if adamw_params: 

234 optimizers["adamw"] = _build_configured_optimizer( 

235 "adamw", 

236 AdamW, 

237 adamw_params, 

238 filtered_adamw_config, 

239 ) 

240 

241 if muon_params: 

242 optimizers["muon"] = _build_configured_optimizer( 

243 "muon", 

244 Muon, 

245 muon_params, 

246 filtered_muon_config, 

247 ) 

248 if muon_ns_backend != "native": 

249 _configure_muon_gram_ns(optimizers["muon"], resolve_coefficients, muon_ns_backend) 

250 logger.info_rank0("Muon NS backend selection: %s", muon_ns_backend) 

251 

252 if sinkhorn_params: 

253 sinkhorn_class = __getattr__("Sinkhorn") 

254 filtered_sinkhorn_config = _filter_optimizer_config("sinkhorn", sinkhorn_class, sinkhorn_kwargs) 

255 optimizers["sinkhorn"] = _build_configured_optimizer( 

256 "sinkhorn", sinkhorn_class, sinkhorn_params, filtered_sinkhorn_config, 

257 ) 

258 

259 flatten = len(optimizers) > 1 

260 

261 return ChainedOptimizer( 

262 model, 

263 optimizers=optimizers, 

264 flatten=flatten, 

265 ) 

266 

267 

268__all__ = [ 

269 'SwapOptimizer', 

270 'SwapOptimizerConfig', 

271 'get_hyper_optimizer', 

272 'swap_optimizer', 

273]