Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / swap_adam.py: 88%

190 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Adam/AdamW optimizer-state swap adapters.""" 

16 

17# pylint: disable=protected-access 

18 

19from __future__ import annotations 

20 

21import inspect 

22from collections import defaultdict 

23from dataclasses import dataclass 

24from typing import Any, Callable, Dict, List, NamedTuple, Sequence, Tuple 

25 

26import torch 

27 

28from hyper_parallel.core.optimizer.swap_optimizer_base import ( 

29 StateSwapAdapter, 

30 SwapOptimizer as CoreSwapOptimizer, 

31 SwapSlot, 

32 _slot_tensor, 

33) 

34 

35#: Logical state keys an Adam/AdamW optimizer owns. 

36ADAM_STATE_KEYS = ("exp_avg", "exp_avg_sq", "max_exp_avg_sq") 

37 

38 

39class GroupArgs(NamedTuple): 

40 """Per-parameter argument lists one functional Adam/AdamW step consumes.""" 

41 

42 params: List[Any] 

43 grads: List[Any] 

44 exp_avgs: List[Any] 

45 exp_avg_sqs: List[Any] 

46 max_exp_avg_sqs: List[Any] 

47 state_steps: List[Any] 

48 

49 

50@dataclass 

51class AdamUpdateUnit: 

52 """Per-parameter Adam/AdamW update unit consumed by the pipeline runtime. 

53 

54 ``adapter_index`` identifies the Torch parameter group owning ``param``. 

55 """ 

56 

57 adapter_index: int 

58 param: Any 

59 grad: Any 

60 slots: List[SwapSlot] 

61 

62 

63class AdamSwapAdapter(StateSwapAdapter): 

64 """Adam/AdamW optimizer-state swap adapter.""" 

65 

66 functional_name = "adam" 

67 decoupled_weight_decay = False 

68 is_new_adamw = False 

69 supports_fused = False 

70 

71 def validate(self) -> None: 

72 """Validate unsupported optimizer flags.""" 

73 for group in self.optimizer.param_groups: 

74 if group.get("foreach", False) is True: 

75 raise ValueError("Swap optimizer does not support foreach=True.") 

76 if group.get("fused", False) is True and not self.supports_fused: 

77 raise ValueError("Swap optimizer does not support fused=True.") 

78 if group.get("differentiable", False): 

79 raise ValueError("Swap optimizer does not support differentiable=True.") 

80 if group.get("capturable", False): 

81 raise ValueError("Swap optimizer does not support capturable=True.") 

82 

83 def prepare_step(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: 

84 """Initialize lazy state and collect this step's update units.""" 

85 if args or kwargs: 

86 raise ValueError("Torch swap optimizer step does not support closure or extra arguments.") 

87 if self.runtime.packed_enabled: 

88 return self._prepare_packed_step() 

89 

90 units = [] 

91 for group_index, group in enumerate(self.optimizer.param_groups): 

92 if self.is_new_adamw: 

93 group["step"] = (group.get("step") or 0) + 1 

94 for param in group["params"]: 

95 grad = getattr(param, "grad", None) 

96 if grad is None: 

97 continue 

98 if getattr(grad, "is_sparse", False): 

99 raise ValueError("Swap optimizer only supports dense Adam/AdamW gradients.") 

100 state = self.optimizer.state[param] 

101 self._init_param_state(param, group) 

102 slots = self._build_slots(param, state) 

103 units.append(AdamUpdateUnit( 

104 adapter_index=group_index, 

105 param=param, 

106 grad=grad, 

107 slots=slots, 

108 )) 

109 return {"units": units} 

110 

111 def _prepare_packed_step(self) -> Dict[str, Any]: 

112 """Build a stable packed layout while retaining inactive materialized states.""" 

113 records = [] 

114 for group_index, group in enumerate(self.optimizer.param_groups): 

115 if self.is_new_adamw: 

116 group["step"] = (group.get("step") or 0) + 1 

117 for param in group["params"]: 

118 grad = getattr(param, "grad", None) 

119 state = self.optimizer.state.get(param) 

120 if grad is not None: 

121 if getattr(grad, "is_sparse", False): 

122 raise ValueError("Swap optimizer only supports dense Adam/AdamW gradients.") 

123 state = self.optimizer.state[param] 

124 self._init_param_state(param, group) 

125 if state: 

126 self._register_present_slots(param, state) 

127 has_slots = any((id(param), key) in self._slots for key in self._configured_state_keys()) 

128 if grad is None and not has_slots: 

129 continue 

130 records.append((group_index, param, grad)) 

131 

132 self.runtime.prepare_packed_host(self._ordered_slots()) 

133 self.publish_packed_state() 

134 units = [] 

135 for group_index, param, grad in records: 

136 state = self.optimizer.state[param] 

137 slots = self._build_slots(param, state) 

138 if grad is None and not any(slot.swappable and slot.packed for slot in slots): 

139 continue 

140 units.append(AdamUpdateUnit( 

141 adapter_index=group_index, 

142 param=param, 

143 grad=grad, 

144 slots=slots, 

145 )) 

146 return {"units": units} 

147 

148 def _collect_group_args( 

149 self, 

150 units: Sequence[AdamUpdateUnit], 

151 group: Dict[str, Any], 

152 ) -> GroupArgs: 

153 """Gather the per-parameter argument lists one optimizer step needs. 

154 

155 Units without a gradient are skipped. ``max_exp_avg_sqs`` stays empty 

156 unless the group is amsgrad, and ``state_steps`` holds ``None`` for new 

157 AdamW, which advances the step counter itself. 

158 """ 

159 args = GroupArgs([], [], [], [], [], []) 

160 for unit in units: 

161 if unit.grad is None: 

162 continue 

163 state = self.optimizer.state[unit.param] 

164 args.params.append(unit.param) 

165 args.grads.append(unit.grad) 

166 args.exp_avgs.append(_slot_tensor(unit, "exp_avg", state["exp_avg"])) 

167 args.exp_avg_sqs.append(_slot_tensor(unit, "exp_avg_sq", state["exp_avg_sq"])) 

168 if group.get("amsgrad", False): 

169 args.max_exp_avg_sqs.append( 

170 _slot_tensor(unit, "max_exp_avg_sq", state["max_exp_avg_sq"]) 

171 ) 

172 if self.is_new_adamw: 

173 args.state_steps.append(None) 

174 else: 

175 args.state_steps.append(state["step"]) 

176 return args 

177 

178 def step_batch(self, batch: List[AdamUpdateUnit], step_context: Dict[str, Any]) -> None: 

179 """Run Torch functional Adam/AdamW for one batch.""" 

180 del step_context 

181 by_group: Dict[int, List[AdamUpdateUnit]] = defaultdict(list) 

182 for unit in batch: 

183 by_group[unit.adapter_index].append(unit) 

184 for group_index, units in by_group.items(): 

185 self._step_group(self.optimizer.param_groups[group_index], units) 

186 

187 def _step_group(self, group: Dict[str, Any], units: List[AdamUpdateUnit]) -> None: 

188 """Run one group's parameters through the matching functional Adam/AdamW.""" 

189 args = self._collect_group_args(units, group) 

190 params = args.params 

191 

192 if not params: 

193 return 

194 

195 if self.is_new_adamw: 

196 self._step_new_adamw( 

197 group, 

198 args.params, 

199 args.grads, 

200 args.exp_avgs, 

201 args.exp_avg_sqs, 

202 args.max_exp_avg_sqs, 

203 ) 

204 return 

205 

206 func = getattr(torch.optim._functional, self.functional_name) 

207 kwargs = { 

208 "amsgrad": group["amsgrad"], 

209 "beta1": group["betas"][0], 

210 "beta2": group["betas"][1], 

211 "lr": group["lr"], 

212 "weight_decay": group["weight_decay"], 

213 "eps": group["eps"], 

214 "maximize": group["maximize"], 

215 "foreach": False, 

216 "capturable": False, 

217 "differentiable": False, 

218 "fused": bool(group.get("fused", False)), 

219 "grad_scale": getattr(self.optimizer, "grad_scale", None), 

220 "found_inf": getattr(self.optimizer, "found_inf", None), 

221 "has_complex": False, 

222 } 

223 if self.functional_name == "adam": 

224 if "decoupled_weight_decay" in inspect.signature(func).parameters: 

225 kwargs["decoupled_weight_decay"] = self._decoupled_weight_decay(group) 

226 func( 

227 args.params, 

228 args.grads, 

229 args.exp_avgs, 

230 args.exp_avg_sqs, 

231 args.max_exp_avg_sqs, 

232 args.state_steps, 

233 **kwargs, 

234 ) 

235 

236 def _decoupled_weight_decay(self, group: Dict[str, Any]) -> bool: 

237 """Resolve the decoupled weight decay flag for one parameter group.""" 

238 return self.decoupled_weight_decay or group.get("decoupled_weight_decay", False) 

239 

240 def _step_new_adamw( 

241 self, 

242 group: Dict[str, Any], 

243 params: Sequence[Any], 

244 grads: Sequence[Any], 

245 exp_avgs: Sequence[Any], 

246 exp_avg_sqs: Sequence[Any], 

247 max_exp_avg_sqs: Sequence[Any], 

248 ) -> None: 

249 """Run the new AdamW functional for one group.""" 

250 if params and params[0].device.type == "cpu": 

251 # torch.optim._functional.adamw increments tensor state_steps 

252 # internally. New AdamW already advanced group["step"] in 

253 # prepare_step(), so feed step - 1 to preserve outer-step 

254 # semantics for CPU-only tests. 

255 step_tensor = torch.tensor(float(group["step"] - 1), dtype=torch.float32) 

256 torch.optim._functional.adamw( 

257 params, 

258 grads, 

259 exp_avgs, 

260 exp_avg_sqs, 

261 max_exp_avg_sqs, 

262 [step_tensor] * len(params), 

263 amsgrad=group["amsgrad"], 

264 beta1=group["betas"][0], 

265 beta2=group["betas"][1], 

266 lr=group["lr"], 

267 weight_decay=group["weight_decay"], 

268 eps=group["eps"], 

269 maximize=group["maximize"], 

270 foreach=False, 

271 capturable=False, 

272 differentiable=False, 

273 fused=False, 

274 grad_scale=None, 

275 found_inf=None, 

276 has_complex=False, 

277 ) 

278 return 

279 _new_adamw_func()( 

280 params, 

281 grads, 

282 exp_avgs, 

283 exp_avg_sqs, 

284 max_exp_avg_sqs, 

285 group["step"], 

286 amsgrad=group["amsgrad"], 

287 beta1=group["betas"][0], 

288 beta2=group["betas"][1], 

289 lr=group["lr"], 

290 weight_decay=group["weight_decay"], 

291 eps=group["eps"], 

292 maximize=group["maximize"], 

293 ) 

294 

295 def _init_param_state(self, param: Any, group: Dict[str, Any]) -> None: 

296 """Initialize missing Adam state and swap slots for one parameter.""" 

297 state = self.optimizer.state[param] 

298 if not self.is_new_adamw and len(state) == 0: 

299 step_device = ( 

300 param.device 

301 if group.get("fused", False) 

302 else ("cpu" if self.runtime.packed_enabled else param.device) 

303 ) 

304 state["step"] = torch.zeros((), dtype=torch.float32, device=step_device) 

305 state_keys = ["exp_avg", "exp_avg_sq"] 

306 if group.get("amsgrad", False): 

307 state_keys.append("max_exp_avg_sq") 

308 configured_keys = set(self._configured_state_keys()) 

309 for key in state_keys: 

310 if key in state or (id(param), key) in self._slots: 

311 continue 

312 if ( 

313 key in configured_keys 

314 and not self.runtime.packed_enabled 

315 and self.runtime.is_swappable_tensor(param, self.config.min_numel) 

316 ): 

317 cpu_tensor = self.runtime.make_zero_cpu_tensor_like(param) 

318 device_tensor = torch.empty_like(param, memory_format=torch.preserve_format) 

319 state[key] = device_tensor 

320 slot = self._make_slot(key, device_tensor) 

321 slot.cpu_tensor = cpu_tensor 

322 slot.state = "host" 

323 self._slots[(id(param), key)] = slot 

324 self.runtime.release_device_storage(slot) 

325 continue 

326 if key in configured_keys and self.runtime.is_packable_template(param, self.config.min_numel): 

327 self._slots[(id(param), key)] = self._make_slot(key, None, template=param) 

328 continue 

329 state[key] = torch.zeros_like(param, memory_format=torch.preserve_format) 

330 

331 @staticmethod 

332 def default_state_keys() -> Tuple[str, ...]: 

333 """Return the swap state keys an Adam/AdamW optimizer owns.""" 

334 return ADAM_STATE_KEYS 

335 

336 

337class TorchNativeAdamAdapter(AdamSwapAdapter): 

338 """Adapter for ``torch.optim.Adam``.""" 

339 

340 functional_name = "adam" 

341 

342 @classmethod 

343 def matches(cls, optimizer: Any) -> bool: 

344 """Match ``torch.optim.Adam``, excluding its AdamW subclass.""" 

345 # AdamW inherits Adam in PyTorch. Keep Adam subclasses supported, but 

346 # let AdamW select its dedicated adapter (which preserves fused=True). 

347 return ( 

348 isinstance(optimizer, torch.optim.Adam) 

349 and not isinstance(optimizer, torch.optim.AdamW) 

350 ) 

351 

352 

353class TorchNativeAdamWAdapter(AdamSwapAdapter): 

354 """Adapter for ``torch.optim.AdamW``.""" 

355 

356 functional_name = "adamw" 

357 supports_fused = True 

358 

359 @classmethod 

360 def matches(cls, optimizer: Any) -> bool: 

361 """Match ``torch.optim.AdamW``.""" 

362 return isinstance(optimizer, torch.optim.AdamW) 

363 

364 

365class TorchNewAdamWAdapter(AdamSwapAdapter): 

366 """Adapter for hyper-parallel's fused AdamW.""" 

367 

368 functional_name = "adamw" 

369 is_new_adamw = True 

370 

371 @classmethod 

372 def matches(cls, optimizer: Any) -> bool: 

373 """Match hyper-parallel's lazily imported AdamW.""" 

374 return isinstance(optimizer, _new_adamw_cls()) 

375 

376def _new_adamw_cls() -> type: 

377 """Return hyper-parallel's AdamW class without importing it at module load.""" 

378 from hyper_parallel.core.optimizer.adamw import ( # pylint: disable=import-outside-toplevel 

379 AdamW as new_adamw_cls, 

380 ) 

381 return new_adamw_cls 

382 

383 

384def _new_adamw_func() -> Callable[..., Any]: 

385 """Return hyper-parallel's functional AdamW without importing it at module load.""" 

386 from hyper_parallel.core.optimizer.adamw import ( # pylint: disable=import-outside-toplevel 

387 adamw as new_adamw_func, 

388 ) 

389 return new_adamw_func 

390 

391 

392ADAM_ADAPTERS = ( 

393 TorchNewAdamWAdapter, 

394 TorchNativeAdamAdapter, 

395 TorchNativeAdamWAdapter, 

396) 

397 

398 

399def build_adam_swap_adapter(optimizer: Any, config: Any, runtime: Any) -> AdamSwapAdapter: 

400 """Build the Adam/AdamW swap adapter matching ``optimizer``. 

401 

402 Args: 

403 optimizer: Wrapped Torch or HyperParallel Adam/AdamW optimizer. 

404 config: Resolved ``SwapOptimizerConfig``. 

405 runtime: Shared ``PipelineSwapRuntime``. 

406 

407 Returns: 

408 The adapter instance for ``optimizer``. 

409 

410 Raises: 

411 ValueError: If ``optimizer`` is not a supported Adam/AdamW optimizer. 

412 """ 

413 for adapter_cls in ADAM_ADAPTERS: 

414 if adapter_cls.matches(optimizer): 

415 return adapter_cls(optimizer, config, runtime) 

416 raise ValueError( 

417 "Swap optimizer only supports torch.optim.Adam, torch.optim.AdamW, " 

418 "and hyper_parallel.core.optimizer.adamw.AdamW on the Torch backend. " 

419 f"Got {type(optimizer)!r}." 

420 ) 

421 

422 

423def swap_adam(optimizer: Any, config: Any) -> Any: 

424 """Wrap an Adam/AdamW optimizer with optimizer-state swap. 

425 

426 Args: 

427 optimizer: Supported Adam/AdamW optimizer instance. 

428 config: Resolved ``SwapOptimizerConfig``. 

429 

430 Returns: 

431 The shared core swap-optimizer wrapper. 

432 """ 

433 return CoreSwapOptimizer( 

434 optimizer, 

435 config, 

436 adapter_factory=build_adam_swap_adapter, 

437 )