Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / swap_adam.py: 88%
190 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Adam/AdamW optimizer-state swap adapters."""
17# pylint: disable=protected-access
19from __future__ import annotations
21import inspect
22from collections import defaultdict
23from dataclasses import dataclass
24from typing import Any, Callable, Dict, List, NamedTuple, Sequence, Tuple
26import torch
28from hyper_parallel.core.optimizer.swap_optimizer_base import (
29 StateSwapAdapter,
30 SwapOptimizer as CoreSwapOptimizer,
31 SwapSlot,
32 _slot_tensor,
33)
35#: Logical state keys an Adam/AdamW optimizer owns.
36ADAM_STATE_KEYS = ("exp_avg", "exp_avg_sq", "max_exp_avg_sq")
39class GroupArgs(NamedTuple):
40 """Per-parameter argument lists one functional Adam/AdamW step consumes."""
42 params: List[Any]
43 grads: List[Any]
44 exp_avgs: List[Any]
45 exp_avg_sqs: List[Any]
46 max_exp_avg_sqs: List[Any]
47 state_steps: List[Any]
50@dataclass
51class AdamUpdateUnit:
52 """Per-parameter Adam/AdamW update unit consumed by the pipeline runtime.
54 ``adapter_index`` identifies the Torch parameter group owning ``param``.
55 """
57 adapter_index: int
58 param: Any
59 grad: Any
60 slots: List[SwapSlot]
63class AdamSwapAdapter(StateSwapAdapter):
64 """Adam/AdamW optimizer-state swap adapter."""
66 functional_name = "adam"
67 decoupled_weight_decay = False
68 is_new_adamw = False
69 supports_fused = False
71 def validate(self) -> None:
72 """Validate unsupported optimizer flags."""
73 for group in self.optimizer.param_groups:
74 if group.get("foreach", False) is True:
75 raise ValueError("Swap optimizer does not support foreach=True.")
76 if group.get("fused", False) is True and not self.supports_fused:
77 raise ValueError("Swap optimizer does not support fused=True.")
78 if group.get("differentiable", False):
79 raise ValueError("Swap optimizer does not support differentiable=True.")
80 if group.get("capturable", False):
81 raise ValueError("Swap optimizer does not support capturable=True.")
83 def prepare_step(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
84 """Initialize lazy state and collect this step's update units."""
85 if args or kwargs:
86 raise ValueError("Torch swap optimizer step does not support closure or extra arguments.")
87 if self.runtime.packed_enabled:
88 return self._prepare_packed_step()
90 units = []
91 for group_index, group in enumerate(self.optimizer.param_groups):
92 if self.is_new_adamw:
93 group["step"] = (group.get("step") or 0) + 1
94 for param in group["params"]:
95 grad = getattr(param, "grad", None)
96 if grad is None:
97 continue
98 if getattr(grad, "is_sparse", False):
99 raise ValueError("Swap optimizer only supports dense Adam/AdamW gradients.")
100 state = self.optimizer.state[param]
101 self._init_param_state(param, group)
102 slots = self._build_slots(param, state)
103 units.append(AdamUpdateUnit(
104 adapter_index=group_index,
105 param=param,
106 grad=grad,
107 slots=slots,
108 ))
109 return {"units": units}
111 def _prepare_packed_step(self) -> Dict[str, Any]:
112 """Build a stable packed layout while retaining inactive materialized states."""
113 records = []
114 for group_index, group in enumerate(self.optimizer.param_groups):
115 if self.is_new_adamw:
116 group["step"] = (group.get("step") or 0) + 1
117 for param in group["params"]:
118 grad = getattr(param, "grad", None)
119 state = self.optimizer.state.get(param)
120 if grad is not None:
121 if getattr(grad, "is_sparse", False):
122 raise ValueError("Swap optimizer only supports dense Adam/AdamW gradients.")
123 state = self.optimizer.state[param]
124 self._init_param_state(param, group)
125 if state:
126 self._register_present_slots(param, state)
127 has_slots = any((id(param), key) in self._slots for key in self._configured_state_keys())
128 if grad is None and not has_slots:
129 continue
130 records.append((group_index, param, grad))
132 self.runtime.prepare_packed_host(self._ordered_slots())
133 self.publish_packed_state()
134 units = []
135 for group_index, param, grad in records:
136 state = self.optimizer.state[param]
137 slots = self._build_slots(param, state)
138 if grad is None and not any(slot.swappable and slot.packed for slot in slots):
139 continue
140 units.append(AdamUpdateUnit(
141 adapter_index=group_index,
142 param=param,
143 grad=grad,
144 slots=slots,
145 ))
146 return {"units": units}
148 def _collect_group_args(
149 self,
150 units: Sequence[AdamUpdateUnit],
151 group: Dict[str, Any],
152 ) -> GroupArgs:
153 """Gather the per-parameter argument lists one optimizer step needs.
155 Units without a gradient are skipped. ``max_exp_avg_sqs`` stays empty
156 unless the group is amsgrad, and ``state_steps`` holds ``None`` for new
157 AdamW, which advances the step counter itself.
158 """
159 args = GroupArgs([], [], [], [], [], [])
160 for unit in units:
161 if unit.grad is None:
162 continue
163 state = self.optimizer.state[unit.param]
164 args.params.append(unit.param)
165 args.grads.append(unit.grad)
166 args.exp_avgs.append(_slot_tensor(unit, "exp_avg", state["exp_avg"]))
167 args.exp_avg_sqs.append(_slot_tensor(unit, "exp_avg_sq", state["exp_avg_sq"]))
168 if group.get("amsgrad", False):
169 args.max_exp_avg_sqs.append(
170 _slot_tensor(unit, "max_exp_avg_sq", state["max_exp_avg_sq"])
171 )
172 if self.is_new_adamw:
173 args.state_steps.append(None)
174 else:
175 args.state_steps.append(state["step"])
176 return args
178 def step_batch(self, batch: List[AdamUpdateUnit], step_context: Dict[str, Any]) -> None:
179 """Run Torch functional Adam/AdamW for one batch."""
180 del step_context
181 by_group: Dict[int, List[AdamUpdateUnit]] = defaultdict(list)
182 for unit in batch:
183 by_group[unit.adapter_index].append(unit)
184 for group_index, units in by_group.items():
185 self._step_group(self.optimizer.param_groups[group_index], units)
187 def _step_group(self, group: Dict[str, Any], units: List[AdamUpdateUnit]) -> None:
188 """Run one group's parameters through the matching functional Adam/AdamW."""
189 args = self._collect_group_args(units, group)
190 params = args.params
192 if not params:
193 return
195 if self.is_new_adamw:
196 self._step_new_adamw(
197 group,
198 args.params,
199 args.grads,
200 args.exp_avgs,
201 args.exp_avg_sqs,
202 args.max_exp_avg_sqs,
203 )
204 return
206 func = getattr(torch.optim._functional, self.functional_name)
207 kwargs = {
208 "amsgrad": group["amsgrad"],
209 "beta1": group["betas"][0],
210 "beta2": group["betas"][1],
211 "lr": group["lr"],
212 "weight_decay": group["weight_decay"],
213 "eps": group["eps"],
214 "maximize": group["maximize"],
215 "foreach": False,
216 "capturable": False,
217 "differentiable": False,
218 "fused": bool(group.get("fused", False)),
219 "grad_scale": getattr(self.optimizer, "grad_scale", None),
220 "found_inf": getattr(self.optimizer, "found_inf", None),
221 "has_complex": False,
222 }
223 if self.functional_name == "adam":
224 if "decoupled_weight_decay" in inspect.signature(func).parameters:
225 kwargs["decoupled_weight_decay"] = self._decoupled_weight_decay(group)
226 func(
227 args.params,
228 args.grads,
229 args.exp_avgs,
230 args.exp_avg_sqs,
231 args.max_exp_avg_sqs,
232 args.state_steps,
233 **kwargs,
234 )
236 def _decoupled_weight_decay(self, group: Dict[str, Any]) -> bool:
237 """Resolve the decoupled weight decay flag for one parameter group."""
238 return self.decoupled_weight_decay or group.get("decoupled_weight_decay", False)
240 def _step_new_adamw(
241 self,
242 group: Dict[str, Any],
243 params: Sequence[Any],
244 grads: Sequence[Any],
245 exp_avgs: Sequence[Any],
246 exp_avg_sqs: Sequence[Any],
247 max_exp_avg_sqs: Sequence[Any],
248 ) -> None:
249 """Run the new AdamW functional for one group."""
250 if params and params[0].device.type == "cpu":
251 # torch.optim._functional.adamw increments tensor state_steps
252 # internally. New AdamW already advanced group["step"] in
253 # prepare_step(), so feed step - 1 to preserve outer-step
254 # semantics for CPU-only tests.
255 step_tensor = torch.tensor(float(group["step"] - 1), dtype=torch.float32)
256 torch.optim._functional.adamw(
257 params,
258 grads,
259 exp_avgs,
260 exp_avg_sqs,
261 max_exp_avg_sqs,
262 [step_tensor] * len(params),
263 amsgrad=group["amsgrad"],
264 beta1=group["betas"][0],
265 beta2=group["betas"][1],
266 lr=group["lr"],
267 weight_decay=group["weight_decay"],
268 eps=group["eps"],
269 maximize=group["maximize"],
270 foreach=False,
271 capturable=False,
272 differentiable=False,
273 fused=False,
274 grad_scale=None,
275 found_inf=None,
276 has_complex=False,
277 )
278 return
279 _new_adamw_func()(
280 params,
281 grads,
282 exp_avgs,
283 exp_avg_sqs,
284 max_exp_avg_sqs,
285 group["step"],
286 amsgrad=group["amsgrad"],
287 beta1=group["betas"][0],
288 beta2=group["betas"][1],
289 lr=group["lr"],
290 weight_decay=group["weight_decay"],
291 eps=group["eps"],
292 maximize=group["maximize"],
293 )
295 def _init_param_state(self, param: Any, group: Dict[str, Any]) -> None:
296 """Initialize missing Adam state and swap slots for one parameter."""
297 state = self.optimizer.state[param]
298 if not self.is_new_adamw and len(state) == 0:
299 step_device = (
300 param.device
301 if group.get("fused", False)
302 else ("cpu" if self.runtime.packed_enabled else param.device)
303 )
304 state["step"] = torch.zeros((), dtype=torch.float32, device=step_device)
305 state_keys = ["exp_avg", "exp_avg_sq"]
306 if group.get("amsgrad", False):
307 state_keys.append("max_exp_avg_sq")
308 configured_keys = set(self._configured_state_keys())
309 for key in state_keys:
310 if key in state or (id(param), key) in self._slots:
311 continue
312 if (
313 key in configured_keys
314 and not self.runtime.packed_enabled
315 and self.runtime.is_swappable_tensor(param, self.config.min_numel)
316 ):
317 cpu_tensor = self.runtime.make_zero_cpu_tensor_like(param)
318 device_tensor = torch.empty_like(param, memory_format=torch.preserve_format)
319 state[key] = device_tensor
320 slot = self._make_slot(key, device_tensor)
321 slot.cpu_tensor = cpu_tensor
322 slot.state = "host"
323 self._slots[(id(param), key)] = slot
324 self.runtime.release_device_storage(slot)
325 continue
326 if key in configured_keys and self.runtime.is_packable_template(param, self.config.min_numel):
327 self._slots[(id(param), key)] = self._make_slot(key, None, template=param)
328 continue
329 state[key] = torch.zeros_like(param, memory_format=torch.preserve_format)
331 @staticmethod
332 def default_state_keys() -> Tuple[str, ...]:
333 """Return the swap state keys an Adam/AdamW optimizer owns."""
334 return ADAM_STATE_KEYS
337class TorchNativeAdamAdapter(AdamSwapAdapter):
338 """Adapter for ``torch.optim.Adam``."""
340 functional_name = "adam"
342 @classmethod
343 def matches(cls, optimizer: Any) -> bool:
344 """Match ``torch.optim.Adam``, excluding its AdamW subclass."""
345 # AdamW inherits Adam in PyTorch. Keep Adam subclasses supported, but
346 # let AdamW select its dedicated adapter (which preserves fused=True).
347 return (
348 isinstance(optimizer, torch.optim.Adam)
349 and not isinstance(optimizer, torch.optim.AdamW)
350 )
353class TorchNativeAdamWAdapter(AdamSwapAdapter):
354 """Adapter for ``torch.optim.AdamW``."""
356 functional_name = "adamw"
357 supports_fused = True
359 @classmethod
360 def matches(cls, optimizer: Any) -> bool:
361 """Match ``torch.optim.AdamW``."""
362 return isinstance(optimizer, torch.optim.AdamW)
365class TorchNewAdamWAdapter(AdamSwapAdapter):
366 """Adapter for hyper-parallel's fused AdamW."""
368 functional_name = "adamw"
369 is_new_adamw = True
371 @classmethod
372 def matches(cls, optimizer: Any) -> bool:
373 """Match hyper-parallel's lazily imported AdamW."""
374 return isinstance(optimizer, _new_adamw_cls())
376def _new_adamw_cls() -> type:
377 """Return hyper-parallel's AdamW class without importing it at module load."""
378 from hyper_parallel.core.optimizer.adamw import ( # pylint: disable=import-outside-toplevel
379 AdamW as new_adamw_cls,
380 )
381 return new_adamw_cls
384def _new_adamw_func() -> Callable[..., Any]:
385 """Return hyper-parallel's functional AdamW without importing it at module load."""
386 from hyper_parallel.core.optimizer.adamw import ( # pylint: disable=import-outside-toplevel
387 adamw as new_adamw_func,
388 )
389 return new_adamw_func
392ADAM_ADAPTERS = (
393 TorchNewAdamWAdapter,
394 TorchNativeAdamAdapter,
395 TorchNativeAdamWAdapter,
396)
399def build_adam_swap_adapter(optimizer: Any, config: Any, runtime: Any) -> AdamSwapAdapter:
400 """Build the Adam/AdamW swap adapter matching ``optimizer``.
402 Args:
403 optimizer: Wrapped Torch or HyperParallel Adam/AdamW optimizer.
404 config: Resolved ``SwapOptimizerConfig``.
405 runtime: Shared ``PipelineSwapRuntime``.
407 Returns:
408 The adapter instance for ``optimizer``.
410 Raises:
411 ValueError: If ``optimizer`` is not a supported Adam/AdamW optimizer.
412 """
413 for adapter_cls in ADAM_ADAPTERS:
414 if adapter_cls.matches(optimizer):
415 return adapter_cls(optimizer, config, runtime)
416 raise ValueError(
417 "Swap optimizer only supports torch.optim.Adam, torch.optim.AdamW, "
418 "and hyper_parallel.core.optimizer.adamw.AdamW on the Torch backend. "
419 f"Got {type(optimizer)!r}."
420 )
423def swap_adam(optimizer: Any, config: Any) -> Any:
424 """Wrap an Adam/AdamW optimizer with optimizer-state swap.
426 Args:
427 optimizer: Supported Adam/AdamW optimizer instance.
428 config: Resolved ``SwapOptimizerConfig``.
430 Returns:
431 The shared core swap-optimizer wrapper.
432 """
433 return CoreSwapOptimizer(
434 optimizer,
435 config,
436 adapter_factory=build_adam_swap_adapter,
437 )