Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / optimizer / swap_optimizer_base.py: 96%

1335 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Generic optimizer-state swap runtime, slots and adapter base.""" 

16# pylint: disable=protected-access 

17 

18from __future__ import annotations 

19 

20import contextlib 

21import copy 

22import itertools 

23import logging 

24import weakref 

25from dataclasses import dataclass, field 

26from typing import Any, Callable, Dict, Iterable, List, Optional, Protocol, Sequence, Tuple 

27 

28# Optimizer runtime is Torch-only; the legacy checker still flags core imports. 

29import torch 

30 

31logger = logging.getLogger(__name__) 

32 

33# Logical optimizer-state keys every swap family in this codebase understands. 

34# This is the vocabulary ``SwapOptimizerConfig.state_keys`` may name; each 

35# adapter still validates that a key belongs to its own optimizer family. 

36KNOWN_STATE_KEYS = ( 

37 "exp_avg", 

38 "exp_avg_sq", 

39 "max_exp_avg_sq", 

40 "momentum_buffer", 

41) 

42_PACKED_ALIGNMENT_BYTES = 512 

43_DEVICE_TYPE = "npu" 

44_MISSING_CHECKPOINT_STATE = object() 

45 

46 

47def _restore_dtensor_metadata(template: Any, local_tensor: Any) -> Any: 

48 """Wrap a local tensor with the logical DTensor layout of ``template``.""" 

49 to_local = getattr(template, "to_local", None) 

50 if not callable(to_local): 

51 return local_tensor 

52 

53 layout = getattr(template, "layout", None) 

54 if hasattr(layout, "tensor_stride"): 

55 logical_stride = layout.tensor_stride 

56 else: 

57 logical_stride = tuple(template.stride()) 

58 if logical_stride is None: 

59 logical_stride = [1] * len(template.shape) 

60 for index in range(len(logical_stride) - 2, -1, -1): 

61 logical_stride[index] = logical_stride[index + 1] * max(int(template.shape[index + 1]), 1) 

62 

63 return type(template).from_local( 

64 local_tensor, 

65 device_mesh=template.device_mesh, 

66 placements=template.placements, 

67 run_check=False, 

68 shape=template.shape, 

69 stride=tuple(logical_stride), 

70 ) 

71 

72 

73@dataclass 

74class SwapSlot: 

75 """One logical optimizer state tensor that may be swapped.""" 

76 

77 name: str 

78 tensor: Any 

79 cpu_tensor: Optional[Any] = None 

80 storage_nbytes: int = 0 

81 swappable: bool = True 

82 state: str = "device" 

83 event: Optional[Any] = None 

84 shape: tuple[int, ...] = () 

85 dtype: Optional[Any] = None 

86 device: Optional[Any] = None 

87 numel: int = 0 

88 host_offset: int = 0 

89 packed: bool = False 

90 logical_tensor: Optional[Any] = None 

91 

92 def bind_tensor(self, tensor: Any) -> None: 

93 """Bind the slot to a host or staging tensor view.""" 

94 device = getattr(tensor, "device", None) 

95 device_type = getattr(device, "type", None) 

96 if self.logical_tensor is not None: 

97 # DTensor exposes no public setter for replacing its local shard. 

98 # Rebind the shell on host too, so it cannot retain an arena view. 

99 setattr(self.logical_tensor, "_local_tensor", tensor) 

100 self.logical_tensor.data = tensor 

101 if self.logical_tensor is not None and device_type != "cpu": 

102 self.tensor = self.logical_tensor 

103 else: 

104 self.tensor = tensor 

105 

106 @property 

107 def checkpoint_tensor(self) -> Any: 

108 """Return the CPU tensor when present, otherwise the live tensor.""" 

109 return self.cpu_tensor if self.cpu_tensor is not None else self.tensor 

110 

111 

112class SwapUpdateUnit(Protocol): 

113 """Minimal contract the pipeline runtime requires of an update unit. 

114 

115 The runtime only ever reads a unit's swappable slots, so an update unit is 

116 free to carry algorithm-specific metadata (a single parameter and gradient, 

117 a Muon HSDP assignment, collective ordering, and so on) as long as it 

118 exposes ``slots``. 

119 """ 

120 

121 slots: List[SwapSlot] 

122 

123 

124@dataclass 

125class UpdateUnit: 

126 """Generic per-parameter update unit: parameter, gradient and swap slots. 

127 

128 Kept as a concrete unit for optimizers whose atomic update is one parameter, 

129 and as the runtime's default unit type. ``adapter_index`` identifies the 

130 owning parameter group. Optimizers with a richer atomic unit (Muon HSDP 

131 assignments, for example) define their own unit exposing ``slots``. 

132 """ 

133 

134 adapter_index: int 

135 param: Any 

136 grad: Any 

137 slots: List[SwapSlot] 

138 

139 

140class PipelineSwapRuntime: 

141 """Torch tensor storage/copy runtime and pipeline orchestration.""" 

142 

143 def __init__(self, config: Any) -> None: 

144 """Initialize instance-owned streams and packed storage metadata.""" 

145 self.config = config 

146 self._copy_stream: Optional[Any] = None 

147 self._packed_enabled = bool(getattr(config, "packed_swap", True)) 

148 self._host_buffers: Dict[Any, Any] = {} 

149 self._host_layout_signature = () 

150 self._packed_batch_plans: List[_PackedBatchPlan] = [] 

151 self._staging_arenas: List[Optional[_StagingArena]] = [None, None] 

152 self._packed_ready_events: Dict[int, Any] = {} 

153 self._packed_offload_events: Dict[int, Any] = {} 

154 self._packed_tail_event: Optional[Any] = None 

155 self._packed_device_views: Dict[Any, Any] = {} 

156 self._packed_slot_bind_hook: Optional[Callable[[SwapSlot], None]] = None 

157 

158 @property 

159 def packed_enabled(self) -> bool: 

160 """Return whether this runtime may build packed state candidates.""" 

161 return self._packed_enabled 

162 

163 @property 

164 def copy_stream(self) -> Optional[Any]: 

165 """Return the transfer stream after its lazy initialization.""" 

166 return self._copy_stream 

167 

168 def set_packed_slot_bind_hook(self, hook: Callable[[SwapSlot], None]) -> None: 

169 """Install the adapter hook that republishes a packed slot's live view.""" 

170 self._packed_slot_bind_hook = hook 

171 

172 def _bind_packed_slot(self, slot: SwapSlot, tensor: Any) -> None: 

173 """Bind a packed view and notify the owning state adapter.""" 

174 slot.bind_tensor(tensor) 

175 if self._packed_slot_bind_hook is not None: 

176 self._packed_slot_bind_hook(slot) 

177 

178 def partition( 

179 self, 

180 units: Sequence[SwapUpdateUnit], 

181 unit_costs: Optional[Sequence[int]] = None, 

182 ) -> List[List[SwapUpdateUnit]]: 

183 """Partition consecutive units using explicit costs or local state bytes. 

184 

185 Args: 

186 units: Atomic updates in execution order, including empty local units. 

187 unit_costs: Optional non-negative integer costs, one per unit. A 

188 distributed adapter supplies rank-independent planning weights. 

189 

190 Returns: 

191 Consecutive batches retaining every unit in its original order. 

192 

193 Raises: 

194 ValueError: If explicit costs have the wrong length or invalid values. 

195 """ 

196 ordered_units = list(units) 

197 if unit_costs is None: 

198 unit_costs = [self._unit_cost(unit) for unit in ordered_units] 

199 else: 

200 unit_costs = list(unit_costs) 

201 if len(unit_costs) != len(ordered_units): 

202 raise ValueError("unit_costs must contain exactly one cost per update unit.") 

203 if any(isinstance(cost, bool) or not isinstance(cost, int) or cost < 0 for cost in unit_costs): 

204 raise ValueError("unit_costs must contain non-negative integers.") 

205 if not ordered_units: 

206 return [] 

207 swap_times = max(1, min(int(self.config.swap_times), len(ordered_units))) 

208 unit_costs = [max(1, cost) for cost in unit_costs] 

209 remaining_cost = sum(unit_costs) 

210 batches: List[List[SwapUpdateUnit]] = [] 

211 start = 0 

212 for batch_index in range(swap_times - 1): 

213 remaining_batches = swap_times - batch_index 

214 

215 # Compare against the remaining average without introducing floating-point rounding. 

216 # Bind the loop state as defaults so the closure captures this iteration's values. 

217 def should_stop_growing( 

218 end: int, 

219 current_cost: int, 

220 next_cost: int, 

221 remaining_batches: int = remaining_batches, 

222 remaining_cost: int = remaining_cost, 

223 start: int = start, 

224 ) -> bool: 

225 """Return whether adding the next unit would not improve balance.""" 

226 current_distance = abs(remaining_cost - current_cost * remaining_batches) 

227 next_distance = abs(remaining_cost - (current_cost + next_cost) * remaining_batches) 

228 return end > start and current_distance <= next_distance 

229 

230 max_end = len(ordered_units) - remaining_batches + 1 

231 end, current_cost = self._grow_balanced_segment(unit_costs, start, max_end, should_stop_growing) 

232 batches.append(ordered_units[start:end]) 

233 remaining_cost -= current_cost 

234 start = end 

235 batches.append(ordered_units[start:]) 

236 return batches 

237 

238 def run_pipeline( 

239 self, 

240 batches: Sequence[Sequence[UpdateUnit]], 

241 step_context: Any, 

242 step_batch: Callable[[List[UpdateUnit], Any], Any], 

243 ) -> List[Any]: 

244 """Run one-batch-ahead prefetch while releasing completed offloads before widening the window.""" 

245 results = [] 

246 batch_lists = [list(batch) for batch in batches] 

247 if not batch_lists: 

248 return results 

249 

250 if self.supports_packed_pipeline(batch_lists): 

251 return self._run_packed_pipeline(batch_lists, step_context, step_batch) 

252 

253 try: 

254 self.prefetch(batch_lists[0]) # prefetch 0 

255 for index, batch_list in enumerate(batch_lists): 

256 self.wait_prefetch(batch_list) # wait_prefetch n 

257 

258 previous_index = index - 1 

259 if previous_index >= 0: 

260 self.wait_offload(batch_lists[previous_index]) # wait_offload n-1 

261 

262 next_index = index + 1 

263 if next_index < len(batch_lists): 

264 self.prefetch(batch_lists[next_index]) # prefetch n+1 

265 

266 results.append(step_batch(batch_list, step_context)) # update n 

267 self.refresh_swappable_slots(batch_list) 

268 self.offload(batch_list) # offload n 

269 

270 self.wait_offload(batch_lists[-1]) 

271 finally: 

272 self._drain_pending_transfers(batch_lists) 

273 return results 

274 

275 def _drain_pending_transfers(self, batch_lists: Sequence[Sequence[UpdateUnit]]) -> None: 

276 """Settle slots left in an intermediate ``h2d``/``d2h`` state. 

277 

278 Called from ``run_pipeline``'s ``finally`` so that an exception raised 

279 mid-pipeline (for example in ``step_batch``) cannot leave prefetched or 

280 offloaded slots holding device storage and a pending stream event. Slots 

281 that already reached a terminal state (``device`` or ``host``) are left 

282 untouched. 

283 """ 

284 for batch_list in batch_lists: 

285 self.wait_prefetch(batch_list) 

286 self.wait_offload(batch_list) 

287 

288 def _run_packed_pipeline( 

289 self, 

290 batches: Sequence[List[UpdateUnit]], 

291 step_context: Any, 

292 step_batch: Callable[[List[UpdateUnit], Any], Any], 

293 ) -> List[Any]: 

294 """Run updates with two reusable staging buffers. 

295 

296 Buffer parity is fixed by batch index. D2H batch ``n`` and H2D batch 

297 ``n + 2`` share one copy-stream chain while the other arena updates. 

298 """ 

299 results = [] 

300 try: 

301 self.begin_packed_step(batches) 

302 self.enqueue_packed_prefetch(0, 0) 

303 if len(batches) > 1: 

304 self.enqueue_packed_prefetch(1, 1) 

305 for batch_index, batch in enumerate(batches): 

306 staging_index = batch_index % 2 

307 self.wait_packed_prefetch(batch_index, staging_index) 

308 completed_index = batch_index - 2 

309 if completed_index >= 0: 

310 self.wait_packed_offload(completed_index) 

311 self.finish_packed_offload(completed_index) 

312 self.activate_packed_batch(batch_index, staging_index) 

313 results.append(step_batch(batch, step_context)) 

314 self.refresh_swappable_slots(batch) 

315 next_index = batch_index + 2 

316 self.enqueue_packed_offload_prefetch( 

317 batch_index, 

318 next_index if next_index < len(batches) else None, 

319 staging_index, 

320 ) 

321 drain_start = max(0, len(batches) - 2) 

322 for batch_index in range(drain_start, len(batches)): 

323 self.wait_packed_offload(batch_index) 

324 self.finish_packed_offload(batch_index) 

325 finally: 

326 self.release_packed_step_results(results) 

327 self.end_packed_step() 

328 return results 

329 

330 # Both hooks below are dispatched through ``self`` from the packed pipeline 

331 # above and are overridden per backend, so they stay instance 

332 # methods even though the base bodies only discard their arguments. 

333 def release_packed_step_results(self, results: List[Any]) -> None: 

334 """Release backend-specific update outputs before staging teardown.""" 

335 del results 

336 

337 def refresh_swappable_slots(self, batch: Sequence[UpdateUnit]) -> None: 

338 """Refresh slots that become swappable after the optimizer update.""" 

339 del batch 

340 

341 def synchronize_cpu_mirrors(self, slots: Iterable[SwapSlot]) -> None: 

342 """Ensure CPU mirrors contain latest data for checkpointing.""" 

343 slot_list = [slot for slot in _iter_unique_slot_objects(slots) if slot.swappable] 

344 if not slot_list: 

345 return 

346 

347 compute_stream = self.current_stream() 

348 with self.stream_context(compute_stream): 

349 for event in _iter_unique_events(slot_list): 

350 self.wait_event(event, compute_stream) 

351 

352 for slot in slot_list: 

353 if slot.state == "host": 

354 if slot.cpu_tensor is None: 

355 raise RuntimeError(f"Swap slot {slot.name!r} is host-resident but has no CPU mirror.") 

356 continue 

357 if slot.state != "d2h": 

358 self.copy_to_cpu(slot) 

359 self.wait_offload_slot(slot) 

360 

361 checkpoint_event = self.record_event(compute_stream) if compute_stream is not None else None 

362 

363 # Storage release only needs stream ordering above. Host completion is 

364 # required separately because checkpoint_state_dict reads CPU mirrors. 

365 self.wait_event(checkpoint_event, None) 

366 for slot in slot_list: 

367 slot.event = None 

368 self.restore_packed_host_views(slot_list) 

369 

370 def materialize_slots(self, slots: Iterable[SwapSlot]) -> None: 

371 """Materialize host-resident slots on their original device. 

372 

373 HSDP checkpoint broadcasts operate on device tensors. Swap slots are 

374 normally host-resident between optimizer steps, so checkpoint code uses 

375 this helper to reuse the normal H2D stream/event ordering before issuing 

376 a collective. 

377 """ 

378 slot_list = [ 

379 slot for slot in _iter_unique_slot_objects(slots) 

380 if slot.swappable and slot.state == "host" 

381 ] 

382 if not slot_list: 

383 return 

384 for slot in slot_list: 

385 if slot.packed: 

386 device_tensor = torch.empty(slot.shape, dtype=slot.dtype, device=slot.device) 

387 self._bind_packed_slot(slot, device_tensor) 

388 self.prefetch([UpdateUnit(0, None, None, slot_list)]) 

389 self.wait_prefetch([UpdateUnit(0, None, None, slot_list)]) 

390 

391 def restore_packed_host_views(self, slots: Iterable[SwapSlot]) -> None: 

392 """Restore persistent packed host views after temporary materialization.""" 

393 for slot in _iter_unique_slot_objects(slots): 

394 if (slot.packed and slot.state == "host" and slot.cpu_tensor is not None 

395 and slot.tensor is not slot.cpu_tensor): 

396 self._bind_packed_slot(slot, slot.cpu_tensor) 

397 

398 def prefetch(self, batch: Sequence[UpdateUnit]) -> None: 

399 """Prefetch batch slots from CPU to device.""" 

400 slots = [slot for slot in _iter_unique_slots(batch) if slot.swappable and slot.state == "host"] 

401 if not slots: 

402 return 

403 

404 for slot in slots: 

405 if slot.cpu_tensor is None: 

406 raise RuntimeError(f"Swap slot {slot.name!r} is host-resident but has no CPU mirror.") 

407 

408 restored_slots = [] 

409 try: 

410 for slot in slots: 

411 self.restore_device_storage(slot) 

412 restored_slots.append(slot) 

413 except Exception: 

414 for restored_slot in reversed(restored_slots): 

415 self.release_device_storage(restored_slot) 

416 raise 

417 

418 copy_stream = self._get_copy_stream() 

419 compute_event = self._record_current_stream_event() if copy_stream is not None else None 

420 with self.stream_context(copy_stream): 

421 self.wait_event(compute_event, copy_stream) 

422 for slot in slots: 

423 self.copy_to_device(slot) 

424 slot.state = "h2d" 

425 copy_event = self.record_event(copy_stream) if copy_stream is not None else None 

426 for slot in slots: 

427 slot.event = copy_event 

428 

429 def wait_prefetch(self, batch: Sequence[UpdateUnit]) -> None: 

430 """Wait for batch prefetch copies.""" 

431 slots = [slot for slot in _iter_unique_slots(batch) if slot.swappable and slot.state == "h2d"] 

432 if not slots: 

433 return 

434 

435 compute_stream = self.current_stream() 

436 with self.stream_context(compute_stream): 

437 for event in _iter_unique_events(slots): 

438 self.wait_event(event, compute_stream) 

439 for slot in slots: 

440 self.wait_prefetch_slot(slot) 

441 slot.event = None 

442 

443 def offload(self, batch: Sequence[UpdateUnit]) -> None: 

444 """Offload batch slots from device to CPU.""" 

445 slots = [slot for slot in _iter_unique_slots(batch) if slot.swappable and slot.state == "device"] 

446 if not slots: 

447 return 

448 

449 self._enqueue_offload_slots(slots) 

450 

451 def offload_initial_slots(self, slots: Iterable[SwapSlot]) -> None: 

452 """Offload existing device-resident slots before the first optimizer update.""" 

453 slot_list = [ 

454 slot for slot in _iter_unique_slot_objects(slots) 

455 if slot.swappable and slot.state == "device" 

456 ] 

457 if not slot_list: 

458 return 

459 

460 self._enqueue_offload_slots(slot_list) 

461 # Waiting here also releases device storage. Deferring this until the first 

462 # prefetch would leave cold optimizer states resident during forward/backward. 

463 self._wait_offload_slots(slot_list) 

464 

465 def _enqueue_offload_slots(self, slots: Sequence[SwapSlot]) -> None: 

466 """Enqueue D2H copies for device-resident slots.""" 

467 copy_stream = self._get_copy_stream() 

468 compute_event = self._record_current_stream_event() if copy_stream is not None else None 

469 with self.stream_context(copy_stream): 

470 self.wait_event(compute_event, copy_stream) 

471 for slot in slots: 

472 self.copy_to_cpu(slot) 

473 slot.state = "d2h" 

474 copy_event = self.record_event(copy_stream) if copy_stream is not None else None 

475 for slot in slots: 

476 slot.event = copy_event 

477 

478 def wait_offload(self, batch: Sequence[UpdateUnit]) -> None: 

479 """Wait for batch offload copies.""" 

480 slots = [slot for slot in _iter_unique_slots(batch) if slot.swappable and slot.state == "d2h"] 

481 if not slots: 

482 return 

483 

484 self._wait_offload_slots(slots) 

485 

486 def _wait_offload_slots(self, slots: Sequence[SwapSlot]) -> None: 

487 """Wait for D2H copies and release device storage for copied slots.""" 

488 compute_stream = self.current_stream() 

489 with self.stream_context(compute_stream): 

490 for event in _iter_unique_events(slots): 

491 self.wait_event(event, compute_stream) 

492 for slot in slots: 

493 self.wait_offload_slot(slot) 

494 slot.event = None 

495 

496 @staticmethod 

497 def _unit_cost(unit: UpdateUnit) -> int: 

498 return sum(slot.storage_nbytes for slot in unit.slots if slot.swappable) 

499 

500 @staticmethod 

501 def _grow_balanced_segment( 

502 unit_costs: Sequence[int], 

503 start: int, 

504 max_end: int, 

505 should_stop: Callable[[int, int, int], bool], 

506 ) -> Tuple[int, int]: 

507 """Extend a segment from ``start`` while ``should_stop`` stays false. 

508 

509 Returns the exclusive end index and the accumulated cost of ``[start, end)``. 

510 """ 

511 end = start 

512 current_cost = 0 

513 while end < max_end: 

514 next_cost = unit_costs[end] 

515 if should_stop(end, current_cost, next_cost): 

516 break 

517 current_cost += next_cost 

518 end += 1 

519 return end, current_cost 

520 

521 def _get_copy_stream(self) -> Any: 

522 if self._copy_stream is None: 

523 self._copy_stream = self.new_stream() 

524 return self._copy_stream 

525 

526 def _record_current_stream_event(self) -> Any: 

527 current_stream = self.current_stream() 

528 return self.record_event(current_stream) 

529 

530 def populate_slot_metadata(self, slot: SwapSlot, template: Any) -> None: 

531 """Populate stable logical tensor metadata without allocating device state.""" 

532 storage_tensor = self._storage_tensor(template) 

533 slot.shape = tuple(storage_tensor.shape) 

534 slot.dtype = storage_tensor.dtype 

535 slot.device = storage_tensor.device 

536 slot.numel = int(storage_tensor.numel()) 

537 slot.storage_nbytes = slot.numel * int(storage_tensor.element_size()) 

538 

539 def is_packable_template(self, tensor: Any, min_numel: int) -> bool: 

540 """Return whether a state shaped like ``tensor`` can use packed staging.""" 

541 if not self._packed_enabled: 

542 return False 

543 return self.is_swappable_tensor(tensor, min_numel) 

544 

545 @staticmethod 

546 def is_distributed_tensor(tensor: Any) -> bool: 

547 """Return whether ``tensor`` exposes a DTensor local shard.""" 

548 return tensor is not None and callable(getattr(tensor, "to_local", None)) 

549 

550 def prepare_packed_host(self, slots: Sequence[SwapSlot]) -> None: 

551 """Pack logical optimizer states into persistent pinned buffers by dtype.""" 

552 if not self._packed_enabled: 

553 return 

554 

555 packed_slots = [slot for slot in slots if slot.swappable and slot.packed] 

556 signature = self._validate_packed_host_layout(packed_slots) 

557 if signature == self._host_layout_signature: 

558 return 

559 

560 self._prepare_packed_host_sources(packed_slots) 

561 slots_by_dtype: Dict[Any, List[SwapSlot]] = {} 

562 for slot in packed_slots: 

563 slots_by_dtype.setdefault(slot.dtype, []).append(slot) 

564 

565 self._host_buffers = { 

566 dtype: self._pack_dtype_slots(dtype, dtype_slots) 

567 for dtype, dtype_slots in slots_by_dtype.items() 

568 } 

569 self._host_layout_signature = signature 

570 

571 def _validate_packed_host_layout( 

572 self, packed_slots: Sequence[SwapSlot], 

573 ) -> Tuple[Tuple[Any, ...], ...]: 

574 """Validate candidates and return their persistent layout signature.""" 

575 if self._packed_batch_plans or self._packed_tail_event is not None: 

576 raise RuntimeError("Cannot rebuild packed optimizer layout during an active transfer.") 

577 slot_ids = [id(slot) for slot in packed_slots] 

578 if len(slot_ids) != len(set(slot_ids)): 

579 raise RuntimeError("Packed optimizer layout contains a duplicate state slot.") 

580 return tuple( 

581 (id(slot), slot.dtype, slot.shape, slot.numel, slot.device) 

582 for slot in packed_slots 

583 ) 

584 

585 def _prepare_packed_host_sources(self, packed_slots: Sequence[SwapSlot]) -> None: 

586 """Validate source layouts and settle copies before rebuilding host buffers.""" 

587 for slot in packed_slots: 

588 source = slot.checkpoint_tensor 

589 if source is not None: 

590 local = self._storage_tensor(source) 

591 if local.layout != torch.strided or not local.is_contiguous(): 

592 raise ValueError("Packed optimizer states must be dense and contiguous before host packing.") 

593 # A layout rebuild reads CPU mirrors on the host. Stream ordering alone 

594 # is insufficient, including after an initial-state offload. 

595 if self._copy_stream is not None: 

596 self._copy_stream.synchronize() 

597 for event in _iter_unique_events(packed_slots): 

598 self.wait_event(event, None) 

599 if any(slot.state in ("h2d", "d2h") for slot in packed_slots): 

600 self.synchronize_cpu_mirrors(packed_slots) 

601 

602 def invalidate_packed_layout(self) -> None: 

603 """Discard an idle packed layout after state slots have been rebound. 

604 

605 Persistent slot mirrors retain their values until the next host packing. 

606 The instance's copy stream remains available for subsequent steps. 

607 """ 

608 if self._packed_batch_plans or self._packed_tail_event is not None: 

609 raise RuntimeError("Cannot reset optimizer parameters during a packed transfer.") 

610 for arena in self._staging_arenas: 

611 if arena is not None: 

612 arena.dtype_views.clear() 

613 arena.raw_buffer.untyped_storage().resize_(0) 

614 self._staging_arenas = [None, None] 

615 self._packed_device_views.clear() 

616 self._packed_ready_events.clear() 

617 self._packed_offload_events.clear() 

618 self._host_buffers = {} 

619 self._host_layout_signature = () 

620 

621 def _pack_dtype_slots(self, dtype: Any, dtype_slots: Sequence[SwapSlot]) -> Any: 

622 """Pack every slot of one dtype into a single pinned host buffer. 

623 

624 Each slot is re-pointed at its own view inside the new buffer and marked 

625 host-resident, so the packed buffer becomes the slot's CPU mirror from 

626 here on. Any device copy that fed the mirror is released immediately 

627 afterwards, because the host view now owns the only valid values. 

628 """ 

629 total_numel = sum(slot.numel for slot in dtype_slots) 

630 host_buffer = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=True) 

631 host_offset = 0 

632 for slot in dtype_slots: 

633 flat_view = host_buffer.narrow(0, host_offset, slot.numel) 

634 host_view = flat_view.view(slot.shape) 

635 source = slot.tensor if slot.state == "device" else slot.checkpoint_tensor 

636 if source is None: 

637 host_view.zero_() 

638 else: 

639 source_tensor = self._storage_tensor(source) 

640 host_view.copy_(source_tensor.detach().reshape(-1).view(slot.shape), non_blocking=False) 

641 if source_tensor.device.type != "cpu" and source is slot.tensor: 

642 self.release_device_storage(slot) 

643 slot.host_offset = host_offset 

644 slot.cpu_tensor = host_view 

645 slot.bind_tensor(host_view) 

646 slot.state = "host" 

647 slot.event = None 

648 host_offset += slot.numel 

649 return host_buffer 

650 

651 def is_swappable_tensor(self, tensor: Any, min_numel: int) -> bool: 

652 """Return whether ``tensor`` can participate in swap.""" 

653 storage_tensor = self._storage_tensor(tensor) 

654 if not isinstance(storage_tensor, torch.Tensor): 

655 return False 

656 if not storage_tensor.is_floating_point(): 

657 return False 

658 if int(storage_tensor.numel()) < int(min_numel): 

659 return False 

660 if storage_tensor.is_sparse: 

661 return False 

662 if not storage_tensor.is_contiguous(): 

663 return False 

664 if storage_tensor.device.type == "cpu": 

665 return False 

666 try: 

667 storage_size = int(storage_tensor.untyped_storage().size()) 

668 expected_size = int(storage_tensor.numel()) * int(storage_tensor.element_size()) 

669 if storage_size != expected_size: 

670 return False 

671 except RuntimeError: 

672 return False 

673 return True 

674 

675 def storage_nbytes(self, tensor: Any) -> int: 

676 """Return storage bytes for a Torch tensor.""" 

677 storage_tensor = self._storage_tensor(tensor) 

678 if not isinstance(storage_tensor, torch.Tensor): 

679 return 0 

680 try: 

681 return int(storage_tensor.untyped_storage().size()) 

682 except RuntimeError: 

683 return int(storage_tensor.numel()) * int(storage_tensor.element_size()) 

684 

685 def make_cpu_tensor(self, tensor: Any) -> Any: 

686 """Create a CPU mirror tensor.""" 

687 storage_tensor = self._storage_tensor(tensor) 

688 if isinstance(storage_tensor, torch.Tensor): 

689 source = storage_tensor.detach() 

690 try: 

691 cpu_tensor = torch.empty_like(source, device="cpu", pin_memory=True) 

692 except RuntimeError: 

693 cpu_tensor = torch.empty_like(source, device="cpu") 

694 cpu_tensor.copy_(source, non_blocking=True) 

695 return cpu_tensor 

696 raise ValueError(f"Expected torch.Tensor for CPU mirror, got {type(tensor)!r}.") 

697 

698 def make_zero_cpu_tensor_like(self, tensor: Any) -> Any: 

699 """Create a zero-valued CPU mirror without materializing device state.""" 

700 storage_tensor = self._storage_tensor(tensor) 

701 if not isinstance(storage_tensor, torch.Tensor): 

702 raise ValueError(f"Expected torch.Tensor for CPU mirror, got {type(tensor)!r}.") 

703 try: 

704 cpu_tensor = torch.empty_like(storage_tensor, device="cpu", pin_memory=True) 

705 except RuntimeError: 

706 cpu_tensor = torch.empty_like(storage_tensor, device="cpu") 

707 cpu_tensor.zero_() 

708 return cpu_tensor 

709 

710 @staticmethod 

711 def make_device_tensor_like(param: Any, saved_tensor: Any) -> Any: 

712 """Create a live state tensor on the parameter device.""" 

713 if not isinstance(saved_tensor, torch.Tensor): 

714 raise ValueError(f"Expected torch.Tensor in optimizer state, got {type(saved_tensor)!r}.") 

715 return saved_tensor.detach().to(device=param.device, dtype=saved_tensor.dtype).clone() 

716 

717 def make_empty_device_tensor_like(self, param: Any, saved_tensor: Any) -> Any: 

718 """Create an uninitialized live state tensor shell on the parameter device.""" 

719 if not isinstance(saved_tensor, torch.Tensor): 

720 raise ValueError(f"Expected torch.Tensor in optimizer state, got {type(saved_tensor)!r}.") 

721 if self.is_distributed_tensor(param): 

722 return self.make_empty_state_like(param) 

723 return torch.empty_like(saved_tensor, device=param.device, dtype=saved_tensor.dtype) 

724 

725 def make_empty_state_like(self, param: Any) -> Any: 

726 """Allocate a state shell with the parameter's local and logical metadata.""" 

727 local = torch.empty_like(self._storage_tensor(param), memory_format=torch.preserve_format) 

728 # Explicit from_local preserves layout even inside SkipDTensorDispatch, 

729 # where empty_like(param) would otherwise return a plain local tensor. 

730 return _restore_dtensor_metadata(param, local) 

731 

732 def copy_to_device(self, slot: SwapSlot) -> None: 

733 """Copy one CPU mirror to device tensor.""" 

734 if slot.state != "host": 

735 return 

736 if slot.cpu_tensor is None: 

737 return 

738 self._storage_tensor(slot.tensor).copy_(slot.cpu_tensor, non_blocking=True) 

739 slot.state = "h2d" 

740 

741 def wait_prefetch_slot(self, slot: SwapSlot) -> None: 

742 """Torch fallback copies are synchronous on CPU and stream-ordered on device.""" 

743 slot.state = "device" 

744 

745 def copy_to_cpu(self, slot: SwapSlot) -> None: 

746 """Copy one device tensor to CPU mirror.""" 

747 source = self._storage_tensor(slot.tensor) 

748 if slot.cpu_tensor is None: 

749 slot.cpu_tensor = self.make_cpu_tensor(source) 

750 else: 

751 slot.cpu_tensor.copy_(source.detach(), non_blocking=True) 

752 slot.state = "d2h" 

753 

754 def wait_offload_slot(self, slot: SwapSlot) -> None: 

755 """Release device storage after D2H copy completes.""" 

756 if self._storage_tensor(slot.tensor).device.type != "cpu": 

757 self.release_device_storage(slot) 

758 slot.state = "host" 

759 

760 def restore_device_storage(self, slot: SwapSlot) -> None: 

761 """Restore device tensor storage before H2D.""" 

762 storage_tensor = self._storage_tensor(slot.tensor) 

763 if storage_tensor.device.type == "cpu": 

764 return 

765 storage = storage_tensor.untyped_storage() 

766 if storage.size() != slot.storage_nbytes: 

767 storage.resize_(slot.storage_nbytes) 

768 

769 def release_device_storage(self, slot: SwapSlot) -> None: 

770 """Release device storage for a swappable tensor.""" 

771 storage_tensor = self._storage_tensor(slot.tensor) 

772 if storage_tensor.device.type == "cpu": 

773 return 

774 storage = storage_tensor.untyped_storage() 

775 if storage.size() != 0: 

776 storage.resize_(0) 

777 

778 @staticmethod 

779 def device_handle() -> Any: 

780 """Return the Torch device module (``torch.npu``) that owns streams and events.""" 

781 try: 

782 return getattr(torch, _DEVICE_TYPE) 

783 except AttributeError as error: 

784 raise RuntimeError( 

785 f"Swap optimizer expects device handle: 'torch.{_DEVICE_TYPE}' failed." 

786 ) from error 

787 

788 def current_stream(self) -> Any: 

789 """Return the current compute stream.""" 

790 return self.device_handle().current_stream() 

791 

792 def new_stream(self) -> Any: 

793 """Create the copy stream.""" 

794 return self.device_handle().Stream() 

795 

796 def stream_context(self, stream: Any) -> Any: 

797 """Return a stream context for ``stream``.""" 

798 if stream is None: 

799 return contextlib.nullcontext() 

800 return self.device_handle().stream(stream) 

801 

802 def record_event(self, stream: Any = None) -> Any: 

803 """Record an event on ``stream``.""" 

804 event = self.device_handle().Event() 

805 if stream is None: 

806 event.record() 

807 else: 

808 event.record(stream) 

809 return event 

810 

811 @staticmethod 

812 def wait_event(event: Any, stream: Any = None) -> None: 

813 """Make ``stream`` wait for ``event``.""" 

814 if event is None: 

815 return 

816 if stream is None: 

817 event.synchronize() 

818 return 

819 event.wait(stream) 

820 

821 def supports_packed_pipeline(self, batches: Sequence[Sequence[UpdateUnit]]) -> bool: 

822 """Make the final, step-specific decision to use the packed pipeline.""" 

823 if not self._packed_enabled or not batches: 

824 return False 

825 units = itertools.chain.from_iterable(batches) 

826 slots = itertools.chain.from_iterable(unit.slots for unit in units) 

827 swappable_slots = [slot for slot in slots if slot.swappable] 

828 if not swappable_slots: 

829 return False 

830 if len({id(slot) for slot in swappable_slots}) != len(swappable_slots): 

831 raise RuntimeError("Packed optimizer batches contain a duplicate state slot.") 

832 devices = {slot.device for slot in swappable_slots} 

833 eligible = len(devices) == 1 and None not in devices and all( 

834 slot.packed 

835 and slot.cpu_tensor is not None 

836 and slot.dtype in self._host_buffers 

837 for slot in swappable_slots 

838 ) 

839 # A packed slot always carries a host mirror and a host-buffer dtype once 

840 # ``prepare_packed_host`` has run, so the only remaining way for 

841 # ``eligible`` to be false is a device mismatch. Per-tensor swap cannot 

842 # recover from that -- the slot tensor is already a CPU host view -- so 

843 # every packed slot must fail loudly rather than fall through silently. 

844 if not eligible and any(slot.packed for slot in swappable_slots): 

845 raise RuntimeError( 

846 "Packed optimizer states cannot fall back to per-tensor swap after " 

847 f"host packing; packed pipeline eligibility failed for local devices {sorted(map(str, devices))}." 

848 ) 

849 return eligible 

850 

851 @staticmethod 

852 def _storage_tensor(tensor: Any) -> Any: 

853 """Return the local tensor whose storage is managed by this runtime.""" 

854 to_local = getattr(tensor, "to_local", None) 

855 return to_local() if callable(to_local) else tensor 

856 

857 @staticmethod 

858 def _first_swappable_slot(batches: Sequence[Sequence[UpdateUnit]]) -> SwapSlot: 

859 """Return the first swappable slot across ``batches``, in iteration order.""" 

860 units = itertools.chain.from_iterable(batches) 

861 slots = itertools.chain.from_iterable(unit.slots for unit in units) 

862 swappable = (slot for slot in slots if slot.swappable) 

863 return next(swappable) 

864 

865 def begin_packed_step(self, batches: Sequence[Sequence[UpdateUnit]]) -> None: 

866 """Build batch transfer plans and materialize two raw staging buffers.""" 

867 first_slot = self._first_swappable_slot(batches) 

868 # FSDP may leave gradient reductions/reshards queued on auxiliary 

869 # streams. A current-stream event cannot cover those streams, so the 

870 # device-wide synchronization is required before optimizer reads state. 

871 getattr(torch, first_slot.device.type).synchronize(first_slot.device) 

872 self._packed_tail_event = None 

873 self._packed_batch_plans = [self._build_packed_batch_plan(batch) for batch in batches] 

874 dtype_layouts, total_bytes, device = self._packed_layout(self._packed_batch_plans) 

875 

876 for staging_index in range(2): 

877 arena = self._materialize_staging_arena(staging_index, total_bytes, device) 

878 layout_signature = tuple( 

879 (dtype, offset, num_bytes) for dtype, (offset, num_bytes) in dtype_layouts.items() 

880 ) 

881 if arena.layout_signature != layout_signature: 

882 arena.dtype_views = { 

883 dtype: arena.raw_buffer.narrow(0, offset, num_bytes).view(dtype) 

884 for dtype, (offset, num_bytes) in dtype_layouts.items() 

885 } 

886 arena.layout_signature = layout_signature 

887 self._drop_packed_views(staging_index) 

888 self._packed_ready_events = {} 

889 self._packed_offload_events = {} 

890 

891 def enqueue_packed_prefetch(self, batch_index: int, staging_index: int) -> None: 

892 """Enqueue one packed H2D and record its ready event.""" 

893 copy_stream = self._get_copy_stream() 

894 with self.stream_context(copy_stream): 

895 self._copy_packed_to_device(batch_index, staging_index) 

896 ready_event = self.record_event(copy_stream) 

897 self._packed_ready_events[batch_index] = ready_event 

898 self._packed_tail_event = ready_event 

899 

900 def wait_packed_prefetch(self, batch_index: int, staging_index: int) -> None: 

901 """Order the compute stream after the batch's packed transfer chain.""" 

902 del staging_index 

903 ready_event = self._packed_ready_events.get(batch_index) 

904 if ready_event is None: 

905 raise RuntimeError(f"Packed optimizer batch {batch_index} has no ready event.") 

906 self.wait_event(ready_event, self.current_stream()) 

907 

908 def activate_packed_batch(self, batch_index: int, staging_index: int) -> None: 

909 """Bind each swap slot to its slice of one staging arena.""" 

910 batch_plan = self._packed_batch_plans[batch_index] 

911 arena = self._require_staging_arena(staging_index) 

912 for dtype, region in batch_plan.regions.items(): 

913 dtype_view = arena.dtype_views[dtype] 

914 for slot in region.slots: 

915 relative_offset = slot.host_offset - region.host_offset 

916 cache_key = ( 

917 staging_index, 

918 id(arena.raw_buffer), 

919 id(dtype_view), 

920 id(slot), 

921 dtype, 

922 relative_offset, 

923 slot.numel, 

924 slot.shape, 

925 ) 

926 device_view = self._packed_device_views.get(cache_key) 

927 if device_view is None: 

928 device_view = dtype_view.narrow(0, relative_offset, slot.numel).view(slot.shape) 

929 self._packed_device_views[cache_key] = device_view 

930 self._bind_packed_slot(slot, device_view) 

931 slot.state = "device" 

932 slot.event = None 

933 

934 def enqueue_packed_offload_prefetch( 

935 self, 

936 batch_index: int, 

937 next_index: Optional[int], 

938 staging_index: int, 

939 ) -> None: 

940 """Serialize current D2H and next same-parity H2D on the copy stream.""" 

941 copy_stream = self._get_copy_stream() 

942 compute_event = self._record_current_stream_event() 

943 with self.stream_context(copy_stream): 

944 self.wait_event(compute_event, copy_stream) 

945 self._copy_packed_to_host(batch_index, staging_index) 

946 # This arena may be overwritten by the following H2D. Exception 

947 # cleanup must use the host result once its D2H has been submitted. 

948 for region in self._packed_batch_plans[batch_index].regions.values(): 

949 for slot in region.slots: 

950 slot.state = "d2h" 

951 if next_index is not None: 

952 self._copy_packed_to_device(next_index, staging_index) 

953 chain_event = self.record_event(copy_stream) 

954 self._packed_offload_events[batch_index] = chain_event 

955 self._packed_tail_event = chain_event 

956 if next_index is not None: 

957 self._packed_ready_events[next_index] = chain_event 

958 

959 def wait_packed_offload(self, batch_index: int) -> None: 

960 """Order the compute stream after a packed transfer chain.""" 

961 offload_event = self._packed_offload_events.get(batch_index) 

962 if offload_event is None: 

963 raise RuntimeError(f"Packed optimizer batch {batch_index} has no offload event.") 

964 self.wait_event(offload_event, self.current_stream()) 

965 

966 def finish_packed_offload(self, batch_index: int) -> None: 

967 """Make persistent pinned views authoritative after D2H completion.""" 

968 batch_plan = self._packed_batch_plans[batch_index] 

969 for region in batch_plan.regions.values(): 

970 for slot in region.slots: 

971 self._bind_packed_slot(slot, slot.cpu_tensor) 

972 slot.state = "host" 

973 slot.event = None 

974 

975 def end_packed_step(self) -> None: 

976 """Drain the packed copy chain and release step-local staging storage.""" 

977 if self._copy_stream is not None: 

978 # A submission can fail after enqueueing a copy but before recording 

979 # its tail event, even when no batch has activated a device view yet. 

980 self._copy_stream.synchronize() 

981 elif self._packed_tail_event is not None: 

982 self.wait_event(self._packed_tail_event, None) 

983 active_slots: Dict[int, Any] = {} 

984 for batch_plan in self._packed_batch_plans: 

985 for region in batch_plan.regions.values(): 

986 for slot in region.slots: 

987 if slot.state in ("device", "d2h"): 

988 active_slots[id(slot)] = slot 

989 if active_slots: 

990 compute_stream = self.current_stream() 

991 synchronize = getattr(compute_stream, "synchronize", None) 

992 if synchronize is not None: 

993 synchronize() 

994 for slot in active_slots.values(): 

995 if slot.state == "device": 

996 slot.cpu_tensor.copy_(self._storage_tensor(slot.tensor).detach(), non_blocking=False) 

997 self._bind_packed_slot(slot, slot.cpu_tensor) 

998 slot.state = "host" 

999 slot.event = None 

1000 for arena in self._staging_arenas: 

1001 if arena is None: 

1002 continue 

1003 # Drop views before shrinking the raw storage; otherwise a view 

1004 # can keep the device allocation alive after the step ends. 

1005 arena.dtype_views = {} 

1006 arena.layout_signature = None 

1007 storage = arena.raw_buffer.untyped_storage() 

1008 if storage.size() != 0: 

1009 storage.resize_(0) 

1010 self._packed_device_views = {} 

1011 self._packed_batch_plans = [] 

1012 self._packed_ready_events = {} 

1013 self._packed_offload_events = {} 

1014 self._packed_tail_event = None 

1015 

1016 @staticmethod 

1017 def _build_packed_batch_plan(batch: Sequence[UpdateUnit]) -> _PackedBatchPlan: 

1018 """Group a batch's packed slots into contiguous host regions by dtype.""" 

1019 slots_by_dtype: Dict[Any, List[SwapSlot]] = {} 

1020 seen_slots = set() 

1021 for unit in batch: 

1022 for slot in unit.slots: 

1023 if not slot.swappable or not slot.packed: 

1024 continue 

1025 if id(slot) in seen_slots: 

1026 raise RuntimeError("Packed optimizer batch contains a duplicate state slot.") 

1027 slots_by_dtype.setdefault(slot.dtype, []).append(slot) 

1028 seen_slots.add(id(slot)) 

1029 

1030 regions = {} 

1031 for dtype, slots in slots_by_dtype.items(): 

1032 host_offset = slots[0].host_offset 

1033 expected_offset = host_offset 

1034 for slot in slots: 

1035 if slot.host_offset != expected_offset: 

1036 raise RuntimeError( 

1037 f"Packed optimizer batch has a non-contiguous {dtype} host range at slot {slot.name!r}." 

1038 ) 

1039 expected_offset += slot.numel 

1040 regions[dtype] = _PackedBatchRegion(dtype, host_offset, expected_offset - host_offset, slots) 

1041 return _PackedBatchPlan(regions) 

1042 

1043 def _materialize_staging_arena(self, staging_index: int, total_bytes: int, device: Any) -> _StagingArena: 

1044 """Allocate or resize one packed device staging arena for the requested layout.""" 

1045 arena = self._staging_arenas[staging_index] 

1046 if arena is None or arena.raw_buffer.device != device: 

1047 raw_buffer = torch.empty(total_bytes, dtype=torch.uint8, device=device) 

1048 arena = _StagingArena(raw_buffer) 

1049 self._staging_arenas[staging_index] = arena 

1050 self._drop_packed_views(staging_index) 

1051 return arena 

1052 

1053 raw_buffer = arena.raw_buffer 

1054 storage = raw_buffer.untyped_storage() 

1055 if storage.size() < total_bytes: 

1056 storage.resize_(total_bytes) 

1057 arena.dtype_views = {} 

1058 arena.layout_signature = None 

1059 self._drop_packed_views(staging_index) 

1060 raw_buffer.set_(storage, 0, (total_bytes,), (1,)) 

1061 return arena 

1062 

1063 def _drop_packed_views(self, staging_index: int) -> None: 

1064 """Drop cached views for one arena after its storage/layout changes.""" 

1065 self._packed_device_views = { 

1066 key: value for key, value in self._packed_device_views.items() if key[0] != staging_index 

1067 } 

1068 

1069 def _copy_packed_to_device(self, batch_index: int, staging_index: int) -> None: 

1070 batch_plan = self._packed_batch_plans[batch_index] 

1071 arena = self._require_staging_arena(staging_index) 

1072 for dtype, region in batch_plan.regions.items(): 

1073 host_view = self._host_buffers[dtype].narrow(0, region.host_offset, region.numel) 

1074 arena.dtype_views[dtype].narrow(0, 0, region.numel).copy_(host_view, non_blocking=True) 

1075 

1076 def _copy_packed_to_host(self, batch_index: int, staging_index: int) -> None: 

1077 batch_plan = self._packed_batch_plans[batch_index] 

1078 arena = self._require_staging_arena(staging_index) 

1079 for dtype, region in batch_plan.regions.items(): 

1080 host_view = self._host_buffers[dtype].narrow(0, region.host_offset, region.numel) 

1081 host_view.copy_(arena.dtype_views[dtype].narrow(0, 0, region.numel), non_blocking=True) 

1082 

1083 def _require_staging_arena(self, staging_index: int) -> _StagingArena: 

1084 arena = self._staging_arenas[staging_index] 

1085 if arena is None: 

1086 raise RuntimeError(f"Packed optimizer staging arena {staging_index} is not materialized.") 

1087 return arena 

1088 

1089 @staticmethod 

1090 def _align_bytes(num_bytes: int) -> int: 

1091 return ((num_bytes + _PACKED_ALIGNMENT_BYTES - 1) // _PACKED_ALIGNMENT_BYTES) * _PACKED_ALIGNMENT_BYTES 

1092 

1093 def _packed_layout(self, batch_plans: Sequence[_PackedBatchPlan]) -> Tuple[Dict[Any, Tuple[int, int]], int, Any]: 

1094 """Return the dtype -> (offset, bytes) staging layout shared by every batch.""" 

1095 max_numel_by_dtype: Dict[Any, int] = {} 

1096 device = None 

1097 for batch_plan in batch_plans: 

1098 for dtype, region in batch_plan.regions.items(): 

1099 max_numel_by_dtype[dtype] = max(max_numel_by_dtype.get(dtype, 0), region.numel) 

1100 if device is None and region.slots: 

1101 device = region.slots[0].device 

1102 if device is None: 

1103 raise RuntimeError("Packed optimizer pipeline has no device-resident state metadata.") 

1104 

1105 dtype_layouts = {} 

1106 byte_offset = 0 

1107 for dtype in sorted(max_numel_by_dtype, key=str): 

1108 byte_offset = self._align_bytes(byte_offset) 

1109 element_size = int(self._host_buffers[dtype].element_size()) 

1110 num_bytes = max_numel_by_dtype[dtype] * element_size 

1111 dtype_layouts[dtype] = (byte_offset, num_bytes) 

1112 byte_offset += num_bytes 

1113 return dtype_layouts, self._align_bytes(byte_offset), device 

1114 

1115 

1116@dataclass 

1117class _PackedBatchRegion: 

1118 """One dtype-contiguous host range transferred for a pipeline batch.""" 

1119 

1120 dtype: Any 

1121 host_offset: int 

1122 numel: int 

1123 slots: List[SwapSlot] 

1124 

1125 

1126@dataclass 

1127class _PackedBatchPlan: 

1128 """Packed transfer regions for one optimizer pipeline batch.""" 

1129 

1130 regions: Dict[Any, _PackedBatchRegion] = field(default_factory=dict) 

1131 

1132 

1133@dataclass 

1134class _StagingArena: 

1135 """One raw device allocation and its dtype-specific views.""" 

1136 

1137 raw_buffer: Any 

1138 dtype_views: Dict[Any, Any] = field(default_factory=dict) 

1139 layout_signature: Any = None 

1140 

1141 

1142def _iter_unique_slots(units: Sequence[UpdateUnit]) -> Iterable[SwapSlot]: 

1143 """Yield slots once by object identity.""" 

1144 slots = itertools.chain.from_iterable(unit.slots for unit in units) 

1145 return _iter_unique_slot_objects(slots) 

1146 

1147 

1148def _iter_unique_slot_objects(slots: Iterable[SwapSlot]) -> Iterable[SwapSlot]: 

1149 """Yield slots once by tensor object identity.""" 

1150 unique_slots: Dict[int, SwapSlot] = {} 

1151 for slot in slots: 

1152 unique_slots.setdefault(id(slot.tensor), slot) 

1153 return unique_slots.values() 

1154 

1155 

1156def _iter_unique_events(slots: Iterable[SwapSlot]) -> Iterable[Any]: 

1157 """Yield non-empty events once by object identity.""" 

1158 seen = set() 

1159 for slot in slots: 

1160 event = slot.event 

1161 if event is None: 

1162 continue 

1163 key = id(event) 

1164 if key in seen: 

1165 continue 

1166 seen.add(key) 

1167 yield event 

1168 

1169 

1170def _slot_tensor(unit: SwapUpdateUnit, key: str, fallback: Any) -> Any: 

1171 """Return an active swap slot tensor for ``key``, or ``fallback``. 

1172 

1173 A slot is authoritative only while it is swappable, still resident on the 

1174 local device and bound to a live tensor; otherwise the optimizer state entry 

1175 is the source of truth for this step. 

1176 """ 

1177 for slot in unit.slots: 

1178 if slot.name == key and slot.swappable and slot.state == "device" and slot.tensor is not None: 

1179 return slot.tensor 

1180 return fallback 

1181 

1182 

1183def validate_state_keys(state_keys: Optional[Sequence[str]]) -> Optional[tuple[str, ...]]: 

1184 """Validate user-provided optimizer state keys. 

1185 

1186 Only lexical validity is checked here: a key must appear in 

1187 :data:`KNOWN_STATE_KEYS`. Whether the wrapped optimizer family actually 

1188 owns that state is decided later by the family adapter, which fails during 

1189 ``validate()``/wrap time. 

1190 """ 

1191 if state_keys is None: 

1192 return None 

1193 normalized = tuple(state_keys) 

1194 invalid = sorted(set(normalized) - set(KNOWN_STATE_KEYS)) 

1195 if invalid: 

1196 raise ValueError( 

1197 "SwapOptimizerConfig.state_keys only supports optimizer state slots " 

1198 f"{KNOWN_STATE_KEYS}, but got {invalid}." 

1199 ) 

1200 return normalized 

1201 

1202 

1203#: Builds the family adapter for a wrapped optimizer. 

1204AdapterFactory = Callable[[Any, Any, PipelineSwapRuntime], "StateSwapAdapter"] 

1205 

1206 

1207def _build_profiled_step(wrapper: Any) -> Callable[..., Any]: 

1208 """Build a PyTorch step-hook dispatcher without retaining ``wrapper`` strongly.""" 

1209 wrapper_ref = weakref.ref(wrapper) 

1210 

1211 def hooked_leaf_step(optimizer: Any, *args: Any, **kwargs: Any) -> Any: 

1212 """Dispatch one raw optimizer leaf through its swap wrapper.""" 

1213 current = wrapper_ref() 

1214 if current is None: 

1215 raise RuntimeError("Swap optimizer wrapper has been released.") 

1216 if optimizer is not current.optimizer: 

1217 raise ValueError("Optimizer step hooks cannot replace optimizer identity.") 

1218 return current._step_impl(*args, **kwargs) 

1219 

1220 return torch.optim.Optimizer.profile_hook_step(hooked_leaf_step) 

1221 

1222 

1223class StateSwapAdapter: 

1224 """Algorithm-independent optimizer-state swap adapter base. 

1225 

1226 Owns the slot map, the configured/default state-key selection framework, the 

1227 checkpoint strip/export/restore path and packed-state publication. The 

1228 numerical update and the per-parameter unit construction belong to the 

1229 optimizer family subclasses, which implement :meth:`default_state_keys`, 

1230 :meth:`prepare_step`, :meth:`iter_update_units` and :meth:`step_batch`. 

1231 """ 

1232 

1233 @staticmethod 

1234 def default_state_keys() -> Tuple[str, ...]: 

1235 """Return the family's default swap state keys. 

1236 

1237 Subclasses must override this; the base class deliberately assumes 

1238 nothing about an optimizer's state layout, so there is no default. 

1239 

1240 Raises: 

1241 NotImplementedError: Always, unless a subclass overrides it. 

1242 """ 

1243 raise NotImplementedError 

1244 

1245 def __init__(self, optimizer: Any, config: Any, runtime: Any) -> None: 

1246 """Bind a leaf optimizer to its state slots and transfer runtime.""" 

1247 self.optimizer = optimizer 

1248 self.config = config 

1249 self.runtime = runtime 

1250 self._slots: Dict[Tuple[int, str], SwapSlot] = {} 

1251 self._packed_state_bindings: Dict[int, Tuple[Any, str]] = {} 

1252 self._checkpoint_active = False 

1253 self._checkpoint_broadcasted = False 

1254 self._checkpoint_state_bindings: List[Tuple[Any, str, Any]] = [] 

1255 self.runtime.set_packed_slot_bind_hook(self.on_packed_slot_bound) 

1256 

1257 def iter_update_units(self, step_context: Dict[str, Any]) -> List[UpdateUnit]: 

1258 """Return units collected in ``prepare_step``.""" 

1259 return step_context["units"] 

1260 

1261 def partition_costs( 

1262 self, 

1263 units: Sequence[SwapUpdateUnit], 

1264 step_context: Any, 

1265 ) -> Optional[Sequence[int]]: 

1266 """Return optional planning costs; default to the runtime's local bytes.""" 

1267 del units, step_context 

1268 

1269 def prepare_pipeline( 

1270 self, 

1271 batches: Sequence[Sequence[SwapUpdateUnit]], 

1272 step_context: Any, 

1273 ) -> None: 

1274 """Pack local slots in execution order after logical partitioning.""" 

1275 del step_context 

1276 if not self.runtime.packed_enabled: 

1277 return 

1278 self._refresh_packed_state_bindings() 

1279 active_slots = [slot for batch in batches for unit in batch for slot in unit.slots] 

1280 active_ids = {id(slot) for slot in active_slots} 

1281 # Inactive state stays on host at the layout tail. It must not leave 

1282 # holes inside active dtype regions or keep old packed buffers alive. 

1283 inactive_slots = [slot for slot in self._ordered_slots() if id(slot) not in active_ids] 

1284 self.runtime.prepare_packed_host(active_slots + inactive_slots) 

1285 self.publish_packed_state() 

1286 

1287 def _refresh_packed_state_bindings(self) -> None: 

1288 """Index packed slots by their live optimizer-state owner.""" 

1289 bindings = {} 

1290 for group in self.optimizer.param_groups: 

1291 for param in group["params"]: 

1292 for key in self._configured_state_keys(): 

1293 slot = self._slots.get((id(param), key)) 

1294 if slot is not None and slot.packed: 

1295 bindings[id(slot)] = (param, key) 

1296 self._packed_state_bindings = bindings 

1297 

1298 def on_packed_slot_bound(self, slot: SwapSlot) -> None: 

1299 """Publish a packed host or staging view into optimizer state.""" 

1300 binding = self._packed_state_bindings.get(id(slot)) 

1301 if binding is None: 

1302 raise RuntimeError(f"Packed optimizer slot {slot.name!r} has no live state binding.") 

1303 param, key = binding 

1304 state = self.optimizer.state.get(param) 

1305 if state is None: 

1306 raise RuntimeError(f"Packed optimizer slot {slot.name!r} has no optimizer state entry.") 

1307 state[key] = slot.tensor 

1308 

1309 def initial_slots(self) -> Iterable[SwapSlot]: 

1310 """Discover optimizer states materialized before the swap wrapper was created.""" 

1311 slots = [] 

1312 for group in self.optimizer.param_groups: 

1313 for param in group["params"]: 

1314 state = self.optimizer.state.get(param) 

1315 if state: 

1316 slots.extend(self._build_slots(param, state)) 

1317 return tuple(slots) 

1318 

1319 def finish_step(self, step_context: Any) -> Any: 

1320 """Finish one outer optimizer step.""" 

1321 del step_context 

1322 

1323 def _checkpoint_broadcast_state_keys(self) -> Tuple[str, ...]: 

1324 """Return the wrapped optimizer state fields consumed by its broadcast.""" 

1325 get_state_keys = getattr(self.optimizer, "_state_keys_for_checkpoint_broadcast", None) 

1326 if not callable(get_state_keys): 

1327 return () 

1328 return tuple(get_state_keys()) 

1329 

1330 def _make_checkpoint_cpu_tensor(self, param: Any) -> torch.Tensor: 

1331 """Allocate one host receiver matching the optimizer broadcast contract.""" 

1332 local_param = self.runtime._storage_tensor(param) 

1333 try: 

1334 return torch.empty_like(local_param, dtype=torch.float32, device="cpu", pin_memory=True) 

1335 except RuntimeError: 

1336 return torch.empty_like(local_param, dtype=torch.float32, device="cpu") 

1337 

1338 def _publish_checkpoint_cpu_state(self) -> None: 

1339 """Expose CPU mirrors and host receivers to the fused state broadcast. 

1340 

1341 The wrapped broadcast packs these tensors into one bounded device buffer 

1342 per batch. Existing swap slots therefore stay host-resident, while ranks 

1343 without owner state receive directly into temporary CPU tensors. 

1344 """ 

1345 broadcast_info = getattr(self.optimizer, "_param_to_broadcast_info", None) 

1346 state_keys = self._checkpoint_broadcast_state_keys() 

1347 if not broadcast_info or not state_keys: 

1348 return 

1349 

1350 for param in broadcast_info: 

1351 param_state = self.optimizer.state.get(param) 

1352 if param_state is None: 

1353 param_state = {} 

1354 for key in state_keys: 

1355 slot = self._slots.get((id(param), key)) 

1356 replacement = None 

1357 if slot is not None and slot.swappable: 

1358 if slot.state != "host" or slot.cpu_tensor is None: 

1359 raise RuntimeError( 

1360 f"Checkpoint state slot {key!r} must have an authoritative CPU mirror." 

1361 ) 

1362 replacement = slot.cpu_tensor 

1363 elif key not in param_state: 

1364 replacement = self._make_checkpoint_cpu_tensor(param) 

1365 

1366 if replacement is None: 

1367 continue 

1368 if param not in self.optimizer.state: 

1369 self.optimizer.state[param] = param_state 

1370 previous = param_state.get(key, _MISSING_CHECKPOINT_STATE) 

1371 self._checkpoint_state_bindings.append((param, key, previous)) 

1372 param_state[key] = replacement 

1373 

1374 def _restore_checkpoint_state_bindings(self) -> None: 

1375 """Restore live state entries after exporting the CPU checkpoint values.""" 

1376 for param, key, previous in reversed(self._checkpoint_state_bindings): 

1377 param_state = self.optimizer.state.get(param) 

1378 if param_state is None: 

1379 continue 

1380 if previous is _MISSING_CHECKPOINT_STATE: 

1381 param_state.pop(key, None) 

1382 if not param_state: 

1383 self.optimizer.state.pop(param, None) 

1384 else: 

1385 param_state[key] = previous 

1386 self._checkpoint_state_bindings.clear() 

1387 

1388 def prepare_checkpoint(self) -> None: 

1389 """Publish host-resident swap state for an optional HSDP broadcast.""" 

1390 if self._checkpoint_active: 

1391 return 

1392 self._checkpoint_active = True 

1393 self._checkpoint_broadcasted = False 

1394 try: 

1395 self._refresh_packed_state_bindings() 

1396 # A previous optimizer step may still have a completed D2H event or 

1397 # a live device state. Make the host mirror authoritative first; 

1398 # the broadcast streams it through its bounded device buffer. 

1399 self.runtime.synchronize_cpu_mirrors(self.all_slots()) 

1400 self._publish_checkpoint_cpu_state() 

1401 except Exception: 

1402 try: 

1403 self.runtime.synchronize_cpu_mirrors(self.all_slots()) 

1404 except RuntimeError: # pragma: no cover - best effort after a transfer failure 

1405 logger.exception("Failed to restore swap slots after checkpoint preparation failed") 

1406 finally: 

1407 self._restore_checkpoint_state_bindings() 

1408 self._checkpoint_active = False 

1409 self._checkpoint_broadcasted = False 

1410 raise 

1411 

1412 def finish_checkpoint(self) -> None: 

1413 """Return checkpoint-broadcast state to the normal host-resident form.""" 

1414 if not self._checkpoint_active: 

1415 return 

1416 try: 

1417 self.runtime.synchronize_cpu_mirrors(self.all_slots()) 

1418 finally: 

1419 self._restore_checkpoint_state_bindings() 

1420 self._checkpoint_active = False 

1421 self._checkpoint_broadcasted = False 

1422 

1423 def all_slots(self) -> Iterable[SwapSlot]: 

1424 """Iterate known swap slots.""" 

1425 return tuple(self._ordered_slots()) 

1426 

1427 def tracked_slots(self) -> Iterable[SwapSlot]: 

1428 """Return all registered slots, including parameters removed from groups.""" 

1429 return tuple({id(slot): slot for slot in self._slots.values()}.values()) 

1430 

1431 def rebind_parameter_slots(self, new_to_old_param: Dict[Any, Any]) -> None: 

1432 """Rebuild state bindings using an explicit parameter identity mapping. 

1433 

1434 Args: 

1435 new_to_old_param: New-to-old identities; unrelated leaf entries are 

1436 ignored. CPU mirrors must already be synchronized. 

1437 

1438 Raises: 

1439 ValueError: If a state cannot be mapped to the new local shape. 

1440 """ 

1441 params = [param for group in self.optimizer.param_groups for param in group["params"]] 

1442 local_ids = {id(param) for param in self._params_with_local_state(params)} 

1443 rebound_slots = {} 

1444 rebound_state = {} 

1445 for param in params: 

1446 if id(param) not in local_ids: 

1447 continue 

1448 old_param = new_to_old_param.get(param, param) 

1449 state = self.optimizer.state.get(param, self.optimizer.state.get(old_param)) 

1450 if state is None: 

1451 continue 

1452 state = dict(state) 

1453 for key in self._configured_state_keys(): 

1454 if key not in state: 

1455 continue 

1456 slot = self._slots.get((id(param), key)) 

1457 if slot is None: 

1458 slot = self._slots.get((id(old_param), key)) 

1459 slot = self._rebind_state_slot(param, key, state[key], slot, param is not old_param) 

1460 state[key] = slot.tensor 

1461 rebound_slots[(id(param), key)] = slot 

1462 rebound_state[param] = state 

1463 

1464 # Publish only after all sources have been read. A failed conversion must 

1465 # leave the old slot map (and its authoritative CPU mirrors) recoverable. 

1466 for param in list(self.optimizer.state): 

1467 if isinstance(param, torch.Tensor) and param not in rebound_state: 

1468 self.optimizer.state.pop(param) 

1469 self.optimizer.state.update(rebound_state) 

1470 self._slots = rebound_slots 

1471 self._refresh_packed_state_bindings() 

1472 

1473 def _rebind_state_slot( 

1474 self, param: Any, key: str, tensor: Any, slot: Optional[SwapSlot], replaced: bool, 

1475 ) -> SwapSlot: 

1476 """Preserve a compatible binding or recreate it from authoritative values.""" 

1477 local_param = self.runtime._storage_tensor(param) 

1478 if slot is not None and (tensor is slot.tensor or tensor is slot.cpu_tensor): 

1479 if (not replaced and slot.shape == tuple(local_param.shape) 

1480 and slot.dtype == local_param.dtype and slot.device == local_param.device 

1481 and not self.runtime.is_distributed_tensor(param)): 

1482 return slot 

1483 source = slot.checkpoint_tensor 

1484 else: 

1485 # Identity verifies a missing map; it never infers a replacement. 

1486 if any(tensor is tracked.tensor or tensor is tracked.cpu_tensor for tracked in self._slots.values()): 

1487 raise ValueError("Replacing a swap parameter requires an explicit new_to_old_param mapping.") 

1488 source = tensor 

1489 source = self.runtime._storage_tensor(source) 

1490 if source.numel() != local_param.numel(): 

1491 raise ValueError( 

1492 f"Cannot rebind {key!r} with local shape {tuple(source.shape)} " 

1493 f"to {tuple(local_param.shape)}; restore state for the new layout first." 

1494 ) 

1495 cpu_tensor = self._cast_swappable_tensor_to_cpu(param, source).reshape(local_param.shape) 

1496 device_tensor = self.runtime.make_empty_state_like(param) 

1497 rebound = self._make_slot(key, device_tensor) 

1498 if rebound.swappable: 

1499 rebound.cpu_tensor = self.runtime.make_cpu_tensor(cpu_tensor) 

1500 rebound.state = "host" 

1501 self.runtime.release_device_storage(rebound) 

1502 if rebound.packed: 

1503 rebound.bind_tensor(rebound.cpu_tensor) 

1504 else: 

1505 self.runtime._storage_tensor(device_tensor).copy_(cpu_tensor) 

1506 return rebound 

1507 

1508 def checkpoint_state_dict(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: 

1509 """Return optimizer checkpoint state using CPU mirrors for swapped slots.""" 

1510 del args, kwargs 

1511 self._refresh_packed_state_bindings() 

1512 self.runtime.synchronize_cpu_mirrors(self.all_slots()) 

1513 return self.export_swappable_state(self.optimizer.state_dict()) 

1514 

1515 @contextlib.contextmanager 

1516 def checkpoint_load_context(self, state_dict: Dict[str, Any], flatten: bool) -> Iterable[None]: 

1517 """Provide state keys for FQN loading without an initialization step. 

1518 

1519 Args: 

1520 state_dict: Chain checkpoint keyed by model FQN. 

1521 flatten: Whether the checkpoint uses flattened FQN entries. 

1522 """ 

1523 previous_state = self.optimizer.state 

1524 schema = copy.copy(previous_state) 

1525 schema.clear() 

1526 for group in self.optimizer.param_groups: 

1527 for param in group["params"]: 

1528 name = param.model_name 

1529 if flatten: 

1530 prefix = f"state.{name}." 

1531 schema[param] = { 

1532 key[len(prefix):].split(".", 1)[0]: None 

1533 for key in state_dict if key.startswith(prefix) 

1534 } 

1535 else: 

1536 schema[param] = dict(state_dict.get("state", {}).get(name, {})) 

1537 # DCP uses live state keys to unflatten and calls step() if state is 

1538 # empty. Empty owner ranks must never initiate that extra collective. 

1539 self.optimizer.state = schema 

1540 try: 

1541 yield 

1542 finally: 

1543 if self.optimizer.state is schema: 

1544 self.optimizer.state = previous_state 

1545 

1546 def load_checkpoint_state_dict( 

1547 self, 

1548 state_dict: Dict[str, Any], 

1549 *args: Any, 

1550 **kwargs: Any, 

1551 ) -> None: 

1552 """Load optimizer checkpoint state while restoring swap-managed slots.""" 

1553 del args, kwargs 

1554 stripped, removed = self.strip_swappable_state(state_dict) 

1555 self.optimizer.load_state_dict(stripped) 

1556 self.load_swappable_state(state_dict, removed) 

1557 

1558 def _params_with_local_state(self, params: Sequence[Any]) -> Iterable[Any]: 

1559 """Return parameters whose swapped state belongs to this rank. 

1560 

1561 The default restores every checkpoint entry. Distributed optimizer 

1562 adapters may filter the sequence to preserve rank-local state ownership. 

1563 """ 

1564 return params 

1565 

1566 def publish_packed_state(self) -> None: 

1567 """Publish persistent packed CPU mirrors to the wrapped optimizer state.""" 

1568 if not self.runtime.packed_enabled: 

1569 return 

1570 self._refresh_packed_state_bindings() 

1571 for group in self.optimizer.param_groups: 

1572 for param in group["params"]: 

1573 state = self.optimizer.state.get(param) 

1574 for key in self._configured_state_keys(): 

1575 slot = self._slots.get((id(param), key)) 

1576 if slot is None or not slot.packed or slot.cpu_tensor is None: 

1577 continue 

1578 if state is None: 

1579 state = self.optimizer.state[param] 

1580 state[key] = slot.cpu_tensor 

1581 

1582 def export_swappable_state(self, state_dict: Dict[str, Any]) -> Dict[str, Any]: 

1583 """Build a checkpoint-safe Torch optimizer state dict. 

1584 

1585 Torch optimizer state dicts are keyed by saved parameter ids, while the 

1586 adapter tracks live swap slots by the current parameter objects. This 

1587 method walks both orders together and exports each parameter's optimizer 

1588 state with the data source that currently owns the valid tensor values. 

1589 

1590 If an Adam state tensor such as ``exp_avg`` or ``exp_avg_sq`` has been 

1591 offloaded, the live device tensor may only be a placeholder with its 

1592 storage released. In that case, write a cloned CPU mirror into the 

1593 exported state dict so checkpoints contain the real optimizer values. 

1594 Non-swappable state, metadata, and tensors that are still resident on 

1595 device are deep-copied from the original Torch state dict unchanged. 

1596 """ 

1597 exported = { 

1598 key: self._copy_checkpoint_value(value) 

1599 for key, value in state_dict.items() 

1600 if key not in ("state", "param_groups") 

1601 } 

1602 exported["param_groups"] = self._copy_checkpoint_value( 

1603 state_dict.get("param_groups", []) 

1604 ) 

1605 exported["state"] = {} 

1606 saved_groups = exported.get("param_groups", []) 

1607 params_in_order = [] 

1608 for group in self.optimizer.param_groups: 

1609 params_in_order.extend(group["params"]) 

1610 ids_in_order = [] 

1611 for group in saved_groups: 

1612 ids_in_order.extend(group["params"]) 

1613 for param, param_id in zip(params_in_order, ids_in_order): 

1614 saved_state = state_dict.get("state", {}).get(param_id) 

1615 if not saved_state: 

1616 continue 

1617 exported_state = {} 

1618 for key in self._state_keys_for_param(param): 

1619 slot = self._slots.get((id(param), key)) 

1620 if self.has_host_mirror(slot, key, saved_state): 

1621 local_copy = slot.cpu_tensor.detach().clone() 

1622 exported_state[key] = _restore_dtensor_metadata(param, local_copy) 

1623 for key, value in saved_state.items(): 

1624 if key not in exported_state: 

1625 exported_state[key] = self._copy_checkpoint_value(value) 

1626 exported["state"][param_id] = exported_state 

1627 return exported 

1628 

1629 @staticmethod 

1630 def _copy_checkpoint_value(value: Any) -> Any: 

1631 """Clone checkpoint values without dispatching ``deepcopy`` on DTensor. 

1632 

1633 HSDP state broadcasts may leave a distributed tensor in the wrapped 

1634 optimizer state. Hyper DTensor intentionally rejects unregistered 

1635 ``__deepcopy__`` dispatch, so clone its local shard and restore the 

1636 global layout explicitly. Nested containers are copied recursively; 

1637 scalar metadata still uses regular ``copy.deepcopy`` semantics. 

1638 """ 

1639 if isinstance(value, torch.Tensor): 

1640 to_local = getattr(value, "to_local", None) 

1641 local = to_local() if callable(to_local) else value 

1642 if local is not value: 

1643 local_copy = local.detach().cpu().clone() 

1644 return _restore_dtensor_metadata(value, local_copy) 

1645 return copy.deepcopy(value) 

1646 if isinstance(value, dict): 

1647 return { 

1648 key: StateSwapAdapter._copy_checkpoint_value(item) 

1649 for key, item in value.items() 

1650 } 

1651 if isinstance(value, list): 

1652 return [StateSwapAdapter._copy_checkpoint_value(item) for item in value] 

1653 if isinstance(value, tuple): 

1654 return tuple(StateSwapAdapter._copy_checkpoint_value(item) for item in value) 

1655 return copy.deepcopy(value) 

1656 

1657 @staticmethod 

1658 def has_host_mirror(slot: Optional[SwapSlot], key: str, saved_state: Dict[str, Any]) -> bool: 

1659 """Return whether ``slot`` currently owns the authoritative CPU mirror for ``key``. 

1660 

1661 A slot qualifies only when it has been offloaded to host *and* still 

1662 holds a CPU mirror: after :meth:`export_swappable_state`'s device-side 

1663 fast path, the live tensor may be a storage-released placeholder, so the 

1664 CPU mirror is the only source of the real optimizer values. 

1665 """ 

1666 return ( 

1667 slot is not None 

1668 and slot.state == "host" 

1669 and slot.cpu_tensor is not None 

1670 and key in saved_state 

1671 ) 

1672 

1673 def strip_swappable_state(self, state_dict: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[int, Dict[str, Any]]]: 

1674 """Split Adam state tensors out before delegating to Torch loading. 

1675 

1676 PyTorch's ``load_state_dict`` eagerly restores tensors into the 

1677 optimizer state. For swap-managed Adam buffers, that would bypass the 

1678 adapter/runtime bookkeeping and can place large tensors directly on the 

1679 device. This method therefore copies the checkpoint state dict's 

1680 *containers*, removes the Adam buffers that may be swap-managed, and 

1681 returns them in a side table keyed by the checkpoint parameter id. 

1682 

1683 Only the container skeleton is rebuilt: leaf tensors are shared with the 

1684 caller's checkpoint instead of being duplicated. The removed buffers are 

1685 consumed by :meth:`load_swappable_state`, which either takes a reference 

1686 to the checkpoint storage (CPU checkpoints, where ``to(device="cpu")`` is 

1687 a no-op) or copies it once into a host mirror. Deep-copying the whole 

1688 state dict would hold one extra full copy of every optimizer tensor on 

1689 host memory for the duration of the load, which for large models is a 

1690 needless doubling of the checkpoint-loading peak. 

1691 

1692 The stripped state dict is safe to pass to the wrapped optimizer's 

1693 ``load_state_dict`` for ordinary fields such as parameter groups and 

1694 step counters: Torch does not mutate the tensors it reads from it. The 

1695 removed tensors must be handed to ``load_swappable_state`` afterwards so 

1696 they can be restored with the correct CPU mirror/device placeholder 

1697 layout. 

1698 """ 

1699 stripped = self._copy_state_dict_containers(state_dict) 

1700 removed: Dict[int, Dict[str, Any]] = {} 

1701 swappable_keys = self._configured_state_keys() 

1702 for param_id, saved_state in list(stripped.get("state", {}).items()): 

1703 if not isinstance(saved_state, dict): 

1704 continue 

1705 for key in swappable_keys: 

1706 if key in saved_state: 

1707 removed.setdefault(param_id, {})[key] = saved_state.pop(key) 

1708 return stripped, removed 

1709 

1710 @staticmethod 

1711 def _copy_state_dict_containers(state_dict: Dict[str, Any]) -> Dict[str, Any]: 

1712 """Shallow-copy a checkpoint state dict, sharing every leaf tensor. 

1713 

1714 Rebuilds only the nesting Torch's ``load_state_dict`` walks: the top 

1715 level, the ``state`` per-parameter mappings, and the ``param_groups`` 

1716 entries. Do not add tensor copies here: this helper exists to keep the 

1717 checkpoint-loading host peak free of a duplicate optimizer state. 

1718 """ 

1719 state = state_dict.get("state") 

1720 copied_state = ( 

1721 {param_id: dict(saved_state) if isinstance(saved_state, dict) else saved_state 

1722 for param_id, saved_state in state.items()} 

1723 if isinstance(state, dict) 

1724 else state 

1725 ) 

1726 param_groups = state_dict.get("param_groups") 

1727 copied_groups = ( 

1728 [dict(group) if isinstance(group, dict) else group for group in param_groups] 

1729 if isinstance(param_groups, list) 

1730 else param_groups 

1731 ) 

1732 copied = {key: value for key, value in state_dict.items() if key not in ("state", "param_groups")} 

1733 if state is not None: 

1734 copied["state"] = copied_state 

1735 if param_groups is not None: 

1736 copied["param_groups"] = copied_groups 

1737 return copied 

1738 

1739 def load_swappable_state(self, original_state_dict: Dict[str, Any], removed: Dict[int, Dict[str, Any]]) -> None: 

1740 """Restore removed Adam buffers under swap runtime control. 

1741 

1742 This is the second half of checkpoint loading. After Torch has loaded 

1743 the stripped state dict, this method maps checkpoint parameter ids back 

1744 to the current parameter objects by walking saved and current parameter 

1745 groups in order. Each removed Adam buffer is then recreated as an 

1746 optimizer state entry and registered as a ``SwapSlot``. 

1747 

1748 Packed runtimes place checkpoint values directly in persistent pinned 

1749 host views. Legacy runtimes retain an empty device placeholder whose 

1750 storage is restored only during prefetch. Buffers that do not meet the 

1751 runtime's swappability criteria are materialized directly on the 

1752 parameter's device and tracked as normal device-resident slots. 

1753 """ 

1754 saved_groups = original_state_dict.get("param_groups", []) 

1755 self._slots = {} 

1756 # Walk saved and current parameter groups in order to map checkpoint ids 

1757 # back to the live parameter objects. 

1758 saved_ids = [ 

1759 saved_id 

1760 for saved_group, current_group in zip(saved_groups, self.optimizer.param_groups) 

1761 for saved_id in saved_group["params"] 

1762 ] 

1763 groups = self.optimizer.param_groups 

1764 current_params = [param for group in groups for param in group["params"]] 

1765 local_param_ids = { 

1766 id(param) for param in self._params_with_local_state(current_params) 

1767 } 

1768 for saved_id, param in zip(saved_ids, current_params): 

1769 if id(param) not in local_param_ids: 

1770 self.optimizer.state.pop(param, None) 

1771 continue 

1772 for key, saved_tensor in removed.get(saved_id, {}).items(): 

1773 self._restore_swappable_entry(param, saved_id, saved_groups, key, saved_tensor) 

1774 if self.runtime.packed_enabled: 

1775 self.runtime.prepare_packed_host(self._ordered_slots()) 

1776 self.publish_packed_state() 

1777 

1778 def _restore_swappable_entry( 

1779 self, 

1780 param: Any, 

1781 saved_id: int, 

1782 saved_groups: List[Dict[str, Any]], 

1783 key: str, 

1784 saved_tensor: Any, 

1785 ) -> None: 

1786 """Restore one removed checkpoint buffer as a host or device-resident slot.""" 

1787 state = self.optimizer.state[param] 

1788 cpu_tensor = self._cast_swappable_tensor_to_cpu(param, saved_tensor) 

1789 if self.runtime.packed_enabled and self.runtime.is_packable_template(param, self.config.min_numel): 

1790 if self.runtime.is_distributed_tensor(param): 

1791 logical_tensor = self.runtime.make_empty_state_like(param) 

1792 slot = self._make_slot(key, logical_tensor) 

1793 state[key] = logical_tensor 

1794 self.runtime.release_device_storage(slot) 

1795 else: 

1796 slot = self._make_slot(key, None, template=param) 

1797 state[key] = cpu_tensor 

1798 slot.tensor = cpu_tensor 

1799 slot.cpu_tensor = cpu_tensor 

1800 slot.state = "host" 

1801 self._slots[(id(param), key)] = slot 

1802 return 

1803 device_tensor = self.runtime.make_empty_device_tensor_like(param, cpu_tensor) 

1804 slot = self._make_slot(key, device_tensor) 

1805 if slot.swappable: 

1806 state[key] = device_tensor 

1807 slot.cpu_tensor = self.runtime.make_cpu_tensor(cpu_tensor) 

1808 slot.state = "host" 

1809 self._slots[(id(param), key)] = slot 

1810 self.runtime.release_device_storage(slot) 

1811 else: 

1812 device_tensor = self._cast_state_tensor_like_torch( 

1813 param, 

1814 saved_tensor, 

1815 saved_id, 

1816 saved_groups, 

1817 key, 

1818 ) 

1819 state[key] = device_tensor 

1820 self._slots[(id(param), key)] = self._make_slot(key, device_tensor) 

1821 

1822 @staticmethod 

1823 def _cast_state_tensor_like_torch( 

1824 param: Any, 

1825 saved_tensor: Any, 

1826 saved_id: int, 

1827 saved_groups: List[Dict[str, Any]], 

1828 key: str, 

1829 ) -> Any: 

1830 """Cast a loaded state tensor using PyTorch optimizer load semantics.""" 

1831 if not isinstance(saved_tensor, torch.Tensor): 

1832 raise ValueError(f"Expected torch.Tensor in optimizer state, got {type(saved_tensor)!r}.") 

1833 process = getattr(torch.optim.Optimizer, "_process_value_according_to_param_policy", None) 

1834 if process is not None: 

1835 return process(param, saved_tensor, saved_id, saved_groups, key).detach().clone() 

1836 if key == "step": 

1837 return saved_tensor.detach().clone() 

1838 if param.is_floating_point(): 

1839 return saved_tensor.detach().to(dtype=param.dtype, device=param.device).clone() 

1840 return saved_tensor.detach().to(device=param.device).clone() 

1841 

1842 @staticmethod 

1843 def _cast_swappable_tensor_to_cpu(param: Any, saved_tensor: Any) -> Any: 

1844 """Cast swappable state dtype like PyTorch while keeping values on CPU.""" 

1845 if not isinstance(saved_tensor, torch.Tensor): 

1846 raise ValueError(f"Expected torch.Tensor in optimizer state, got {type(saved_tensor)!r}.") 

1847 to_local = getattr(saved_tensor, "to_local", None) 

1848 source = to_local() if callable(to_local) else saved_tensor 

1849 if param.is_floating_point(): 

1850 return source.detach().to(dtype=param.dtype, device="cpu") 

1851 return source.detach().to(device="cpu") 

1852 

1853 def _register_present_slots(self, param: Any, state: Dict[str, Any]) -> None: 

1854 """Register configured state tensors that already exist in an optimizer state mapping.""" 

1855 for key in self._configured_state_keys(): 

1856 tensor = state.get(key) 

1857 if tensor is None or (id(param), key) in self._slots: 

1858 continue 

1859 self._slots[(id(param), key)] = self._make_slot(key, tensor) 

1860 

1861 def _build_slots(self, param: Any, state: Dict[str, Any]) -> List[SwapSlot]: 

1862 """Return swap slots associated with the current parameter state.""" 

1863 slots = [] 

1864 for key in self._state_keys_for_param(param): 

1865 tensor = state.get(key) 

1866 if tensor is None: 

1867 continue 

1868 slot = self._slots.get((id(param), key)) 

1869 if slot is None: 

1870 slot = self._make_slot(key, tensor) 

1871 self._slots[(id(param), key)] = slot 

1872 elif slot.tensor is not tensor and slot.cpu_tensor is not tensor: 

1873 slot = self._make_slot(key, tensor) 

1874 self._slots[(id(param), key)] = slot 

1875 slots.append(slot) 

1876 return slots 

1877 

1878 def _make_slot(self, key: str, tensor: Any, template: Optional[Any] = None) -> SwapSlot: 

1879 """Create a swap slot for a state tensor or a packed-state template.""" 

1880 metadata_tensor = tensor if tensor is not None else template 

1881 if metadata_tensor is None: 

1882 raise ValueError(f"Cannot build swap slot {key!r} without a tensor or template.") 

1883 if tensor is None: 

1884 swappable = self.runtime.is_packable_template(metadata_tensor, self.config.min_numel) 

1885 else: 

1886 swappable = self.runtime.is_swappable_tensor(tensor, self.config.min_numel) 

1887 packed = bool(self.runtime.packed_enabled and swappable) 

1888 slot = SwapSlot( 

1889 name=key, 

1890 tensor=tensor, 

1891 cpu_tensor=None, 

1892 swappable=swappable, 

1893 state="device" if tensor is not None else "pending", 

1894 packed=packed, 

1895 logical_tensor=tensor if packed and self.runtime.is_distributed_tensor(tensor) else None, 

1896 ) 

1897 self.runtime.populate_slot_metadata(slot, metadata_tensor) 

1898 return slot 

1899 

1900 def _ordered_slots(self) -> List[SwapSlot]: 

1901 """Return slots in optimizer parameter and configured state-key order.""" 

1902 slots = [] 

1903 seen_slots = set() 

1904 for group in self.optimizer.param_groups: 

1905 for param in group["params"]: 

1906 for key in self._configured_state_keys(): 

1907 slot = self._slots.get((id(param), key)) 

1908 if slot is not None and id(slot) not in seen_slots: 

1909 slots.append(slot) 

1910 seen_slots.add(id(slot)) 

1911 return slots 

1912 

1913 def _state_keys_for_param(self, param: Any) -> Tuple[str, ...]: 

1914 state = self.optimizer.state[param] 

1915 keys = self._configured_state_keys() 

1916 result = [] 

1917 for key in keys: 

1918 if key in state: 

1919 result.append(key) 

1920 elif self.config.state_keys is not None: 

1921 raise ValueError(f"Requested state key '{key}' is not present for parameter.") 

1922 return tuple(result) 

1923 

1924 def _configured_state_keys(self) -> Tuple[str, ...]: 

1925 """Return optimizer state keys selected for swap by the current config.""" 

1926 return tuple(self.config.state_keys or self.default_state_keys()) 

1927 

1928 

1929class SwapOptimizer(torch.optim.Optimizer): 

1930 """Torch optimizer wrapper around a family-provided swap adapter. 

1931 

1932 The wrapper is algorithm agnostic: it owns the pipeline runtime, exposes the 

1933 wrapped optimizer's ``param_groups``/``state``/``defaults`` and drives one 

1934 step as ``prepare_step`` -> ``partition`` -> ``run_pipeline`` -> 

1935 ``finish_step``. Every step passes through PyTorch's hook dispatcher, 

1936 including steps with no hooks currently registered, so hooks added later 

1937 take effect. Step hooks must not access optimizer state because swapped 

1938 tensors are not materialized for hook execution. The concrete adapter is 

1939 built by the family factory passed as ``adapter_factory`` (see 

1940 ``swap_adam.py``/``swap_muon.py``). 

1941 """ 

1942 

1943 _is_swap_optimizer = True 

1944 

1945 def __init__(self, optimizer: Any, config: Any, adapter_factory: AdapterFactory) -> None: 

1946 """Wrap a leaf with its family adapter and independent transfer runtime.""" 

1947 # Do not call ``torch.optim.Optimizer.__init__``: the wrapped base 

1948 # optimizer already owns param_groups/state/defaults. Inheriting keeps 

1949 # PyTorch LR schedulers and isinstance checks happy while this wrapper 

1950 # delegates all optimizer state to ``self.optimizer``. 

1951 self.optimizer = optimizer 

1952 self.config = config 

1953 self._hooked_leaf_step = _build_profiled_step(self) 

1954 self._step_active = False 

1955 self._reset_active = False 

1956 self._reset_required = False 

1957 self._export_active = False 

1958 self.runtime = PipelineSwapRuntime(config) 

1959 self.adapter = adapter_factory(optimizer, config, self.runtime) 

1960 self.adapter.validate() 

1961 # Torch Adam states are normally lazy, but callers may materialize them 

1962 # before wrapping to avoid first-step initialization in the measured loop. 

1963 initial_slots = tuple(self.adapter.initial_slots()) 

1964 self.runtime.offload_initial_slots(initial_slots) 

1965 self.runtime.prepare_packed_host(initial_slots) 

1966 self.adapter.publish_packed_state() 

1967 

1968 def __getattr__(self, name: str) -> Any: 

1969 """Delegate unknown attributes to the base optimizer.""" 

1970 return getattr(self.optimizer, name) 

1971 

1972 @property 

1973 def param_groups(self) -> List[Dict[str, Any]]: 

1974 """Proxy parameter groups.""" 

1975 return self.optimizer.param_groups 

1976 

1977 @param_groups.setter 

1978 def param_groups(self, value: List[Dict[str, Any]]) -> None: 

1979 """Replace the underlying optimizer's parameter groups.""" 

1980 self.optimizer.param_groups = value 

1981 

1982 @property 

1983 def state(self) -> Dict[Any, Any]: 

1984 """Proxy optimizer state.""" 

1985 return self.optimizer.state 

1986 

1987 @property 

1988 def defaults(self) -> Dict[str, Any]: 

1989 """Proxy optimizer defaults.""" 

1990 return self.optimizer.defaults 

1991 

1992 def add_param_group(self, param_group: Dict[str, Any]) -> None: 

1993 """Proxy param group addition.""" 

1994 self.optimizer.add_param_group(param_group) 

1995 

1996 def zero_grad(self, set_to_none: bool = True) -> None: 

1997 """Proxy gradient clearing.""" 

1998 self.optimizer.zero_grad(set_to_none=set_to_none) 

1999 

2000 def reset_optimizer_parameters( 

2001 self, new_to_old_param: Optional[Dict[torch.nn.Parameter, torch.nn.Parameter]] = None, 

2002 ) -> None: 

2003 """Rebuild optimizer metadata and rebind state at an idle step boundary. 

2004 

2005 Args: 

2006 new_to_old_param: Explicit new-to-old identities after param_groups 

2007 replacement. Omit when parameter identities are unchanged. 

2008 

2009 Raises: 

2010 RuntimeError: If a step, checkpoint or transfer is still active. 

2011 """ 

2012 slots = tuple(self.adapter.tracked_slots()) 

2013 if (self._step_active or self._reset_active or self._export_active or self.adapter._checkpoint_active 

2014 or self.runtime._packed_batch_plans or self.runtime._packed_tail_event is not None 

2015 or any(slot.state in ("h2d", "d2h") for slot in slots)): 

2016 raise RuntimeError("Cannot reset optimizer parameters during a step, checkpoint or transfer.") 

2017 self._reset_active = True 

2018 self._reset_required = True 

2019 try: 

2020 with self._no_grad_context(): 

2021 self.runtime.synchronize_cpu_mirrors(slots) 

2022 reset = getattr(self.optimizer, "reset_optimizer_parameters", None) 

2023 if callable(reset): 

2024 reset() 

2025 self.adapter.rebind_parameter_slots(new_to_old_param or {}) 

2026 self.runtime.invalidate_packed_layout() 

2027 self._reset_required = False 

2028 finally: 

2029 self._reset_active = False 

2030 

2031 def step(self, closure: Optional[Any] = None) -> Any: 

2032 """Run one optimizer step through PyTorch's leaf hook dispatcher. 

2033 

2034 The swap pipeline never runs a closure, so a non-None value is rejected 

2035 before step hooks run, before any swap transfer is enqueued and before 

2036 any optimizer state is rewritten. Omitted and explicit ``None`` closures 

2037 are both forwarded to leaf hooks as ``closure=None``. 

2038 

2039 Args: 

2040 closure: Unsupported. Must be omitted or None. 

2041 

2042 Raises: 

2043 RuntimeError: If called recursively. 

2044 ValueError: If a non-None closure is supplied. 

2045 """ 

2046 if closure is not None: 

2047 raise ValueError("Swap optimizer does not support closure.") 

2048 if self._step_active: 

2049 raise RuntimeError("Recursive SwapOptimizer.step() is not supported.") 

2050 self._check_reset_complete() 

2051 self._step_active = True 

2052 try: 

2053 return self._hooked_leaf_step(self.optimizer, closure=closure) 

2054 finally: 

2055 self._step_active = False 

2056 

2057 def _step_impl(self, *args: Any, **kwargs: Any) -> Any: 

2058 """Execute the swap pipeline between PyTorch pre- and post-hooks.""" 

2059 # ``step()`` already rejected the caller's closure; whatever arrives here 

2060 # is either the forwarded ``closure=None`` or a pre-hook rewrite. 

2061 if len(args) > 1 or set(kwargs) - {"closure"}: 

2062 raise ValueError("Swap optimizer does not support extra step arguments.") 

2063 if args and "closure" in kwargs: 

2064 raise TypeError("closure was passed both positionally and by keyword.") 

2065 closure = args[0] if args else kwargs.get("closure") 

2066 if closure is not None: 

2067 raise ValueError("Swap optimizer does not support closure.") 

2068 

2069 with self._no_grad_context(): 

2070 step_context = self.adapter.prepare_step() 

2071 units = self.adapter.iter_update_units(step_context) 

2072 unit_costs = self.adapter.partition_costs(units, step_context) 

2073 batches = self.runtime.partition(units, unit_costs=unit_costs) 

2074 self.adapter.prepare_pipeline(batches, step_context) 

2075 self.runtime.run_pipeline(batches, step_context, self.adapter.step_batch) 

2076 return self.adapter.finish_step(step_context) 

2077 

2078 def _check_reset_complete(self) -> None: 

2079 """Prevent using a partially rebuilt optimizer after reset failed.""" 

2080 if self._reset_required: 

2081 raise RuntimeError("Optimizer parameter reset has not completed; retry reset before step or checkpoint.") 

2082 

2083 def state_dict(self) -> Dict[str, Any]: 

2084 """Return optimizer state dict using CPU mirrors for swappable tensors.""" 

2085 self._check_reset_complete() 

2086 self._export_active = True 

2087 try: 

2088 return self._export_state_dict() 

2089 finally: 

2090 self._export_active = False 

2091 

2092 def _export_state_dict(self) -> Dict[str, Any]: 

2093 """Reuse an outer chain checkpoint window or open a leaf-owned window.""" 

2094 broadcast = getattr(self.optimizer, "_broadcast_state_fused_for_ckpt", None) 

2095 if not callable(broadcast): 

2096 return self.adapter.checkpoint_state_dict() 

2097 

2098 owns_checkpoint_window = not self.adapter._checkpoint_active 

2099 try: 

2100 if owns_checkpoint_window: 

2101 self._prepare_swap_checkpoint() 

2102 if not self.adapter._checkpoint_broadcasted: 

2103 self._broadcast_state_fused_for_ckpt() 

2104 return self.adapter.checkpoint_state_dict() 

2105 finally: 

2106 if owns_checkpoint_window: 

2107 self.cleanup_synced_state() 

2108 

2109 def _prepare_swap_checkpoint(self) -> None: 

2110 """Publish CPU swap state needed by an HSDP checkpoint broadcast.""" 

2111 self._check_reset_complete() 

2112 broadcast = getattr(self.optimizer, "_broadcast_state_fused_for_ckpt", None) 

2113 if callable(broadcast): 

2114 self.adapter.prepare_checkpoint() 

2115 

2116 @contextlib.contextmanager 

2117 def _swap_checkpoint_load_context(self, state_dict: Dict[str, Any], flatten: bool) -> Iterable[None]: 

2118 """Let chain loading discover state keys without updating any parameters.""" 

2119 previous_active = self._export_active 

2120 self._export_active = True 

2121 try: 

2122 with self.adapter.checkpoint_load_context(state_dict, flatten): 

2123 yield 

2124 finally: 

2125 self._export_active = previous_active 

2126 

2127 def _finish_swap_checkpoint(self) -> None: 

2128 """Return checkpoint-broadcast state to its normal swapped form.""" 

2129 self.adapter.finish_checkpoint() 

2130 

2131 def _broadcast_state_fused_for_ckpt(self) -> None: 

2132 """Broadcast wrapped optimizer state through bounded device buffers.""" 

2133 broadcast = getattr(self.optimizer, "_broadcast_state_fused_for_ckpt", None) 

2134 if not callable(broadcast): 

2135 return 

2136 prepared_here = not self.adapter._checkpoint_active 

2137 try: 

2138 if prepared_here: 

2139 self.adapter.prepare_checkpoint() 

2140 broadcast() 

2141 self.adapter._checkpoint_broadcasted = True 

2142 except Exception: 

2143 if prepared_here: 

2144 self.cleanup_synced_state() 

2145 raise 

2146 

2147 def cleanup_synced_state(self) -> None: 

2148 """Release HSDP checkpoint copies and restore host-resident slots.""" 

2149 try: 

2150 self.adapter.finish_checkpoint() 

2151 finally: 

2152 cleanup = getattr(self.optimizer, "cleanup_synced_state", None) 

2153 if callable(cleanup): 

2154 cleanup() 

2155 

2156 def load_state_dict(self, state_dict: Dict[str, Any]) -> None: 

2157 """Load optimizer state dict while keeping swappable tensors on CPU mirrors.""" 

2158 previous_active = self._export_active 

2159 self._export_active = True 

2160 try: 

2161 self.adapter.load_checkpoint_state_dict(state_dict) 

2162 finally: 

2163 self._export_active = previous_active 

2164 

2165 @contextlib.contextmanager 

2166 def _no_grad_context(self) -> Iterable[None]: 

2167 with torch.no_grad(): 

2168 yield