Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / fully_shard / api.py: 79%

345 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""hybrid shard data parallel interface""" 

16import warnings 

17from collections import namedtuple 

18from typing import Any, List, Mapping, cast, Optional, Union 

19 

20import torch 

21import torch.distributed as dist 

22from torch import Tensor, nn 

23 

24from hyper_parallel.core.fully_shard.utils import ( 

25 MixedPrecisionPolicy, 

26 OffloadPolicy, 

27 SourceShardMetaInfo, 

28 load_into_param, 

29 parameters_dict, 

30 wait_grad_handle, 

31) 

32from hyper_parallel.core.utils.communication import get_device_handle 

33from hyper_parallel.core.fully_shard.hsdp_scheduler import HSDPSchedulerV2 

34from hyper_parallel.core.fully_shard.state_dict_utils import ( 

35 get_model_state_dict as get_model_state_dict_impl, 

36 set_model_state_dict as set_model_state_dict_impl, 

37) 

38from hyper_parallel import DeviceMesh, init_device_mesh 

39from hyper_parallel.core.dtensor.dtensor import DTensor, distribute_tensor 

40from hyper_parallel.core.fully_shard.hsdp_utils import ( 

41 get_managed_modules_parameters, 

42) 

43 

44origin_class_to_extend_class = {} 

45 

46 

47def _resolve_comm_fusion_zero_copy_default( 

48 comm_fusion: bool, 

49 comm_fusion_zero_copy: Optional[bool], 

50) -> bool: 

51 """Resolve the default for the comm_fusion zero-copy path.""" 

52 if comm_fusion_zero_copy is not None: 

53 return comm_fusion_zero_copy 

54 return comm_fusion 

55 

56 

57def _check_strict_keys( 

58 module: nn.Module, state_dict: Mapping[str, Any], 

59) -> None: 

60 """Raise ``RuntimeError`` if *state_dict* keys do not match *module*.""" 

61 expected_keys = set(module.state_dict().keys()) 

62 missing = expected_keys - set(state_dict.keys()) 

63 unexpected = set(state_dict.keys()) - expected_keys 

64 error_msgs: list[str] = [] 

65 if missing: 

66 error_msgs.append( 

67 "Missing key(s): " + ", ".join(repr(k) for k in sorted(missing)) 

68 ) 

69 if unexpected: 

70 error_msgs.append( 

71 "Unexpected key(s): " + ", ".join(repr(k) for k in sorted(unexpected)) 

72 ) 

73 if error_msgs: 

74 raise RuntimeError( 

75 f"Error(s) in loading state_dict for " 

76 f"{module.__class__.__name__}:\n\t" 

77 + "\n\t".join(error_msgs) 

78 ) 

79 

80 

81def _resolve_local_tensor( 

82 key: str, val: Tensor, target: DTensor, 

83) -> Tensor: 

84 """Return the local shard tensor to be loaded into *target*.""" 

85 if isinstance(val, DTensor): 

86 return val.to_local() 

87 local_shape = tuple(target.local_shape) 

88 global_shape = tuple(target.shape) 

89 val_shape = tuple(val.shape) 

90 if val_shape == local_shape: 

91 return val 

92 if val_shape == global_shape: 

93 wrapped = distribute_tensor( 

94 val, target.device_mesh, 

95 target.layout.alias_placements if target.layout else target.placements, 

96 ) 

97 return wrapped.to_local() 

98 

99 raise ValueError( 

100 f"load '{key}': plain tensor shape {val_shape} " 

101 f"matches neither local shard {local_shape} " 

102 f"nor global {global_shape}." 

103 ) 

104 

105 

106class _UnshardHandle: 

107 """Unshard handle for user call HSDPModule.unshard(async_op=True)""" 

108 def __init__(self, hsdp_state=None): 

109 """ 

110 Initialize an async unshard handle. 

111 

112 Args: 

113 hsdp_state (HSDPState, optional): The state to wait on. None means a no-op handle. 

114 """ 

115 self._hsdp_state = hsdp_state 

116 

117 def wait(self): 

118 """Block until the async unshard operation completes.""" 

119 if self._hsdp_state is not None: 

120 self._hsdp_state.wait_for_unshard() 

121 self._hsdp_state = None 

122 

123 

124class HSDPModule: 

125 """ 

126 The hsdp block of neural networks with hsdp interface. 

127 """ 

128 

129 def __init__(self): 

130 """Initialize HSDPModule.""" 

131 self.hsdp_scheduler = None # Initialized in hsdp_init() 

132 

133 def _apply(self, *args: Any, **kwargs: Any) -> Any: 

134 """Apply a tensor conversion while preserving HSDP parameter storage. 

135 

136 ``nn.Module._apply`` can replace the DTensor local storage while an 

137 unsharded parameter buffer is still materialized on the device. HSDP 

138 must reshard before delegating to the conversion and rebuild its 

139 sharded-parameter bookkeeping afterwards. The bookkeeping reset is a 

140 storage operation and must not become part of an autograd graph. 

141 

142 The module's original class is added after HSDPModule in the runtime 

143 MRO, so static analysis cannot resolve the delegated ``_apply``. 

144 """ 

145 if not hasattr(self, "hsdp_scheduler") or self.hsdp_scheduler is None: 

146 return super()._apply(*args, **kwargs) # pylint: disable=no-member 

147 self.reshard() 

148 result = super()._apply(*args, **kwargs) # pylint: disable=no-member 

149 hsdp_state = self.hsdp_scheduler.hsdp_state 

150 if hsdp_state is None: 

151 return result 

152 with torch.no_grad(): 

153 for hsdp_param in hsdp_state.hsdp_params: 

154 hsdp_param.reset_sharded_param() 

155 if hsdp_state.param_group is not None: 

156 for bucket in hsdp_state.param_group.all_gather_buckets: 

157 # Conversion rebinds shards; an obsolete zero-copy buffer 

158 # must not keep the previous device allocation resident. 

159 if not bucket.is_flat_buffer_valid(): 

160 bucket.flat_param_buffer = None 

161 return result 

162 

163 def hsdp_init(self, module, mesh, reshard_after_forward, 

164 shard_placement_fn, mp_policy, offload_policy, ignored_params, replicate_params, device, 

165 comm_fusion, comm_fusion_zero_copy: Optional[bool] = None, 

166 source_shard_infos: Optional[Mapping[nn.Parameter, SourceShardMetaInfo]] = None): 

167 """init hsdp2 scheduler.""" 

168 resolved_comm_fusion_zero_copy = _resolve_comm_fusion_zero_copy_default( 

169 comm_fusion, 

170 comm_fusion_zero_copy, 

171 ) 

172 

173 self.hsdp_scheduler = HSDPSchedulerV2(module, 

174 mesh, 

175 reshard_after_forward, 

176 shard_placement_fn, 

177 mp_policy, 

178 offload_policy, 

179 ignored_params, 

180 replicate_params, 

181 device, 

182 comm_fusion, 

183 resolved_comm_fusion_zero_copy, 

184 source_shard_infos=source_shard_infos, 

185 ) 

186 

187 def set_requires_gradient_sync(self, requires_grad_sync: bool, *, recurse: bool = True) -> None: 

188 """Set gradient synchronization for this module and optionally its descendants.""" 

189 if not isinstance(requires_grad_sync, bool): 

190 raise ValueError("requires_grad_sync must be bool") 

191 if not isinstance(recurse, bool): 

192 raise ValueError("recurse must be bool") 

193 self_module = cast(nn.Module, self) 

194 for _, module in self_module.named_modules() if recurse else (("", self),): 

195 if isinstance(module, HSDPModule): 

196 module.hsdp_scheduler.set_requires_grad_sync(requires_grad_sync) 

197 

198 def zero_grad(self, set_to_none: bool = True) -> None: 

199 """Clear parameter gradients according to set_to_none.""" 

200 super().zero_grad(set_to_none=set_to_none) 

201 

202 def set_modules_to_forward_prefetch(self, modules): 

203 """set forward prefetch module list to prefetch all gather for unsharded parameters""" 

204 if not isinstance(modules, (tuple, list)): 

205 raise ValueError("modules must be HSDPModule list") 

206 for module in modules: 

207 if not isinstance(module, HSDPModule): 

208 raise ValueError(f"modules must be HSDPModule list but got {type(module)} in list.") 

209 if not hasattr(self, "hsdp_scheduler"): 

210 raise ValueError("call hsdp interface first.") 

211 self.hsdp_scheduler.set_forward_prefetch_cells(modules) 

212 

213 def set_modules_to_backward_prefetch(self, modules): 

214 """set backward prefetch module list to prefetch all gather for unsharded parameters""" 

215 if not isinstance(modules, (tuple, list)): 

216 raise ValueError("modules must be HSDPModule list") 

217 for module in modules: 

218 if not isinstance(module, HSDPModule): 

219 raise ValueError(f"modules must be HSDPModule list but got {type(module)} in list.") 

220 if not hasattr(self, "hsdp_scheduler"): 

221 raise ValueError("call fully_shard interface first.") 

222 self.hsdp_scheduler.set_backward_prefetch_cells(modules) 

223 

224 def reshard(self) -> None: 

225 """reshard all sharded parameters""" 

226 if not self.hsdp_scheduler: 

227 raise ValueError("hsdp_scheduler is None") 

228 hsdp_state = self.hsdp_scheduler.hsdp_state 

229 if hsdp_state: 

230 hsdp_state.shard() 

231 

232 def unshard(self, async_op: bool = False): 

233 """unshard all sharded parameters""" 

234 if not isinstance(async_op, bool): 

235 raise ValueError(f"async_op should be a bool, got {type(async_op)}") 

236 if not self.hsdp_scheduler: 

237 raise ValueError("hsdp_scheduler is None") 

238 hsdp_state = self.hsdp_scheduler.hsdp_state 

239 if hsdp_state: 

240 hsdp_state.unshard(async_op) # pylint: disable=too-many-function-args 

241 if async_op: 

242 return _UnshardHandle(hsdp_state=hsdp_state) 

243 return None 

244 

245 def load_state_dict( 

246 self, 

247 state_dict: Mapping[str, Any], 

248 strict: bool = True, 

249 assign: bool = False, 

250 ): 

251 """ 

252 Load state dict by copying directly into local shards. 

253 

254 Bypasses ``super().load_state_dict()`` because the standard PyTorch 

255 implementation triggers ``copy_`` through the DTensor dispatcher, which 

256 is not registered in the hyper-parallel layout system. 

257 

258 Each value in ``state_dict`` is dispatched by type: 

259 - hyper DTensor: extract local shard and copy directly. 

260 - plain Tensor whose shape == local shard shape: copy as-is. 

261 - plain Tensor whose shape == global shape: distribute via 

262 ``distribute_tensor``, then copy the local shard. 

263 

264 Args: 

265 state_dict (Mapping[str, Any]): Fully-qualified parameter/buffer 

266 names mapped to tensors (DTensor or plain Tensor). 

267 strict (bool): If ``True`` (default), missing or unexpected keys 

268 raise ``RuntimeError``, matching ``nn.Module.load_state_dict`` 

269 semantics. 

270 assign (bool): When ``True`` *and* every value in ``state_dict`` is 

271 already a hyper DTensor, defer to the standard 

272 ``nn.Module.load_state_dict(assign=True)``, which replaces the 

273 module's parameters/buffers with the given DTensors instead of 

274 copying into existing storage. This is required when loading 

275 sharded DTensors onto a meta-device model (e.g. 

276 ``cpu_ram_efficient_loading``). If ``state_dict`` contains any 

277 plain tensor (local-shard or global shape), ``assign`` is 

278 ignored and the copy/distribute path below is used so the 

279 target stays a properly sharded DTensor. 

280 

281 Raises: 

282 RuntimeError: When ``strict`` is ``True`` and keys do not match. 

283 ValueError: When a plain tensor shape matches neither the local 

284 shard shape nor the global shape of the target DTensor. 

285 """ 

286 if assign and state_dict and all( 

287 isinstance(val, DTensor) for val in state_dict.values() 

288 ): 

289 return super().load_state_dict(state_dict, strict=strict, assign=True) 

290 self_module = cast(nn.Module, self) 

291 

292 target_map: dict[str, Tensor] = {} 

293 for name, p in parameters_dict(self_module): 

294 target_map[name] = p 

295 for name, b in self_module.named_buffers(): 

296 target_map[name] = b 

297 

298 if strict: 

299 _check_strict_keys(self_module, state_dict) 

300 

301 with torch.no_grad(): 

302 for key, val in state_dict.items(): 

303 target = target_map.get(key) 

304 if target is None: 

305 continue 

306 

307 if isinstance(target, DTensor): 

308 val = _resolve_local_tensor(key, val, target) 

309 load_into_param(target, val) 

310 

311 # Trigger load_state_dict post-hooks so that HSDP internal 

312 # bookkeeping (e.g. _sharded_param_data) stays in sync. 

313 # Pass an IncompatibleKeys with the same attribute names as PyTorch 

314 # so external hooks can safely read .missing_keys/.unexpected_keys. 

315 _IK = namedtuple("IncompatibleKeys", ["missing_keys", "unexpected_keys"]) 

316 incompatible_keys = _IK([], []) 

317 for _, module in self_module.named_modules(): 

318 hooks = module._load_state_dict_post_hooks # pylint: disable=protected-access 

319 for hook in hooks.values(): 

320 hook(module, incompatible_keys) 

321 

322 def set_is_last_backward(self, is_last_backward: bool): 

323 """set is_last_backward flag""" 

324 self.hsdp_scheduler.scheduler_ctx.is_last_backward = is_last_backward 

325 

326 def set_reduce_comm_interval(self, interval: int = 1) -> None: 

327 """Set the non-fused reduction interval for the entire FSDP module tree. 

328 

329 Call on the root module before its first forward. A unit's reduce-scatter 

330 is waited after the next ``interval`` non-fused units finish backward; 

331 units without gradients still count. Larger intervals retain more 

332 communication buffers. Root backward always drains the remaining work. 

333 

334 Args: 

335 interval: Positive number of subsequent non-fused units to overlap. 

336 

337 Raises: 

338 ValueError: If the interval is invalid, fully_shard has not initialized 

339 the module, or the root has already started its first forward. 

340 

341 Note: 

342 Communication fusion supports only interval=1. Configuring a larger 

343 interval on a fused root warns and falls back to one for the tree. 

344 """ 

345 if isinstance(interval, bool) or not isinstance(interval, int) or interval < 1: 

346 raise ValueError(f"interval must be a positive int, but got {interval!r}.") 

347 if self.hsdp_scheduler is None: 

348 raise ValueError("call fully_shard before setting reduce communication interval") 

349 ctx = self.hsdp_scheduler.scheduler_ctx 

350 if ctx.lazy_init_done: 

351 raise ValueError("set reduce communication interval before the root module's first forward") 

352 if self.hsdp_scheduler.comm_fusion_policy.enable_comm_fusion and interval > 1: 

353 warnings.warn( 

354 "reduce communication intervals greater than one are not supported when " 

355 "comm_fusion=True; using interval=1.", 

356 UserWarning, 

357 stacklevel=2, 

358 ) 

359 interval = 1 

360 ctx.reduce_interval = interval 

361 

362 def reset_iter_state(self, recursive: bool = True) -> None: 

363 """Reset fully_shard iteration bookkeeping without clearing optimizer gradients. 

364 

365 This method must be called only after communication for the iteration has 

366 completed. It does not wait in-flight collectives or clear 

367 ``sharded_param.grad``/``main_grad``. 

368 

369 Args: 

370 recursive: Whether to reset every HSDPModule in this module tree. 

371 

372 Raises: 

373 ValueError: If ``recursive`` is not a bool. 

374 """ 

375 if not isinstance(recursive, bool): 

376 raise ValueError(f"recursive should be a bool, got {type(recursive)}") 

377 self_module = cast(nn.Module, self) 

378 modules = ( 

379 [module for _, module in self_module.named_modules()] 

380 if recursive 

381 else [self_module] 

382 ) 

383 for module in modules: 

384 if isinstance(module, HSDPModule): 

385 module.hsdp_scheduler.reset_iter_state() 

386 

387 def set_requires_all_reduce(self, requires_all_reduce: bool, *, recurse: bool = True) -> None: 

388 """Set all-reduce for this module and optionally its descendants.""" 

389 if not isinstance(requires_all_reduce, bool): 

390 raise ValueError("requires_all_reduce must be bool") 

391 if not isinstance(recurse, bool): 

392 raise ValueError("recurse must be bool") 

393 self_module = cast(nn.Module, self) 

394 for _, module in self_module.named_modules() if recurse else (("", self),): 

395 if isinstance(module, HSDPModule): 

396 module.hsdp_scheduler.set_requires_all_reduce(requires_all_reduce) 

397 

398 def set_reshard_after_forward(self, reshard_after_forward: bool, recurse: bool = True) -> None: 

399 """Set forward resharding for this module and optionally its descendants.""" 

400 if not isinstance(reshard_after_forward, bool): 

401 raise ValueError("reshard_after_forward must be bool") 

402 if not isinstance(recurse, bool): 

403 raise ValueError("recurse must be bool") 

404 self_module = cast(nn.Module, self) 

405 for _, module in self_module.named_modules() if recurse else (("", self),): 

406 if isinstance(module, HSDPModule): 

407 module.hsdp_scheduler.set_reshard_after_forward(reshard_after_forward) 

408 

409 def set_reshard_after_backward(self, reshard_after_backward: bool, recurse: bool = True) -> None: 

410 """Set backward resharding for this module and optionally its descendants.""" 

411 if not isinstance(reshard_after_backward, bool): 

412 raise ValueError("reshard_after_backward must be bool") 

413 if not isinstance(recurse, bool): 

414 raise ValueError("recurse must be bool") 

415 self_module = cast(nn.Module, self) 

416 for _, module in self_module.named_modules() if recurse else (("", self),): 

417 if isinstance(module, HSDPModule): 

418 module.hsdp_scheduler.set_reshard_after_backward(reshard_after_backward) 

419 

420 def set_reduce_op_type(self, reduce_op_type, recurse: bool = True) -> None: 

421 """ 

422 set reduce_op_type for all reduce operations in HSDP 

423 support reduce_op_type "avg" and "sum", default is "avg" 

424 """ 

425 self_module = cast(nn.Module, self) 

426 if recurse: 

427 sub_modules = [m for _, m in self_module.named_modules()] 

428 else: 

429 sub_modules = [self_module] 

430 for module in sub_modules: 

431 if isinstance(module, HSDPModule): 

432 hsdp_state = module.hsdp_scheduler.hsdp_state 

433 if hsdp_state: 

434 hsdp_state.set_reduce_op_type(reduce_op_type) 

435 

436 def set_gradient_scaling_factor(self, factor=None): 

437 """ 

438 Set a multiplicative scaling factor applied to gradients after 

439 reduce-scatter / all-reduce and before they are written into 

440 ``sharded_param.grad``. 

441 

442 ``factor`` may be ``None`` (disable scaling), a Python ``float``/``int``, 

443 or a 0-dim/1-element tensor. Setting ``factor`` to ``None`` (the default 

444 on construction) skips the scaling op entirely so no extra device-side 

445 ``mul_`` is launched on the hot path. 

446 

447 Args: 

448 factor (None | float | int | torch.Tensor): Scaling coefficient. 

449 Use ``None`` to disable scaling. 

450 

451 Raises: 

452 ValueError: If ``factor`` is not one of the supported types or is a 

453 tensor with more than one element. 

454 """ 

455 if factor is not None: 

456 if isinstance(factor, bool): 

457 raise ValueError( 

458 f"gradient_scaling_factor must be None, float, int or a 1-element Tensor, " 

459 f"but got bool {factor}." 

460 ) 

461 if isinstance(factor, Tensor): 

462 if factor.numel() != 1: 

463 raise ValueError( 

464 f"gradient_scaling_factor tensor must have exactly 1 element, " 

465 f"but got shape {tuple(factor.shape)}." 

466 ) 

467 elif not isinstance(factor, (float, int)): 

468 raise ValueError( 

469 f"gradient_scaling_factor must be None, float, int or a 1-element Tensor, " 

470 f"but got {type(factor).__name__}." 

471 ) 

472 hsdp_state = self.hsdp_scheduler.hsdp_state 

473 if hsdp_state: 

474 hsdp_state.set_gradient_scaling_factor(factor) 

475 

476 

477def _extend_module_with_hsdp_interface(module): 

478 """Dynamically extend module's class to inherit from HSDPModule, adding HSDP capabilities.""" 

479 origin_class = module.__class__ 

480 extend_class = origin_class_to_extend_class.get(origin_class, None) 

481 if extend_class is None: 

482 extend_class = type(f"HSDP{origin_class.__name__}", (HSDPModule, origin_class), {}) 

483 origin_class_to_extend_class[origin_class] = extend_class 

484 module.__class__ = extend_class 

485 

486 

487def _get_root_modules(modules: List[nn.Module]) -> List[nn.Module]: 

488 """ 

489 Returns the modules in ``modules`` that are root modules (i.e. parent-less) 

490 with respect to the set ``modules``. In other words, these are the modules 

491 in ``modules`` that are not the child of any other module in ``modules``. 

492 

493 Aligned with PyTorch torch.distributed.utils._get_root_modules. 

494 """ 

495 root_modules: List[nn.Module] = [] 

496 

497 def _get_submodules(mod): 

498 return set(mod.modules()) 

499 

500 module_to_modules: dict[nn.Module, set] = { 

501 m: _get_submodules(m) for m in modules 

502 } 

503 for candidate in modules: 

504 is_root = True 

505 for mod, submodules in module_to_modules.items(): 

506 if candidate is not mod and candidate in submodules: 

507 is_root = False 

508 break 

509 if is_root: 

510 root_modules.append(candidate) 

511 return root_modules 

512 

513 

514def _check_module_valid(module): 

515 """check module valid""" 

516 if not isinstance(module, nn.Module): 

517 raise ValueError(f"module's type must be nn.Module but got {type(module)}.") 

518 

519 

520def _validate_module_for_fully_shard( 

521 module: Union[nn.Module, List[nn.Module]] 

522) -> None: 

523 """Validate module(s) for fully_shard.""" 

524 if isinstance(module, list): 

525 if len(module) == 0: 

526 raise ValueError("fully_shard does not support empty list of modules.") 

527 for i, m in enumerate(module): 

528 try: 

529 _check_module_valid(m) 

530 except ValueError: 

531 raise ValueError( 

532 f"fully_shard expects nn.Module or list[nn.Module], " 

533 f"but got list with {type(m).__name__} at index {i}." 

534 ) from None 

535 else: 

536 _check_module_valid(module) 

537 

538 

539HsdpValidationOptions = namedtuple( 

540 "HsdpValidationOptions", 

541 [ 

542 "shard_size", 

543 "threshold", 

544 "optimizer_level", 

545 "enable_grad_accumulation", 

546 "grad_scale", 

547 "reduce_dtype", 

548 "comm_async", 

549 "comm_fusion", 

550 "bucket_size", 

551 ], 

552) 

553 

554 

555def _validate_hsdp_shard_size(shard_size: int) -> None: 

556 if not isinstance(shard_size, int) or (shard_size <= 0 and shard_size != -1): 

557 raise ValueError(f"shard_size must be a positive integer, but got {shard_size}.") 

558 

559 

560def _validate_hsdp_threshold(threshold: int) -> None: 

561 if not isinstance(threshold, int) or threshold < 0: 

562 raise ValueError(f"threshold must be a positive integer or 0, but got {threshold}.") 

563 

564 

565def _validate_hsdp_optimizer_level(optimizer_level: str) -> None: 

566 if optimizer_level not in ["level1", "level2", "level3"]: 

567 raise ValueError( 

568 f"Optimizer level should in ['level1', 'level2', 'level3'], but got {optimizer_level}." 

569 ) 

570 

571 

572def _validate_hsdp_reduce_dtype(reduce_dtype) -> None: 

573 """Check ``reduce_dtype`` is a torch dtype or ``None``.""" 

574 if reduce_dtype is not None and not isinstance(reduce_dtype, torch.dtype): 

575 raise ValueError(f"reduce_dtype must be torch.dtype but got {reduce_dtype}.") 

576 

577 

578def _check_hsdp_input_valid(module, options: HsdpValidationOptions): 

579 """check hsdp input valid""" 

580 _check_module_valid(module) 

581 _validate_hsdp_shard_size(options.shard_size) 

582 _validate_hsdp_threshold(options.threshold) 

583 _validate_hsdp_optimizer_level(options.optimizer_level) 

584 if not isinstance(options.enable_grad_accumulation, bool): 

585 raise ValueError( 

586 f"enable_grad_accumulation must be bool but got {options.enable_grad_accumulation}." 

587 ) 

588 if not isinstance(options.grad_scale, float): 

589 raise ValueError(f"grad_scale must be float but got {options.grad_scale}.") 

590 _validate_hsdp_reduce_dtype(options.reduce_dtype) 

591 if not isinstance(options.comm_async, bool): 

592 raise ValueError(f"comm_async must be bool but got {options.comm_async}.") 

593 if not isinstance(options.comm_fusion, bool): 

594 raise ValueError(f"comm_fusion must be bool but got {options.comm_fusion}.") 

595 if not isinstance(options.bucket_size, int) or ( 

596 options.bucket_size < 0 and options.bucket_size != -1 

597 ): 

598 raise ValueError( 

599 f"bucket_size must be a positive integer or 0, but got {options.bucket_size}." 

600 ) 

601 

602 

603def _get_device_from_mesh(mesh: DeviceMesh): 

604 """Extract and validate the torch device from the device mesh.""" 

605 device = None 

606 device_type = mesh.device_type 

607 if device_type not in ("npu", "cuda"): 

608 raise AssertionError( 

609 f"hyper_parallel.fully_shard support device in [torch.npu, torch.cuda], " 

610 f"but got '{device_type}'" 

611 ) 

612 device_handle = get_device_handle(device_type) 

613 if device_handle is None: 

614 raise ValueError( 

615 f"hyper_parallel.fully_shard can't find device_handle of " 

616 f"'torch.{device_type}', check the environment." 

617 ) 

618 if device_handle.is_available(): 

619 device = torch.device(device_handle.current_device()) 

620 return device 

621 

622 

623def _normalize_replicate_params( 

624 replicate_params: Optional[set[nn.Parameter]], 

625) -> set[nn.Parameter]: 

626 """ 

627 Normalize replicate_params for fully_shard 

628 Args: 

629 replicate_params (Optional[set[nn.Parameter]]): Set of parameters to exclude from sharding. 

630 Returns: 

631 set[nn.Parameter]: Set of parameters to exclude from sharding. 

632 """ 

633 if replicate_params is None: 

634 return set() 

635 out = set(replicate_params) 

636 for p in out: 

637 if not isinstance(p, (nn.Parameter, DTensor)): 

638 raise TypeError( 

639 "replicate_params must contain only nn.Parameter or DTensor, " 

640 f"got {type(p).__name__}." 

641 ) 

642 return out 

643 

644 

645def _get_modules_parameters(modules, ignored_params=None): 

646 """Collect deduplicated parameters from module roots.""" 

647 return get_managed_modules_parameters(modules, ignored_params) 

648 

649 

650def _validate_managed_params_source_shard_infos( 

651 managed_parameters: set[nn.Parameter], 

652 source_shard_infos: Optional[Mapping[nn.Parameter, SourceShardMetaInfo]], 

653) -> None: 

654 """Validate the parameter-identity metadata consumed by one fully_shard unit.""" 

655 if source_shard_infos is None: 

656 return 

657 if not isinstance(source_shard_infos, Mapping): 

658 raise ValueError("source_shard_infos must be a mapping from Parameter to SourceShardMetaInfo") 

659 

660 managed_parameters = set(managed_parameters) 

661 

662 if any(isinstance(parameter, DTensor) for parameter in managed_parameters): 

663 raise ValueError( 

664 "source_shard_infos cannot be provided when fully_shard manages a native DTensor parameter" 

665 ) 

666 

667 invalid_values = [ 

668 parameter 

669 for parameter, metadata in source_shard_infos.items() 

670 if not isinstance(metadata, SourceShardMetaInfo) 

671 ] 

672 if invalid_values: 

673 raise ValueError("source_shard_infos values must be SourceShardMetaInfo instances") 

674 

675 invalid_origins = [ 

676 parameter 

677 for parameter, metadata in source_shard_infos.items() 

678 if metadata.origin_is_dtensor and not isinstance(parameter, DTensor) 

679 ] 

680 if invalid_origins: 

681 raise ValueError( 

682 "source_shard_infos origin_is_dtensor=True requires a native DTensor parameter" 

683 ) 

684 

685 missing_parameters = managed_parameters.difference(source_shard_infos) 

686 if missing_parameters: 

687 raise ValueError( 

688 "source_shard_infos must cover every parameter managed by this fully_shard call" 

689 ) 

690 unexpected_parameters = set(source_shard_infos).difference(managed_parameters) 

691 if unexpected_parameters: 

692 raise ValueError( 

693 "source_shard_infos contains a parameter not managed by this fully_shard call" 

694 ) 

695 

696 

697def fully_shard( 

698 module: Union[nn.Module, List[nn.Module]], 

699 *, 

700 mesh: Optional[DeviceMesh] = None, 

701 reshard_after_forward: bool = True, 

702 shard_placement_fn: None = None, 

703 mp_policy: MixedPrecisionPolicy = MixedPrecisionPolicy(), 

704 offload_policy: OffloadPolicy = OffloadPolicy(), 

705 ignored_params: Optional[set[nn.Parameter]] = None, 

706 replicate_params: Optional[set[nn.Parameter]] = None, 

707 comm_fusion: bool = False, 

708 comm_fusion_zero_copy: Optional[bool] = None, 

709 source_shard_infos: Optional[Mapping[nn.Parameter, SourceShardMetaInfo]] = None, 

710) -> Union[nn.Module, List[nn.Module]]: 

711 

712 """ 

713 Apply fully_shard to a module (or list of modules) for distributed training with parameter sharding. 

714 

715 This interface provides PyTorch-compatible HSDP (Hybrid Sharded Data Parallelism) 

716 functionality, enabling efficient training of large models by sharding parameters 

717 across multiple devices. The module is automatically enhanced with distributed 

718 capabilities including parameter sharding, gradient synchronization, and memory 

719 management. 

720 

721 When a list of modules is passed, they are treated as one FSDP unit (parameters 

722 grouped together). 

723 

724 Parameters: 

725 module (nn.Module or List[nn.Module]): 

726 The module(s) to apply fully_shard to. Modified in-place. When a list 

727 is passed, parameters from all modules are grouped as one FSDP unit. 

728 

729 mesh (Optional[DeviceMesh], default=None): 

730 The device mesh defining the process topology for distributed training. 

731 If None, fully_shard creates a default 1D mesh for local parameters. 

732 Native DTensor parameters require an explicit DP submesh. 

733 

734 reshard_after_forward (bool, default=True): 

735 Whether to automatically reshard parameters after forward. When True, 

736 parameters are resharded immediately after they are no longer needed, 

737 freeing memory for subsequent operations. Set to False if you want to 

738 keep parameters unsharded for backward pass or manual control. 

739 

740 shard_placement_fn (Callable, default=None): 

741 A callable that determines how to shard each parameter. The function 

742 should accept a parameter and return a Shard object specifying the 

743 sharding dimension, or None to use default sharding (dimension 0) 

744 

745 mp_policy (MixedPrecisionPolicy, default=MixedPrecisionPolicy()): 

746 Mixed precision training policy controlling data type conversions. 

747 

748 offload_policy (OffloadPolicy, default=OffloadPolicy()): 

749 Memory offload policy for reducing device memory usage. 

750 

751 ignored_params (Optional[set[nn.Parameter]], default=None): 

752 Set of parameters to exclude from fully_shard management entirely. 

753 These parameters are left on the original module as regular parameters, 

754 are not sharded, and do not participate in fully_shard gradient 

755 synchronization. Use this for parameters that should remain outside 

756 the fully_shard lifecycle. 

757 

758 replicate_params (Optional[set[nn.Parameter]], default=None): 

759 Set of parameters to keep replicated while still managing them under 

760 fully_shard. These parameters are not sharded, but their gradients 

761 are still synchronized with DDP-style all-reduce over the current 

762 fully_shard communication domain. This differs from ``ignored_params``, 

763 which skips fully_shard management and gradient synchronization 

764 entirely for the selected parameters. 

765 

766 comm_fusion (bool, default=False): 

767 Whether enable all_gather fusion and reduce_scatter fusion. 

768 

769 comm_fusion_zero_copy (Optional[bool], default=None): 

770 Whether allow the experimental zero-copy path for 

771 ``comm_fusion``. When set to ``None``, fully_shard enables it 

772 automatically whenever ``comm_fusion=True``. 

773 When enabled, fully_shard may rebase sharded local parameter storage 

774 into one shared flat buffer so fused all-gather can read directly from 

775 contiguous memory. This path depends on optimizer compatibility with 

776 view-backed parameters. 

777 source_shard_infos (Optional[Mapping[nn.Parameter, SourceShardMetaInfo]]): 

778 Source TP/EP mesh and placements for the plain-parameter dual mode. 

779 

780 Returns: 

781 nn.Module or List[nn.Module]: The input module(s) with HSDP capabilities added. 

782 """ 

783 _validate_module_for_fully_shard(module) 

784 

785 arg_module = module 

786 if isinstance(module, list): 

787 modules = tuple(_get_root_modules(module)) 

788 else: 

789 modules = (module,) 

790 

791 for mod in modules: 

792 _extend_module_with_hsdp_interface(mod) 

793 

794 params = _get_modules_parameters(modules, ignored_params) 

795 has_dtensor_param = any(isinstance(param, DTensor) for param in params) 

796 _validate_managed_params_source_shard_infos(set(params), source_shard_infos) 

797 

798 replicate_params = _normalize_replicate_params(replicate_params) 

799 

800 if mesh is None: 

801 mesh = init_device_mesh(device_type="npu", mesh_shape=(dist.get_world_size(),)) 

802 if has_dtensor_param: 

803 raise ValueError( 

804 "fully_shard does not support mesh=None with a native DTensor parameter; " 

805 "pass an explicit DP submesh instead." 

806 ) 

807 device = _get_device_from_mesh(mesh) 

808 

809 init_modules = modules 

810 modules[0].hsdp_init( 

811 init_modules, 

812 mesh, 

813 reshard_after_forward, 

814 shard_placement_fn, 

815 mp_policy, 

816 offload_policy, 

817 ignored_params, 

818 replicate_params, 

819 device, 

820 comm_fusion, 

821 comm_fusion_zero_copy, 

822 source_shard_infos=source_shard_infos, 

823 ) 

824 # Share the same scheduler handle with other roots so mods[i].unshard()/prefetch work 

825 if len(modules) > 1: 

826 for mod in modules[1:]: 

827 mod.hsdp_scheduler = modules[0].hsdp_scheduler 

828 return arg_module 

829 

830 

831def get_model_state_dict(model: Any, *, options: Any = None) -> dict[str, Any]: 

832 """Get the model state dict of a fully_shard-managed model. 

833 

834 Args: 

835 model: The model whose state dict to retrieve. 

836 options: Optional :class:`StateDictOptions` controlling gathering 

837 (``full_state_dict``), offloading (``cpu_offload``) and frozen 

838 parameter filtering (``ignore_frozen_params``). 

839 

840 Returns: 

841 The model state dict. By default values are sharded DTensors; when 

842 ``full_state_dict=True`` values are full Tensors (CPU on rank 0 only 

843 when ``cpu_offload=True``). 

844 """ 

845 return get_model_state_dict_impl(model, options=options) 

846 

847 

848def set_model_state_dict(model: Any, model_state_dict: dict[str, Any], *, options: Any = None) -> None: 

849 """Load ``model_state_dict`` into a fully_shard-managed model. 

850 

851 Full tensors are scattered into DTensor shards matching the model's layout 

852 before the in-place load. 

853 

854 Args: 

855 model: The model to load state into. 

856 model_state_dict: State dict to load. Values may be plain (global) 

857 tensors when ``full_state_dict=True`` or sharded DTensors otherwise. 

858 options: Optional :class:`StateDictOptions` controlling scattering 

859 (``full_state_dict``), device placement (``cpu_offload``) and 

860 strictness (``strict``). 

861 

862 Returns: 

863 None. 

864 """ 

865 return set_model_state_dict_impl(model, model_state_dict, options=options) 

866 

867 

868def hsdp_sync_stream(): 

869 """Wait for hsdp gradient handle to be completed.""" 

870 wait_grad_handle()