Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / fully_shard / api.py: 79%
345 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""hybrid shard data parallel interface"""
16import warnings
17from collections import namedtuple
18from typing import Any, List, Mapping, cast, Optional, Union
20import torch
21import torch.distributed as dist
22from torch import Tensor, nn
24from hyper_parallel.core.fully_shard.utils import (
25 MixedPrecisionPolicy,
26 OffloadPolicy,
27 SourceShardMetaInfo,
28 load_into_param,
29 parameters_dict,
30 wait_grad_handle,
31)
32from hyper_parallel.core.utils.communication import get_device_handle
33from hyper_parallel.core.fully_shard.hsdp_scheduler import HSDPSchedulerV2
34from hyper_parallel.core.fully_shard.state_dict_utils import (
35 get_model_state_dict as get_model_state_dict_impl,
36 set_model_state_dict as set_model_state_dict_impl,
37)
38from hyper_parallel import DeviceMesh, init_device_mesh
39from hyper_parallel.core.dtensor.dtensor import DTensor, distribute_tensor
40from hyper_parallel.core.fully_shard.hsdp_utils import (
41 get_managed_modules_parameters,
42)
44origin_class_to_extend_class = {}
47def _resolve_comm_fusion_zero_copy_default(
48 comm_fusion: bool,
49 comm_fusion_zero_copy: Optional[bool],
50) -> bool:
51 """Resolve the default for the comm_fusion zero-copy path."""
52 if comm_fusion_zero_copy is not None:
53 return comm_fusion_zero_copy
54 return comm_fusion
57def _check_strict_keys(
58 module: nn.Module, state_dict: Mapping[str, Any],
59) -> None:
60 """Raise ``RuntimeError`` if *state_dict* keys do not match *module*."""
61 expected_keys = set(module.state_dict().keys())
62 missing = expected_keys - set(state_dict.keys())
63 unexpected = set(state_dict.keys()) - expected_keys
64 error_msgs: list[str] = []
65 if missing:
66 error_msgs.append(
67 "Missing key(s): " + ", ".join(repr(k) for k in sorted(missing))
68 )
69 if unexpected:
70 error_msgs.append(
71 "Unexpected key(s): " + ", ".join(repr(k) for k in sorted(unexpected))
72 )
73 if error_msgs:
74 raise RuntimeError(
75 f"Error(s) in loading state_dict for "
76 f"{module.__class__.__name__}:\n\t"
77 + "\n\t".join(error_msgs)
78 )
81def _resolve_local_tensor(
82 key: str, val: Tensor, target: DTensor,
83) -> Tensor:
84 """Return the local shard tensor to be loaded into *target*."""
85 if isinstance(val, DTensor):
86 return val.to_local()
87 local_shape = tuple(target.local_shape)
88 global_shape = tuple(target.shape)
89 val_shape = tuple(val.shape)
90 if val_shape == local_shape:
91 return val
92 if val_shape == global_shape:
93 wrapped = distribute_tensor(
94 val, target.device_mesh,
95 target.layout.alias_placements if target.layout else target.placements,
96 )
97 return wrapped.to_local()
99 raise ValueError(
100 f"load '{key}': plain tensor shape {val_shape} "
101 f"matches neither local shard {local_shape} "
102 f"nor global {global_shape}."
103 )
106class _UnshardHandle:
107 """Unshard handle for user call HSDPModule.unshard(async_op=True)"""
108 def __init__(self, hsdp_state=None):
109 """
110 Initialize an async unshard handle.
112 Args:
113 hsdp_state (HSDPState, optional): The state to wait on. None means a no-op handle.
114 """
115 self._hsdp_state = hsdp_state
117 def wait(self):
118 """Block until the async unshard operation completes."""
119 if self._hsdp_state is not None:
120 self._hsdp_state.wait_for_unshard()
121 self._hsdp_state = None
124class HSDPModule:
125 """
126 The hsdp block of neural networks with hsdp interface.
127 """
129 def __init__(self):
130 """Initialize HSDPModule."""
131 self.hsdp_scheduler = None # Initialized in hsdp_init()
133 def _apply(self, *args: Any, **kwargs: Any) -> Any:
134 """Apply a tensor conversion while preserving HSDP parameter storage.
136 ``nn.Module._apply`` can replace the DTensor local storage while an
137 unsharded parameter buffer is still materialized on the device. HSDP
138 must reshard before delegating to the conversion and rebuild its
139 sharded-parameter bookkeeping afterwards. The bookkeeping reset is a
140 storage operation and must not become part of an autograd graph.
142 The module's original class is added after HSDPModule in the runtime
143 MRO, so static analysis cannot resolve the delegated ``_apply``.
144 """
145 if not hasattr(self, "hsdp_scheduler") or self.hsdp_scheduler is None:
146 return super()._apply(*args, **kwargs) # pylint: disable=no-member
147 self.reshard()
148 result = super()._apply(*args, **kwargs) # pylint: disable=no-member
149 hsdp_state = self.hsdp_scheduler.hsdp_state
150 if hsdp_state is None:
151 return result
152 with torch.no_grad():
153 for hsdp_param in hsdp_state.hsdp_params:
154 hsdp_param.reset_sharded_param()
155 if hsdp_state.param_group is not None:
156 for bucket in hsdp_state.param_group.all_gather_buckets:
157 # Conversion rebinds shards; an obsolete zero-copy buffer
158 # must not keep the previous device allocation resident.
159 if not bucket.is_flat_buffer_valid():
160 bucket.flat_param_buffer = None
161 return result
163 def hsdp_init(self, module, mesh, reshard_after_forward,
164 shard_placement_fn, mp_policy, offload_policy, ignored_params, replicate_params, device,
165 comm_fusion, comm_fusion_zero_copy: Optional[bool] = None,
166 source_shard_infos: Optional[Mapping[nn.Parameter, SourceShardMetaInfo]] = None):
167 """init hsdp2 scheduler."""
168 resolved_comm_fusion_zero_copy = _resolve_comm_fusion_zero_copy_default(
169 comm_fusion,
170 comm_fusion_zero_copy,
171 )
173 self.hsdp_scheduler = HSDPSchedulerV2(module,
174 mesh,
175 reshard_after_forward,
176 shard_placement_fn,
177 mp_policy,
178 offload_policy,
179 ignored_params,
180 replicate_params,
181 device,
182 comm_fusion,
183 resolved_comm_fusion_zero_copy,
184 source_shard_infos=source_shard_infos,
185 )
187 def set_requires_gradient_sync(self, requires_grad_sync: bool, *, recurse: bool = True) -> None:
188 """Set gradient synchronization for this module and optionally its descendants."""
189 if not isinstance(requires_grad_sync, bool):
190 raise ValueError("requires_grad_sync must be bool")
191 if not isinstance(recurse, bool):
192 raise ValueError("recurse must be bool")
193 self_module = cast(nn.Module, self)
194 for _, module in self_module.named_modules() if recurse else (("", self),):
195 if isinstance(module, HSDPModule):
196 module.hsdp_scheduler.set_requires_grad_sync(requires_grad_sync)
198 def zero_grad(self, set_to_none: bool = True) -> None:
199 """Clear parameter gradients according to set_to_none."""
200 super().zero_grad(set_to_none=set_to_none)
202 def set_modules_to_forward_prefetch(self, modules):
203 """set forward prefetch module list to prefetch all gather for unsharded parameters"""
204 if not isinstance(modules, (tuple, list)):
205 raise ValueError("modules must be HSDPModule list")
206 for module in modules:
207 if not isinstance(module, HSDPModule):
208 raise ValueError(f"modules must be HSDPModule list but got {type(module)} in list.")
209 if not hasattr(self, "hsdp_scheduler"):
210 raise ValueError("call hsdp interface first.")
211 self.hsdp_scheduler.set_forward_prefetch_cells(modules)
213 def set_modules_to_backward_prefetch(self, modules):
214 """set backward prefetch module list to prefetch all gather for unsharded parameters"""
215 if not isinstance(modules, (tuple, list)):
216 raise ValueError("modules must be HSDPModule list")
217 for module in modules:
218 if not isinstance(module, HSDPModule):
219 raise ValueError(f"modules must be HSDPModule list but got {type(module)} in list.")
220 if not hasattr(self, "hsdp_scheduler"):
221 raise ValueError("call fully_shard interface first.")
222 self.hsdp_scheduler.set_backward_prefetch_cells(modules)
224 def reshard(self) -> None:
225 """reshard all sharded parameters"""
226 if not self.hsdp_scheduler:
227 raise ValueError("hsdp_scheduler is None")
228 hsdp_state = self.hsdp_scheduler.hsdp_state
229 if hsdp_state:
230 hsdp_state.shard()
232 def unshard(self, async_op: bool = False):
233 """unshard all sharded parameters"""
234 if not isinstance(async_op, bool):
235 raise ValueError(f"async_op should be a bool, got {type(async_op)}")
236 if not self.hsdp_scheduler:
237 raise ValueError("hsdp_scheduler is None")
238 hsdp_state = self.hsdp_scheduler.hsdp_state
239 if hsdp_state:
240 hsdp_state.unshard(async_op) # pylint: disable=too-many-function-args
241 if async_op:
242 return _UnshardHandle(hsdp_state=hsdp_state)
243 return None
245 def load_state_dict(
246 self,
247 state_dict: Mapping[str, Any],
248 strict: bool = True,
249 assign: bool = False,
250 ):
251 """
252 Load state dict by copying directly into local shards.
254 Bypasses ``super().load_state_dict()`` because the standard PyTorch
255 implementation triggers ``copy_`` through the DTensor dispatcher, which
256 is not registered in the hyper-parallel layout system.
258 Each value in ``state_dict`` is dispatched by type:
259 - hyper DTensor: extract local shard and copy directly.
260 - plain Tensor whose shape == local shard shape: copy as-is.
261 - plain Tensor whose shape == global shape: distribute via
262 ``distribute_tensor``, then copy the local shard.
264 Args:
265 state_dict (Mapping[str, Any]): Fully-qualified parameter/buffer
266 names mapped to tensors (DTensor or plain Tensor).
267 strict (bool): If ``True`` (default), missing or unexpected keys
268 raise ``RuntimeError``, matching ``nn.Module.load_state_dict``
269 semantics.
270 assign (bool): When ``True`` *and* every value in ``state_dict`` is
271 already a hyper DTensor, defer to the standard
272 ``nn.Module.load_state_dict(assign=True)``, which replaces the
273 module's parameters/buffers with the given DTensors instead of
274 copying into existing storage. This is required when loading
275 sharded DTensors onto a meta-device model (e.g.
276 ``cpu_ram_efficient_loading``). If ``state_dict`` contains any
277 plain tensor (local-shard or global shape), ``assign`` is
278 ignored and the copy/distribute path below is used so the
279 target stays a properly sharded DTensor.
281 Raises:
282 RuntimeError: When ``strict`` is ``True`` and keys do not match.
283 ValueError: When a plain tensor shape matches neither the local
284 shard shape nor the global shape of the target DTensor.
285 """
286 if assign and state_dict and all(
287 isinstance(val, DTensor) for val in state_dict.values()
288 ):
289 return super().load_state_dict(state_dict, strict=strict, assign=True)
290 self_module = cast(nn.Module, self)
292 target_map: dict[str, Tensor] = {}
293 for name, p in parameters_dict(self_module):
294 target_map[name] = p
295 for name, b in self_module.named_buffers():
296 target_map[name] = b
298 if strict:
299 _check_strict_keys(self_module, state_dict)
301 with torch.no_grad():
302 for key, val in state_dict.items():
303 target = target_map.get(key)
304 if target is None:
305 continue
307 if isinstance(target, DTensor):
308 val = _resolve_local_tensor(key, val, target)
309 load_into_param(target, val)
311 # Trigger load_state_dict post-hooks so that HSDP internal
312 # bookkeeping (e.g. _sharded_param_data) stays in sync.
313 # Pass an IncompatibleKeys with the same attribute names as PyTorch
314 # so external hooks can safely read .missing_keys/.unexpected_keys.
315 _IK = namedtuple("IncompatibleKeys", ["missing_keys", "unexpected_keys"])
316 incompatible_keys = _IK([], [])
317 for _, module in self_module.named_modules():
318 hooks = module._load_state_dict_post_hooks # pylint: disable=protected-access
319 for hook in hooks.values():
320 hook(module, incompatible_keys)
322 def set_is_last_backward(self, is_last_backward: bool):
323 """set is_last_backward flag"""
324 self.hsdp_scheduler.scheduler_ctx.is_last_backward = is_last_backward
326 def set_reduce_comm_interval(self, interval: int = 1) -> None:
327 """Set the non-fused reduction interval for the entire FSDP module tree.
329 Call on the root module before its first forward. A unit's reduce-scatter
330 is waited after the next ``interval`` non-fused units finish backward;
331 units without gradients still count. Larger intervals retain more
332 communication buffers. Root backward always drains the remaining work.
334 Args:
335 interval: Positive number of subsequent non-fused units to overlap.
337 Raises:
338 ValueError: If the interval is invalid, fully_shard has not initialized
339 the module, or the root has already started its first forward.
341 Note:
342 Communication fusion supports only interval=1. Configuring a larger
343 interval on a fused root warns and falls back to one for the tree.
344 """
345 if isinstance(interval, bool) or not isinstance(interval, int) or interval < 1:
346 raise ValueError(f"interval must be a positive int, but got {interval!r}.")
347 if self.hsdp_scheduler is None:
348 raise ValueError("call fully_shard before setting reduce communication interval")
349 ctx = self.hsdp_scheduler.scheduler_ctx
350 if ctx.lazy_init_done:
351 raise ValueError("set reduce communication interval before the root module's first forward")
352 if self.hsdp_scheduler.comm_fusion_policy.enable_comm_fusion and interval > 1:
353 warnings.warn(
354 "reduce communication intervals greater than one are not supported when "
355 "comm_fusion=True; using interval=1.",
356 UserWarning,
357 stacklevel=2,
358 )
359 interval = 1
360 ctx.reduce_interval = interval
362 def reset_iter_state(self, recursive: bool = True) -> None:
363 """Reset fully_shard iteration bookkeeping without clearing optimizer gradients.
365 This method must be called only after communication for the iteration has
366 completed. It does not wait in-flight collectives or clear
367 ``sharded_param.grad``/``main_grad``.
369 Args:
370 recursive: Whether to reset every HSDPModule in this module tree.
372 Raises:
373 ValueError: If ``recursive`` is not a bool.
374 """
375 if not isinstance(recursive, bool):
376 raise ValueError(f"recursive should be a bool, got {type(recursive)}")
377 self_module = cast(nn.Module, self)
378 modules = (
379 [module for _, module in self_module.named_modules()]
380 if recursive
381 else [self_module]
382 )
383 for module in modules:
384 if isinstance(module, HSDPModule):
385 module.hsdp_scheduler.reset_iter_state()
387 def set_requires_all_reduce(self, requires_all_reduce: bool, *, recurse: bool = True) -> None:
388 """Set all-reduce for this module and optionally its descendants."""
389 if not isinstance(requires_all_reduce, bool):
390 raise ValueError("requires_all_reduce must be bool")
391 if not isinstance(recurse, bool):
392 raise ValueError("recurse must be bool")
393 self_module = cast(nn.Module, self)
394 for _, module in self_module.named_modules() if recurse else (("", self),):
395 if isinstance(module, HSDPModule):
396 module.hsdp_scheduler.set_requires_all_reduce(requires_all_reduce)
398 def set_reshard_after_forward(self, reshard_after_forward: bool, recurse: bool = True) -> None:
399 """Set forward resharding for this module and optionally its descendants."""
400 if not isinstance(reshard_after_forward, bool):
401 raise ValueError("reshard_after_forward must be bool")
402 if not isinstance(recurse, bool):
403 raise ValueError("recurse must be bool")
404 self_module = cast(nn.Module, self)
405 for _, module in self_module.named_modules() if recurse else (("", self),):
406 if isinstance(module, HSDPModule):
407 module.hsdp_scheduler.set_reshard_after_forward(reshard_after_forward)
409 def set_reshard_after_backward(self, reshard_after_backward: bool, recurse: bool = True) -> None:
410 """Set backward resharding for this module and optionally its descendants."""
411 if not isinstance(reshard_after_backward, bool):
412 raise ValueError("reshard_after_backward must be bool")
413 if not isinstance(recurse, bool):
414 raise ValueError("recurse must be bool")
415 self_module = cast(nn.Module, self)
416 for _, module in self_module.named_modules() if recurse else (("", self),):
417 if isinstance(module, HSDPModule):
418 module.hsdp_scheduler.set_reshard_after_backward(reshard_after_backward)
420 def set_reduce_op_type(self, reduce_op_type, recurse: bool = True) -> None:
421 """
422 set reduce_op_type for all reduce operations in HSDP
423 support reduce_op_type "avg" and "sum", default is "avg"
424 """
425 self_module = cast(nn.Module, self)
426 if recurse:
427 sub_modules = [m for _, m in self_module.named_modules()]
428 else:
429 sub_modules = [self_module]
430 for module in sub_modules:
431 if isinstance(module, HSDPModule):
432 hsdp_state = module.hsdp_scheduler.hsdp_state
433 if hsdp_state:
434 hsdp_state.set_reduce_op_type(reduce_op_type)
436 def set_gradient_scaling_factor(self, factor=None):
437 """
438 Set a multiplicative scaling factor applied to gradients after
439 reduce-scatter / all-reduce and before they are written into
440 ``sharded_param.grad``.
442 ``factor`` may be ``None`` (disable scaling), a Python ``float``/``int``,
443 or a 0-dim/1-element tensor. Setting ``factor`` to ``None`` (the default
444 on construction) skips the scaling op entirely so no extra device-side
445 ``mul_`` is launched on the hot path.
447 Args:
448 factor (None | float | int | torch.Tensor): Scaling coefficient.
449 Use ``None`` to disable scaling.
451 Raises:
452 ValueError: If ``factor`` is not one of the supported types or is a
453 tensor with more than one element.
454 """
455 if factor is not None:
456 if isinstance(factor, bool):
457 raise ValueError(
458 f"gradient_scaling_factor must be None, float, int or a 1-element Tensor, "
459 f"but got bool {factor}."
460 )
461 if isinstance(factor, Tensor):
462 if factor.numel() != 1:
463 raise ValueError(
464 f"gradient_scaling_factor tensor must have exactly 1 element, "
465 f"but got shape {tuple(factor.shape)}."
466 )
467 elif not isinstance(factor, (float, int)):
468 raise ValueError(
469 f"gradient_scaling_factor must be None, float, int or a 1-element Tensor, "
470 f"but got {type(factor).__name__}."
471 )
472 hsdp_state = self.hsdp_scheduler.hsdp_state
473 if hsdp_state:
474 hsdp_state.set_gradient_scaling_factor(factor)
477def _extend_module_with_hsdp_interface(module):
478 """Dynamically extend module's class to inherit from HSDPModule, adding HSDP capabilities."""
479 origin_class = module.__class__
480 extend_class = origin_class_to_extend_class.get(origin_class, None)
481 if extend_class is None:
482 extend_class = type(f"HSDP{origin_class.__name__}", (HSDPModule, origin_class), {})
483 origin_class_to_extend_class[origin_class] = extend_class
484 module.__class__ = extend_class
487def _get_root_modules(modules: List[nn.Module]) -> List[nn.Module]:
488 """
489 Returns the modules in ``modules`` that are root modules (i.e. parent-less)
490 with respect to the set ``modules``. In other words, these are the modules
491 in ``modules`` that are not the child of any other module in ``modules``.
493 Aligned with PyTorch torch.distributed.utils._get_root_modules.
494 """
495 root_modules: List[nn.Module] = []
497 def _get_submodules(mod):
498 return set(mod.modules())
500 module_to_modules: dict[nn.Module, set] = {
501 m: _get_submodules(m) for m in modules
502 }
503 for candidate in modules:
504 is_root = True
505 for mod, submodules in module_to_modules.items():
506 if candidate is not mod and candidate in submodules:
507 is_root = False
508 break
509 if is_root:
510 root_modules.append(candidate)
511 return root_modules
514def _check_module_valid(module):
515 """check module valid"""
516 if not isinstance(module, nn.Module):
517 raise ValueError(f"module's type must be nn.Module but got {type(module)}.")
520def _validate_module_for_fully_shard(
521 module: Union[nn.Module, List[nn.Module]]
522) -> None:
523 """Validate module(s) for fully_shard."""
524 if isinstance(module, list):
525 if len(module) == 0:
526 raise ValueError("fully_shard does not support empty list of modules.")
527 for i, m in enumerate(module):
528 try:
529 _check_module_valid(m)
530 except ValueError:
531 raise ValueError(
532 f"fully_shard expects nn.Module or list[nn.Module], "
533 f"but got list with {type(m).__name__} at index {i}."
534 ) from None
535 else:
536 _check_module_valid(module)
539HsdpValidationOptions = namedtuple(
540 "HsdpValidationOptions",
541 [
542 "shard_size",
543 "threshold",
544 "optimizer_level",
545 "enable_grad_accumulation",
546 "grad_scale",
547 "reduce_dtype",
548 "comm_async",
549 "comm_fusion",
550 "bucket_size",
551 ],
552)
555def _validate_hsdp_shard_size(shard_size: int) -> None:
556 if not isinstance(shard_size, int) or (shard_size <= 0 and shard_size != -1):
557 raise ValueError(f"shard_size must be a positive integer, but got {shard_size}.")
560def _validate_hsdp_threshold(threshold: int) -> None:
561 if not isinstance(threshold, int) or threshold < 0:
562 raise ValueError(f"threshold must be a positive integer or 0, but got {threshold}.")
565def _validate_hsdp_optimizer_level(optimizer_level: str) -> None:
566 if optimizer_level not in ["level1", "level2", "level3"]:
567 raise ValueError(
568 f"Optimizer level should in ['level1', 'level2', 'level3'], but got {optimizer_level}."
569 )
572def _validate_hsdp_reduce_dtype(reduce_dtype) -> None:
573 """Check ``reduce_dtype`` is a torch dtype or ``None``."""
574 if reduce_dtype is not None and not isinstance(reduce_dtype, torch.dtype):
575 raise ValueError(f"reduce_dtype must be torch.dtype but got {reduce_dtype}.")
578def _check_hsdp_input_valid(module, options: HsdpValidationOptions):
579 """check hsdp input valid"""
580 _check_module_valid(module)
581 _validate_hsdp_shard_size(options.shard_size)
582 _validate_hsdp_threshold(options.threshold)
583 _validate_hsdp_optimizer_level(options.optimizer_level)
584 if not isinstance(options.enable_grad_accumulation, bool):
585 raise ValueError(
586 f"enable_grad_accumulation must be bool but got {options.enable_grad_accumulation}."
587 )
588 if not isinstance(options.grad_scale, float):
589 raise ValueError(f"grad_scale must be float but got {options.grad_scale}.")
590 _validate_hsdp_reduce_dtype(options.reduce_dtype)
591 if not isinstance(options.comm_async, bool):
592 raise ValueError(f"comm_async must be bool but got {options.comm_async}.")
593 if not isinstance(options.comm_fusion, bool):
594 raise ValueError(f"comm_fusion must be bool but got {options.comm_fusion}.")
595 if not isinstance(options.bucket_size, int) or (
596 options.bucket_size < 0 and options.bucket_size != -1
597 ):
598 raise ValueError(
599 f"bucket_size must be a positive integer or 0, but got {options.bucket_size}."
600 )
603def _get_device_from_mesh(mesh: DeviceMesh):
604 """Extract and validate the torch device from the device mesh."""
605 device = None
606 device_type = mesh.device_type
607 if device_type not in ("npu", "cuda"):
608 raise AssertionError(
609 f"hyper_parallel.fully_shard support device in [torch.npu, torch.cuda], "
610 f"but got '{device_type}'"
611 )
612 device_handle = get_device_handle(device_type)
613 if device_handle is None:
614 raise ValueError(
615 f"hyper_parallel.fully_shard can't find device_handle of "
616 f"'torch.{device_type}', check the environment."
617 )
618 if device_handle.is_available():
619 device = torch.device(device_handle.current_device())
620 return device
623def _normalize_replicate_params(
624 replicate_params: Optional[set[nn.Parameter]],
625) -> set[nn.Parameter]:
626 """
627 Normalize replicate_params for fully_shard
628 Args:
629 replicate_params (Optional[set[nn.Parameter]]): Set of parameters to exclude from sharding.
630 Returns:
631 set[nn.Parameter]: Set of parameters to exclude from sharding.
632 """
633 if replicate_params is None:
634 return set()
635 out = set(replicate_params)
636 for p in out:
637 if not isinstance(p, (nn.Parameter, DTensor)):
638 raise TypeError(
639 "replicate_params must contain only nn.Parameter or DTensor, "
640 f"got {type(p).__name__}."
641 )
642 return out
645def _get_modules_parameters(modules, ignored_params=None):
646 """Collect deduplicated parameters from module roots."""
647 return get_managed_modules_parameters(modules, ignored_params)
650def _validate_managed_params_source_shard_infos(
651 managed_parameters: set[nn.Parameter],
652 source_shard_infos: Optional[Mapping[nn.Parameter, SourceShardMetaInfo]],
653) -> None:
654 """Validate the parameter-identity metadata consumed by one fully_shard unit."""
655 if source_shard_infos is None:
656 return
657 if not isinstance(source_shard_infos, Mapping):
658 raise ValueError("source_shard_infos must be a mapping from Parameter to SourceShardMetaInfo")
660 managed_parameters = set(managed_parameters)
662 if any(isinstance(parameter, DTensor) for parameter in managed_parameters):
663 raise ValueError(
664 "source_shard_infos cannot be provided when fully_shard manages a native DTensor parameter"
665 )
667 invalid_values = [
668 parameter
669 for parameter, metadata in source_shard_infos.items()
670 if not isinstance(metadata, SourceShardMetaInfo)
671 ]
672 if invalid_values:
673 raise ValueError("source_shard_infos values must be SourceShardMetaInfo instances")
675 invalid_origins = [
676 parameter
677 for parameter, metadata in source_shard_infos.items()
678 if metadata.origin_is_dtensor and not isinstance(parameter, DTensor)
679 ]
680 if invalid_origins:
681 raise ValueError(
682 "source_shard_infos origin_is_dtensor=True requires a native DTensor parameter"
683 )
685 missing_parameters = managed_parameters.difference(source_shard_infos)
686 if missing_parameters:
687 raise ValueError(
688 "source_shard_infos must cover every parameter managed by this fully_shard call"
689 )
690 unexpected_parameters = set(source_shard_infos).difference(managed_parameters)
691 if unexpected_parameters:
692 raise ValueError(
693 "source_shard_infos contains a parameter not managed by this fully_shard call"
694 )
697def fully_shard(
698 module: Union[nn.Module, List[nn.Module]],
699 *,
700 mesh: Optional[DeviceMesh] = None,
701 reshard_after_forward: bool = True,
702 shard_placement_fn: None = None,
703 mp_policy: MixedPrecisionPolicy = MixedPrecisionPolicy(),
704 offload_policy: OffloadPolicy = OffloadPolicy(),
705 ignored_params: Optional[set[nn.Parameter]] = None,
706 replicate_params: Optional[set[nn.Parameter]] = None,
707 comm_fusion: bool = False,
708 comm_fusion_zero_copy: Optional[bool] = None,
709 source_shard_infos: Optional[Mapping[nn.Parameter, SourceShardMetaInfo]] = None,
710) -> Union[nn.Module, List[nn.Module]]:
712 """
713 Apply fully_shard to a module (or list of modules) for distributed training with parameter sharding.
715 This interface provides PyTorch-compatible HSDP (Hybrid Sharded Data Parallelism)
716 functionality, enabling efficient training of large models by sharding parameters
717 across multiple devices. The module is automatically enhanced with distributed
718 capabilities including parameter sharding, gradient synchronization, and memory
719 management.
721 When a list of modules is passed, they are treated as one FSDP unit (parameters
722 grouped together).
724 Parameters:
725 module (nn.Module or List[nn.Module]):
726 The module(s) to apply fully_shard to. Modified in-place. When a list
727 is passed, parameters from all modules are grouped as one FSDP unit.
729 mesh (Optional[DeviceMesh], default=None):
730 The device mesh defining the process topology for distributed training.
731 If None, fully_shard creates a default 1D mesh for local parameters.
732 Native DTensor parameters require an explicit DP submesh.
734 reshard_after_forward (bool, default=True):
735 Whether to automatically reshard parameters after forward. When True,
736 parameters are resharded immediately after they are no longer needed,
737 freeing memory for subsequent operations. Set to False if you want to
738 keep parameters unsharded for backward pass or manual control.
740 shard_placement_fn (Callable, default=None):
741 A callable that determines how to shard each parameter. The function
742 should accept a parameter and return a Shard object specifying the
743 sharding dimension, or None to use default sharding (dimension 0)
745 mp_policy (MixedPrecisionPolicy, default=MixedPrecisionPolicy()):
746 Mixed precision training policy controlling data type conversions.
748 offload_policy (OffloadPolicy, default=OffloadPolicy()):
749 Memory offload policy for reducing device memory usage.
751 ignored_params (Optional[set[nn.Parameter]], default=None):
752 Set of parameters to exclude from fully_shard management entirely.
753 These parameters are left on the original module as regular parameters,
754 are not sharded, and do not participate in fully_shard gradient
755 synchronization. Use this for parameters that should remain outside
756 the fully_shard lifecycle.
758 replicate_params (Optional[set[nn.Parameter]], default=None):
759 Set of parameters to keep replicated while still managing them under
760 fully_shard. These parameters are not sharded, but their gradients
761 are still synchronized with DDP-style all-reduce over the current
762 fully_shard communication domain. This differs from ``ignored_params``,
763 which skips fully_shard management and gradient synchronization
764 entirely for the selected parameters.
766 comm_fusion (bool, default=False):
767 Whether enable all_gather fusion and reduce_scatter fusion.
769 comm_fusion_zero_copy (Optional[bool], default=None):
770 Whether allow the experimental zero-copy path for
771 ``comm_fusion``. When set to ``None``, fully_shard enables it
772 automatically whenever ``comm_fusion=True``.
773 When enabled, fully_shard may rebase sharded local parameter storage
774 into one shared flat buffer so fused all-gather can read directly from
775 contiguous memory. This path depends on optimizer compatibility with
776 view-backed parameters.
777 source_shard_infos (Optional[Mapping[nn.Parameter, SourceShardMetaInfo]]):
778 Source TP/EP mesh and placements for the plain-parameter dual mode.
780 Returns:
781 nn.Module or List[nn.Module]: The input module(s) with HSDP capabilities added.
782 """
783 _validate_module_for_fully_shard(module)
785 arg_module = module
786 if isinstance(module, list):
787 modules = tuple(_get_root_modules(module))
788 else:
789 modules = (module,)
791 for mod in modules:
792 _extend_module_with_hsdp_interface(mod)
794 params = _get_modules_parameters(modules, ignored_params)
795 has_dtensor_param = any(isinstance(param, DTensor) for param in params)
796 _validate_managed_params_source_shard_infos(set(params), source_shard_infos)
798 replicate_params = _normalize_replicate_params(replicate_params)
800 if mesh is None:
801 mesh = init_device_mesh(device_type="npu", mesh_shape=(dist.get_world_size(),))
802 if has_dtensor_param:
803 raise ValueError(
804 "fully_shard does not support mesh=None with a native DTensor parameter; "
805 "pass an explicit DP submesh instead."
806 )
807 device = _get_device_from_mesh(mesh)
809 init_modules = modules
810 modules[0].hsdp_init(
811 init_modules,
812 mesh,
813 reshard_after_forward,
814 shard_placement_fn,
815 mp_policy,
816 offload_policy,
817 ignored_params,
818 replicate_params,
819 device,
820 comm_fusion,
821 comm_fusion_zero_copy,
822 source_shard_infos=source_shard_infos,
823 )
824 # Share the same scheduler handle with other roots so mods[i].unshard()/prefetch work
825 if len(modules) > 1:
826 for mod in modules[1:]:
827 mod.hsdp_scheduler = modules[0].hsdp_scheduler
828 return arg_module
831def get_model_state_dict(model: Any, *, options: Any = None) -> dict[str, Any]:
832 """Get the model state dict of a fully_shard-managed model.
834 Args:
835 model: The model whose state dict to retrieve.
836 options: Optional :class:`StateDictOptions` controlling gathering
837 (``full_state_dict``), offloading (``cpu_offload``) and frozen
838 parameter filtering (``ignore_frozen_params``).
840 Returns:
841 The model state dict. By default values are sharded DTensors; when
842 ``full_state_dict=True`` values are full Tensors (CPU on rank 0 only
843 when ``cpu_offload=True``).
844 """
845 return get_model_state_dict_impl(model, options=options)
848def set_model_state_dict(model: Any, model_state_dict: dict[str, Any], *, options: Any = None) -> None:
849 """Load ``model_state_dict`` into a fully_shard-managed model.
851 Full tensors are scattered into DTensor shards matching the model's layout
852 before the in-place load.
854 Args:
855 model: The model to load state into.
856 model_state_dict: State dict to load. Values may be plain (global)
857 tensors when ``full_state_dict=True`` or sharded DTensors otherwise.
858 options: Optional :class:`StateDictOptions` controlling scattering
859 (``full_state_dict``), device placement (``cpu_offload``) and
860 strictness (``strict``).
862 Returns:
863 None.
864 """
865 return set_model_state_dict_impl(model, model_state_dict, options=options)
868def hsdp_sync_stream():
869 """Wait for hsdp gradient handle to be completed."""
870 wait_grad_handle()