Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed / _builder / fsdp_adapter.py: 82%
398 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""fsdp_adapter: FSDP2 construction-time adapter (05 §15.2.4).
17Wrap policy, mesh/policy construction, ``fully_shard`` application and
18prefetch configuration all happen at model-construction time (not in the
19Trainer step runtime); ``FSDP2Manager`` / ``_instantiate_fsdp2`` and the
20``parallelize`` signature are unchanged. The source-layout discovery logic
21(FQN/DTensor source-shard resolution) merged into
22``distributed/_builder/source_shard.py`` (05 row 406).
23"""
25from __future__ import annotations
27import fnmatch
28import logging
29from collections import defaultdict
30from collections.abc import Mapping, Sequence
31from dataclasses import dataclass
32from typing import TYPE_CHECKING, Any
34import torch
35from torch.nn import Module as ModuleClass
36from torch.nn import Parameter as ParameterClass
38from hyper_parallel.components.modules.module_family import collect_family_modules
39import hyper_parallel.core.fully_shard.utils as fully_shard_utils
40from hyper_parallel import DeviceMesh, HSDPModule, fully_shard
41from hyper_parallel.core.fully_shard.hsdp_utils import get_managed_modules_parameters
42from hyper_parallel.distributed._builder.source_shard import (
43 SourceShardInfoByFQN, # pylint: disable=unused-import
44 SourceShardInfoByParam, # pylint: disable=unused-import
45 _build_managed_source_shard_info,
46 _build_source_shard_info_by_param,
47 _source_infos_for_fully_shard,
48)
49from hyper_parallel.models.build_options import FSDP2Config
50from hyper_parallel.models.registry import get_model_adapter
52if TYPE_CHECKING:
53 from hyper_parallel.distributed.mesh import (
54 MeshContext,
55 )
57logger = logging.getLogger(__name__)
60@dataclass(frozen=True)
61class _WrapModuleInfo:
62 """One FSDP child unit selected by automatic or adapter policy."""
64 fqn: str
65 module: ModuleClass
68def _is_expert_source_mesh(mesh: DeviceMesh | None) -> bool:
69 """Whether a source mesh carries the expert (EP) axis."""
70 return mesh is not None and "ep" in (getattr(mesh, "mesh_dim_names", None) or ())
73def _get_checkpoint_wrapped_module(module: ModuleClass) -> ModuleClass | None:
74 """Return the module directly wrapped by a supported checkpoint wrapper."""
75 for attr_name in ("_wrapped_module", "_checkpoint_wrapped_module"):
76 wrapped_module = getattr(module, attr_name, None)
77 if isinstance(wrapped_module, ModuleClass):
78 return wrapped_module
79 return None
82def _declared_expert_unit_fqns(model, expert_parameters):
83 """Find family/native execution units by identity rather than an experts path suffix."""
84 expert_module_fqns = set()
85 for boundary in collect_family_modules(model):
86 parameters = set(boundary.module.parameters())
87 if parameters and parameters <= expert_parameters:
88 expert_module_fqns.add(boundary.fqns[0])
89 for fqn, module in model.named_modules():
90 binding = getattr(module, "_hp_parallel_binding", None)
91 if binding is not None and binding.ep_context is not None:
92 expert_module_fqns.add(fqn)
93 expert_module_fqns.discard("")
94 return expert_module_fqns
97class FSDP2Manager:
98 """Configure and apply nested FSDP2 wrapping for a dual-mode model."""
100 def __init__(
101 self,
102 config: FSDP2Config,
103 mesh_context: MeshContext,
104 fp32_main_params: bool = False,
105 ) -> None:
106 """Initialize the manager from resolved YAML config and runtime mesh.
108 Args:
109 config: Resolved top-level ``fsdp_config``.
110 mesh_context: Runtime topology containing the shared world mesh.
111 fp32_main_params: Whether FSDP writes reduced gradients to
112 ``main_grad`` for the fp32 main-parameter optimizer wrapper.
113 """
114 self.config = config
115 self.mesh_context = mesh_context
116 self.fp32_main_params = fp32_main_params
117 self._mp_policy: fully_shard_utils.MixedPrecisionPolicy | None = None
120 def _resolve_replicate_params(
121 self,
122 model: ModuleClass,
123 ) -> set[ParameterClass] | None:
124 """Resolve configured replicate-parameter FQNs on the final model."""
125 if not self.config.replicate_params:
126 return None
128 parameters_by_fqn = dict(model.named_parameters(remove_duplicate=False))
129 replicate_params = set()
130 for parameter_fqn in self.config.replicate_params:
131 parameter = parameters_by_fqn.get(parameter_fqn)
132 if parameter is None:
133 raise ValueError(
134 "replicate_params contains unknown parameter FQN: "
135 f"{parameter_fqn}"
136 )
137 replicate_params.add(parameter)
138 return replicate_params
140 def _build_expert_fsdp_mesh(self) -> DeviceMesh:
141 """Select the expert FSDP child mesh."""
142 expert_mesh = self.mesh_context.fsdp_moe_mesh
143 if expert_mesh is None:
144 raise ValueError("Expert FSDP units require MeshContext.fsdp_moe_mesh")
145 if "edp_replicate" in (expert_mesh.mesh_dim_names or ()):
146 return expert_mesh[("edp_replicate", "edp_shard")]
147 return expert_mesh["edp_shard"]
149 def _build_dense_fsdp_mesh(self) -> DeviceMesh | None:
150 """Select the prebuilt dense FSDP child mesh when available."""
151 dense_mesh = self.mesh_context.fsdp_non_moe_mesh
152 if dense_mesh is None:
153 return None
154 if self.mesh_context.dp_replicate_size > 1:
155 return dense_mesh[("fsdp_replicate", "fsdp_shard")]
156 return dense_mesh["fsdp_shard"]
158 def _build_compatibility_fsdp_mesh(self) -> DeviceMesh:
159 """Build an FSDP mesh from the combined world mesh."""
160 world_mesh = self.mesh_context.device_mesh
161 if world_mesh is None:
162 raise ValueError("FSDP2 requires MeshContext.fsdp_non_moe_mesh")
163 mesh_dim_names = world_mesh.mesh_dim_names
164 if mesh_dim_names is None:
165 raise ValueError("FSDP2 requires named DeviceMesh dimensions")
166 shard_dim_names = tuple(
167 dim_name for dim_name in ("dp_shard", "cp") if dim_name in mesh_dim_names
168 )
169 if not shard_dim_names:
170 raise ValueError("FSDP2 requires a dp_shard or cp dimension in the world mesh")
171 shard_mesh = (
172 world_mesh[shard_dim_names[0]]
173 if len(shard_dim_names) == 1
174 else world_mesh[shard_dim_names].flatten("fsdp_shard")
175 )
176 if "dp_replicate" not in mesh_dim_names:
177 return shard_mesh
178 return DeviceMesh.concatenate([world_mesh["dp_replicate"], shard_mesh])
180 def _build_fsdp_actual_mesh(self, expert: bool = False) -> DeviceMesh:
181 """Select the prebuilt dense or expert FSDP child mesh."""
182 if expert:
183 return self._build_expert_fsdp_mesh()
184 dense_mesh = self._build_dense_fsdp_mesh()
185 if dense_mesh is not None:
186 return dense_mesh
187 return self._build_compatibility_fsdp_mesh()
189 def parameter_mesh_bindings(
190 self, model: ModuleClass, source_shard_info: SourceShardInfoByFQN | None,
191 ignored_params: set[ParameterClass] | None = None,
192 ) -> dict[ParameterClass, DeviceMesh]:
193 """Resolve unit-owned parameter meshes without installing FSDP hooks.
195 Args:
196 model: Final model before graph capture or eager wrapping.
197 source_shard_info: The planner's non-FSDP parameter placements.
198 ignored_params: Parameters managed outside FSDP.
200 The same managed-parameter selector runs inside-out in both paths.
201 Claimed identities model the initialized inner units, including tied
202 aliases; nearest-parent ownership alone would not match fully_shard.
203 """
204 metadata = _build_source_shard_info_by_param(self, model, source_shard_info)
205 units = self._find_wrap_modules(model, metadata)
206 units = [unit.module for unit in sorted(units, key=lambda unit: unit.fqn.count("."), reverse=True)]
207 claimed = set(ignored_params or ())
208 bindings = {}
209 for owner in [*units, model]:
210 parameters = get_managed_modules_parameters((owner,), claimed)
211 source = _build_managed_source_shard_info(self, owner, metadata, claimed)
212 expert = self._uses_expert_mesh(source)
213 if owner is model and expert:
214 raise ValueError("Routed expert parameters must belong to a nested experts FSDP unit")
215 mesh = self._build_fsdp_actual_mesh(expert)
216 bindings.update(dict.fromkeys(parameters, mesh))
217 claimed.update(parameters)
218 return bindings
220 def _resolve_custom_params(
221 self, parameters_by_fqn: Mapping[str, ParameterClass]
222 ) -> dict[ParameterClass, dict[str, Any]]:
223 """Resolve configured names and patterns against the supplied parameter mapping."""
224 custom_params = {}
225 for parameter_fqn, custom_dtypes in self.config.mix_precision.custom_params.items():
226 matched_fqns = [parameter_fqn] if parameter_fqn in parameters_by_fqn else [
227 fqn for fqn in parameters_by_fqn if fnmatch.fnmatchcase(fqn, parameter_fqn)
228 ]
229 if not matched_fqns:
230 raise ValueError(f"mix_precision.custom_params matches no parameter: {parameter_fqn}")
231 for matched_fqn in matched_fqns:
232 parameter = parameters_by_fqn[matched_fqn]
233 if parameter in custom_params and custom_params[parameter] != custom_dtypes:
234 raise ValueError(f"mix_precision.custom_params has conflicting overrides for {matched_fqn}")
235 custom_params[parameter] = dict(custom_dtypes)
236 return custom_params
238 def _build_mixed_precision_policy(
239 self, custom_params: Mapping[ParameterClass, Mapping[str, Any]] | None = None
240 ) -> fully_shard_utils.MixedPrecisionPolicy:
241 """Build or reuse the shared precision policy for the current model."""
242 if custom_params is None and self._mp_policy is not None:
243 return self._mp_policy
244 mix_precision = self.config.mix_precision
245 dtype_by_name = {
246 None: None,
247 "bfloat16": torch.bfloat16,
248 "float16": torch.float16,
249 "float32": torch.float32,
250 }
251 custom_params = {
252 parameter: {name: dtype_by_name[dtype] for name, dtype in custom_dtypes.items()}
253 for parameter, custom_dtypes in (custom_params or {}).items()
254 }
255 self._mp_policy = fully_shard_utils.MixedPrecisionPolicy(
256 param_dtype=dtype_by_name[mix_precision.param_dtype],
257 reduce_dtype=dtype_by_name[mix_precision.reduce_dtype],
258 output_dtype=dtype_by_name[mix_precision.output_dtype],
259 cast_forward_inputs=mix_precision.cast_forward_inputs,
260 apply_grad_on_fp32_main_grad=self.fp32_main_params,
261 custom_params=custom_params if custom_params else None,
262 )
263 return self._mp_policy
265 def _build_offload_policy(self) -> fully_shard_utils.OffloadPolicy:
266 """Return CPU offload when enabled, otherwise the default no-offload policy."""
267 if not self.config.enable_offload:
268 return fully_shard_utils.OffloadPolicy()
269 return fully_shard_utils.CPUOffloadPolicy()
271 def _build_fully_shard_kwargs(
272 self,
273 fsdp_mesh: DeviceMesh,
274 ) -> tuple[dict[str, Any], dict[str, Any]]:
275 """Build child and root kwargs with the same configured reshard policy."""
276 mp_policy = self._build_mixed_precision_policy()
277 offload_policy = self._build_offload_policy()
279 fsdp_sublayer_kwargs = {
280 "mesh": fsdp_mesh,
281 "reshard_after_forward": self.config.reshard_after_forward,
282 "mp_policy": mp_policy,
283 "offload_policy": offload_policy,
284 "comm_fusion": self.config.comm_fusion,
285 "comm_fusion_zero_copy": self.config.comm_fusion_zero_copy,
286 }
287 fsdp_root_kwargs = fsdp_sublayer_kwargs.copy()
288 return fsdp_sublayer_kwargs, fsdp_root_kwargs
290 @staticmethod
291 def _find_transformer_block_modules(
292 model: ModuleClass,
293 excluded_subtree_module_fqns: set[str] | None = None,
294 ) -> tuple[list[_WrapModuleInfo], set[int]]:
295 """Find transformer blocks under HF gradient-checkpointing containers.
297 Args:
298 model: Root model to inspect.
299 excluded_subtree_module_fqns: Roots declared by a model adapter as
300 non-decoder branches or self-contained FSDP units. Automatic
301 decoder discovery skips these roots and their descendants.
302 """
303 wrap_modules = []
304 wrapped_module_ids = set()
305 excluded_subtree_module_fqns = excluded_subtree_module_fqns or set()
307 def is_inside_declared_subtree(module_fqn: str) -> bool:
308 """Whether one module is inside an automatic-discovery exclusion.
310 Args:
311 module_fqn: Fully qualified name of the module.
312 """
313 return any(
314 module_fqn == declared_fqn
315 or module_fqn.startswith(f"{declared_fqn}.")
316 for declared_fqn in excluded_subtree_module_fqns
317 )
319 for container_fqn, container in model.named_modules():
320 if container_fqn and is_inside_declared_subtree(container_fqn):
321 continue
322 if id(container) in wrapped_module_ids:
323 continue
324 if not hasattr(container, "gradient_checkpointing"):
325 continue
326 for child_name, child in container.named_children():
327 child_fqn = f"{container_fqn}.{child_name}" if container_fqn else child_name
328 if is_inside_declared_subtree(child_fqn):
329 continue
330 blocks = list(child.named_children())
331 if not blocks:
332 continue
333 for block_name, block in blocks:
334 if id(block) in wrapped_module_ids:
335 continue
336 wrapped_module_ids.add(id(block))
337 wrapped_module = _get_checkpoint_wrapped_module(block)
338 if wrapped_module is not None:
339 # The wrapper and its direct child represent one logical
340 # transformer block during module-tree traversal.
341 wrapped_module_ids.add(id(wrapped_module))
342 wrap_modules.append(
343 _WrapModuleInfo(f"{child_fqn}.{block_name}", block)
344 )
345 return wrap_modules, wrapped_module_ids
347 def _find_expert_wrap_modules(
348 self,
349 model: ModuleClass,
350 metadata_by_parameter: SourceShardInfoByParam | None,
351 wrapped_module_ids: set[int],
352 ) -> list[_WrapModuleInfo]:
353 """Find nested expert containers whose parameters use the expert mesh."""
354 if metadata_by_parameter is None or self.mesh_context.fsdp_moe_mesh is None:
355 return []
356 expert_parameters = {
357 parameter
358 for parameter, source_shard_info in metadata_by_parameter.items()
359 if _is_expert_source_mesh(source_shard_info.mesh)
360 }
361 expert_module_fqns = {
362 parameter_fqn.split(".experts.", 1)[0] + ".experts"
363 for parameter_fqn, parameter in model.named_parameters(remove_duplicate=False)
364 if parameter in expert_parameters and ".experts." in parameter_fqn
365 }
366 expert_module_fqns.update(_declared_expert_unit_fqns(model, expert_parameters))
367 expert_module_fqns.discard("")
368 module_by_fqn = dict(model.named_modules())
369 expert_wrap_modules = []
370 for expert_module_fqn in sorted(expert_module_fqns):
371 expert_module = module_by_fqn.get(expert_module_fqn)
372 if expert_module is None or id(expert_module) in wrapped_module_ids:
373 continue
374 wrapped_module_ids.add(id(expert_module))
375 expert_wrap_modules.append(
376 _WrapModuleInfo(expert_module_fqn, expert_module)
377 )
378 return expert_wrap_modules
380 @staticmethod
381 def _get_model_adapter_spec(model: ModuleClass) -> Any | None:
382 """Resolve a model adapter from the final model configuration."""
383 if hasattr(model, "_hp_model_adapter_spec"):
384 return model._hp_model_adapter_spec # pylint: disable=protected-access
385 config = getattr(model, "config", None)
386 if config is None:
387 return None
388 identities = (
389 getattr(config, "model_type", ""),
390 *((getattr(config, "architectures", None) or [])[:1]),
391 )
392 for identity in identities:
393 if identity:
394 adapter_spec = get_model_adapter(identity)
395 if adapter_spec is not None:
396 return adapter_spec
397 return None
399 @classmethod
400 def _find_adapter_declared_wrap_modules(
401 cls,
402 model: ModuleClass,
403 ) -> list[_WrapModuleInfo]:
404 """Resolve adapter-declared FSDP units outside HF decoder containers.
406 ``ModelAdapterSpec.fsdp_wrap_modules`` keeps architecture-specific
407 execution boundaries on the family adapter side. It receives the
408 finalized model and returns exact root-relative module FQNs.
410 Args:
411 model: Root model being wrapped.
413 Returns:
414 Additional FSDP child units selected by model declarations.
416 Raises:
417 ValueError: If a declaration has an invalid type, targets no
418 module in the model tree, or aliases the same module twice.
419 """
420 adapter_spec = cls._get_model_adapter_spec(model)
421 if adapter_spec is None or adapter_spec.fsdp_wrap_modules is None:
422 return []
424 declared_fqns = adapter_spec.fsdp_wrap_modules(model)
425 if isinstance(declared_fqns, str) or not isinstance(declared_fqns, Sequence):
426 raise ValueError(
427 "ModelAdapterSpec.fsdp_wrap_modules must return a sequence of exact module FQNs"
428 )
429 module_by_fqn = dict(model.named_modules())
430 declared_wrap_modules = []
431 wrapped_module_ids = set()
432 for module_fqn in declared_fqns:
433 if not isinstance(module_fqn, str) or not module_fqn:
434 raise ValueError(
435 "ModelAdapterSpec.fsdp_wrap_modules entries must be non-empty strings"
436 )
437 wrap_module = module_by_fqn.get(module_fqn)
438 if wrap_module is None:
439 raise ValueError(
440 "ModelAdapterSpec.fsdp_wrap_modules declared an unknown module: "
441 f"{module_fqn}"
442 )
443 if id(wrap_module) in wrapped_module_ids:
444 raise ValueError(
445 "ModelAdapterSpec.fsdp_wrap_modules aliases one module more than once: "
446 f"{module_fqn}"
447 )
448 wrapped_module_ids.add(id(wrap_module))
449 declared_wrap_modules.append(_WrapModuleInfo(module_fqn, wrap_module))
450 return declared_wrap_modules
452 @classmethod
453 def _find_adapter_excluded_subtrees(cls, model: ModuleClass) -> set[str]:
454 """Resolve model branches excluded from automatic decoder discovery."""
455 adapter_spec = cls._get_model_adapter_spec(model)
456 excluded_subtrees_provider = getattr(adapter_spec, "fsdp_excluded_subtrees", None)
457 if excluded_subtrees_provider is None:
458 return set()
459 if not callable(excluded_subtrees_provider):
460 raise ValueError("ModelAdapterSpec.fsdp_excluded_subtrees must be callable")
461 excluded_fqns = excluded_subtrees_provider(model) # pylint: disable=not-callable
462 if isinstance(excluded_fqns, str) or not isinstance(excluded_fqns, Sequence):
463 raise ValueError(
464 "ModelAdapterSpec.fsdp_excluded_subtrees must return a sequence of exact module FQNs"
465 )
466 module_by_fqn = dict(model.named_modules())
467 for module_fqn in excluded_fqns:
468 if not isinstance(module_fqn, str) or not module_fqn:
469 raise ValueError(
470 "ModelAdapterSpec.fsdp_excluded_subtrees entries must be non-empty strings"
471 )
472 if module_fqn not in module_by_fqn:
473 raise ValueError(
474 "ModelAdapterSpec.fsdp_excluded_subtrees declared an unknown module: "
475 f"{module_fqn}"
476 )
477 return set(excluded_fqns)
479 def _find_class_wrap_modules(self, model: ModuleClass) -> list[_WrapModuleInfo]:
480 """Select explicit classes, retaining checkpoint wrappers as the units."""
481 requested = set(self.config.wrapper_class)
482 matched = set()
483 selected = []
484 seen = set()
485 aliases = defaultdict(list)
486 for name, parameter in model.named_parameters(remove_duplicate=False):
487 aliases[parameter].append(name)
488 for fqn, module in model.named_modules():
489 if not fqn or id(module) in seen:
490 continue
491 underlying = _get_checkpoint_wrapped_module(module)
492 candidate = underlying if underlying is not None else module
493 class_name = type(candidate).__name__
494 if class_name not in requested:
495 continue
496 matched.add(class_name)
497 parameters = list(module.parameters())
498 if not parameters:
499 raise ValueError(f"wrapper_class selected a parameterless module: {fqn}")
500 if any(not name.startswith(f"{fqn}.") for param in parameters for name in aliases[param]):
501 raise ValueError(f"wrapper_class module shares parameters outside its subtree: {fqn}")
502 selected.append(_WrapModuleInfo(fqn, module))
503 seen.add(id(module))
504 seen.add(id(candidate))
505 if requested - matched:
506 raise ValueError(f"wrapper_class matched no child module for: {sorted(requested - matched)}")
507 return selected
509 def _find_wrap_modules(
510 self,
511 model: ModuleClass,
512 metadata_by_parameter: SourceShardInfoByParam | None = None,
513 ) -> list[_WrapModuleInfo]:
514 """Find automatic decoder/expert units and adapter-declared units."""
515 if self.config.wrapper_class:
516 wrap_modules = self._find_class_wrap_modules(model)
517 wrapped_module_ids = {id(unit.module) for unit in wrap_modules}
518 else:
519 declared_wrap_modules = self._find_adapter_declared_wrap_modules(model)
520 if declared_wrap_modules:
521 logger.info(
522 "Adapter-declared FSDP child units: %s",
523 [wrap_module.fqn for wrap_module in declared_wrap_modules],
524 )
525 excluded_subtree_module_fqns = {
526 wrap_module.fqn for wrap_module in declared_wrap_modules
527 }
528 excluded_subtree_module_fqns.update(
529 self._find_adapter_excluded_subtrees(model)
530 )
531 wrap_modules, wrapped_module_ids = self._find_transformer_block_modules(
532 model,
533 excluded_subtree_module_fqns=excluded_subtree_module_fqns,
534 )
535 wrapped_module_ids.update(
536 id(wrap_module.module) for wrap_module in declared_wrap_modules
537 )
538 wrap_modules.extend(declared_wrap_modules)
539 wrap_modules.extend(
540 self._find_expert_wrap_modules(
541 model,
542 metadata_by_parameter,
543 wrapped_module_ids,
544 )
545 )
546 if not wrap_modules:
547 raise ValueError(
548 "wrap_policy='transformer_block' found no decoder, expert, or "
549 "adapter-declared FSDP child unit"
550 )
551 return wrap_modules
553 @staticmethod
554 def _resolve_parameter_owners(
555 model: ModuleClass,
556 wrap_modules: list[_WrapModuleInfo],
557 ) -> dict[ParameterClass, ModuleClass]:
558 """Assign each parameter to its deepest common FSDP wrap module."""
559 aliases_by_parameter: dict[ParameterClass, list[str]] = defaultdict(list)
560 for parameter_fqn, parameter in model.named_parameters(remove_duplicate=False):
561 aliases_by_parameter[parameter].append(parameter_fqn)
563 owner_by_parameter = {}
564 for parameter, parameter_fqns in aliases_by_parameter.items():
565 common_wrap_modules = [
566 wrap_module
567 for wrap_module in wrap_modules
568 if all(
569 parameter_fqn.startswith(f"{wrap_module.fqn}.")
570 for parameter_fqn in parameter_fqns
571 )
572 ]
573 if common_wrap_modules:
574 owner_by_parameter[parameter] = max(
575 common_wrap_modules,
576 key=lambda wrap_module: wrap_module.fqn.count("."),
577 ).module
578 else:
579 owner_by_parameter[parameter] = model
580 return owner_by_parameter
583 def _uses_expert_mesh(self, metadata_by_parameter: SourceShardInfoByParam | None) -> bool:
584 """Return whether one FSDP unit owns routed-expert source metadata."""
585 if metadata_by_parameter is None or self.mesh_context.fsdp_moe_mesh is None:
586 return False
587 return any(
588 _is_expert_source_mesh(source_shard_info.mesh)
589 for source_shard_info in metadata_by_parameter.values()
590 )
593 @staticmethod
594 def _build_managed_replicate_params(
595 owner: ModuleClass,
596 owner_by_parameter: Mapping[ParameterClass, ModuleClass],
597 replicate_params: set[ParameterClass] | None,
598 ignored_params: set[ParameterClass] | None = None,
599 ) -> set[ParameterClass] | None:
600 """Select configured replicate parameters owned by one FSDP module."""
601 if replicate_params is None:
602 return None
603 return {
604 parameter
605 for parameter, parameter_owner in owner_by_parameter.items()
606 if parameter_owner is owner and parameter in replicate_params
607 and parameter not in (ignored_params or ())
608 }
610 def _configure_source_layout_gradient_scaling(
611 self,
612 hsdp_module: HSDPModule,
613 managed_source_shard_info: SourceShardInfoByParam | None,
614 ) -> bool:
615 """Restore global-mean gradients for one source-layout FSDP unit.
617 Master defaults every ``fully_shard`` unit to AVG reduction, so units
618 with source-layout metadata explicitly select SUM. The token-weighted
619 backward loss is scaled by the FSDP data domain (DP x CP), so a
620 source-layout SUM-reduced unit must apply its inverse exactly once.
621 With loss parallelism, TP ranks hold shards or partial contributions
622 to one logical loss and TP is not another averaging axis. Without loss
623 parallelism, logits and the loss are replicated over TP, so the SUM
624 path also accumulates ``tp_size`` identical loss replicas and must
625 divide by TP. EP is excluded because expert ranks own different
626 parameters.
628 Returns:
629 Whether a gradient scaling factor was configured for the unit.
630 """
631 if not managed_source_shard_info:
632 return False
633 hsdp_module.set_reduce_op_type("sum", recurse=False)
635 tp_loss_replica_size = (
636 1 if self.mesh_context.loss_parallel else self.mesh_context.tp_size
637 )
638 source_gradient_domain_size = (
639 self.mesh_context.dp_size
640 * self.mesh_context.cp_size
641 * tp_loss_replica_size
642 )
643 if source_gradient_domain_size <= 1:
644 return False
645 hsdp_module.set_gradient_scaling_factor(1.0 / source_gradient_domain_size)
646 return True
648 def _configure_prefetch(self, hsdp_modules: list[HSDPModule]) -> None:
649 """Configure forward and backward prefetch targets by list depth."""
650 for module_index, hsdp_module in enumerate(hsdp_modules):
651 forward_end = module_index + self.config.forward_prefetch_depth + 1
652 hsdp_module.set_modules_to_forward_prefetch(
653 hsdp_modules[module_index + 1:forward_end]
654 )
656 backward_start = max(
657 0, module_index - self.config.backward_prefetch_depth
658 )
659 hsdp_module.set_modules_to_backward_prefetch(
660 list(reversed(hsdp_modules[backward_start:module_index]))
661 )
663 def _order_wrap_modules(
664 self,
665 model: ModuleClass,
666 wrap_modules: list[_WrapModuleInfo],
667 ) -> list[_WrapModuleInfo]:
668 """Order FSDP units by adapter execution order or module traversal."""
669 module_order = {
670 id(module): module_index
671 for module_index, (_, module) in enumerate(model.named_modules())
672 }
673 default_order = sorted(
674 wrap_modules,
675 key=lambda wrap_module: module_order.get(id(wrap_module.module), len(module_order)),
676 )
677 adapter_spec = self._get_model_adapter_spec(model)
678 execution_order_provider = getattr(adapter_spec, "fsdp_execution_order", None)
679 if execution_order_provider is None:
680 return default_order
681 if not callable(execution_order_provider):
682 raise ValueError("ModelAdapterSpec.fsdp_execution_order must be callable")
684 wrap_module_by_fqn = {
685 wrap_module.fqn: wrap_module for wrap_module in wrap_modules
686 }
687 execution_fqns = execution_order_provider( # pylint: disable=not-callable
688 model,
689 tuple(wrap_module.fqn for wrap_module in default_order),
690 )
691 if isinstance(execution_fqns, str) or not isinstance(execution_fqns, Sequence):
692 raise ValueError(
693 "ModelAdapterSpec.fsdp_execution_order must return a sequence of module FQNs"
694 )
695 if any(not isinstance(module_fqn, str) for module_fqn in execution_fqns):
696 raise ValueError(
697 "ModelAdapterSpec.fsdp_execution_order entries must be module FQN strings"
698 )
699 if (
700 len(execution_fqns) != len(wrap_module_by_fqn)
701 or len(set(execution_fqns)) != len(execution_fqns)
702 or set(execution_fqns) != set(wrap_module_by_fqn)
703 ):
704 raise ValueError(
705 "ModelAdapterSpec.fsdp_execution_order must return every selected FSDP "
706 "child-unit FQN exactly once"
707 )
708 logger.info("Adapter-declared FSDP execution order: %s", list(execution_fqns))
709 return [wrap_module_by_fqn[module_fqn] for module_fqn in execution_fqns]
711 def _parallelize_child_units(
712 self,
713 wrap_modules: list[_WrapModuleInfo],
714 owner_by_parameter,
715 metadata_by_parameter,
716 replicate_params,
717 dense_fsdp_kwargs: dict[str, Any],
718 ignored_params: set[ParameterClass] | None = None,
719 ) -> int:
720 """Apply FSDP to child units and return the gradient-scaled count."""
721 gradient_scaled_units = 0
722 wrapping_order = sorted(
723 wrap_modules, key=lambda wrap_module: wrap_module.fqn.count("."), reverse=True
724 )
725 for wrap_module in wrapping_order:
726 managed_source_info = _build_managed_source_shard_info(
727 self,
728 wrap_module.module,
729 metadata_by_parameter,
730 ignored_params,
731 )
732 fsdp_sublayer_kwargs = dense_fsdp_kwargs
733 if self._uses_expert_mesh(managed_source_info):
734 if self.mesh_context.fsdp_moe_mesh is None:
735 raise ValueError("Expert TP metadata requires MeshContext.fsdp_moe_mesh")
736 fsdp_sublayer_kwargs, _ = self._build_fully_shard_kwargs(
737 self._build_fsdp_actual_mesh(expert=True)
738 )
739 fully_shard( # pylint: disable=unexpected-keyword-arg
740 wrap_module.module,
741 source_shard_infos=_source_infos_for_fully_shard(managed_source_info),
742 replicate_params=self._build_managed_replicate_params(
743 wrap_module.module, owner_by_parameter, replicate_params, ignored_params
744 ),
745 ignored_params=ignored_params,
746 **fsdp_sublayer_kwargs,
747 )
748 if self._configure_source_layout_gradient_scaling(wrap_module.module, managed_source_info):
749 gradient_scaled_units += 1
750 return gradient_scaled_units
752 def _parallelize_root(
753 self,
754 model: ModuleClass,
755 owner_by_parameter,
756 metadata_by_parameter,
757 replicate_params,
758 dense_root_kwargs: dict[str, Any],
759 ignored_params: set[ParameterClass] | None = None,
760 ) -> int:
761 """Apply root FSDP and report whether gradient scaling was configured."""
762 root_source_info = _build_managed_source_shard_info(
763 self,
764 model,
765 metadata_by_parameter,
766 ignored_params,
767 )
768 if self._uses_expert_mesh(root_source_info):
769 raise ValueError("Routed expert parameters must belong to a nested experts FSDP unit")
770 fully_shard( # pylint: disable=unexpected-keyword-arg
771 model,
772 source_shard_infos=_source_infos_for_fully_shard(root_source_info),
773 replicate_params=self._build_managed_replicate_params(
774 model, owner_by_parameter, replicate_params, ignored_params
775 ),
776 ignored_params=ignored_params,
777 **dense_root_kwargs,
778 )
779 return int(self._configure_source_layout_gradient_scaling(model, root_source_info))
781 def _log_gradient_scaling(self, gradient_scaled_units: int) -> None:
782 """Log the source-layout gradient scaling applied to FSDP units."""
783 if not gradient_scaled_units:
784 return
785 domain_size = (
786 self.mesh_context.dp_size
787 * self.mesh_context.cp_size
788 * (1 if self.mesh_context.loss_parallel else self.mesh_context.tp_size)
789 )
790 logger.info(
791 "Applied source-layout SUM gradient scaling: dp_size=%d, cp_size=%d, tp_size=%d, "
792 "source_gradient_domain_size=%d, factor=%s, units=%d",
793 self.mesh_context.dp_size,
794 self.mesh_context.cp_size,
795 self.mesh_context.tp_size,
796 domain_size,
797 1.0 / domain_size,
798 gradient_scaled_units,
799 )
801 def parallelize(
802 self,
803 model: ModuleClass,
804 source_shard_info: SourceShardInfoByFQN | None = None,
805 ignored_params: set[ParameterClass] | None = None,
806 ) -> ModuleClass:
807 """Apply configured child FSDP units, root FSDP, and prefetch links.
809 Args:
810 model: Model to wrap in place.
811 source_shard_info: FQN-keyed source-layout metadata returned by
812 ``apply_sharding_plan``. The manager resolves parameter
813 identities before applying nested FSDP units.
815 Returns:
816 The model enhanced with the HyperParallel HSDPModule interface.
818 Raises:
819 ValueError: If TP is enabled without FQN metadata or configured
820 parameter FQNs cannot be resolved.
821 """
822 self._mp_policy = None
823 metadata_by_parameter = _build_source_shard_info_by_param(
824 self,
825 model,
826 source_shard_info,
827 )
828 replicate_params = self._resolve_replicate_params(model)
829 custom_params = None
830 if self.config.mix_precision.custom_params:
831 custom_params = self._resolve_custom_params(dict(model.named_parameters(remove_duplicate=False)))
832 self._build_mixed_precision_policy(custom_params)
834 wrap_modules = self._find_wrap_modules(model, metadata_by_parameter)
835 owner_by_parameter = self._resolve_parameter_owners(model, wrap_modules)
836 dense_fsdp_kwargs, dense_root_kwargs = self._build_fully_shard_kwargs(
837 self._build_fsdp_actual_mesh()
838 )
840 gradient_scaled_units = self._parallelize_child_units(
841 wrap_modules,
842 owner_by_parameter,
843 metadata_by_parameter,
844 replicate_params,
845 dense_fsdp_kwargs,
846 ignored_params,
847 )
848 gradient_scaled_units += self._parallelize_root(
849 model,
850 owner_by_parameter,
851 metadata_by_parameter,
852 replicate_params,
853 dense_root_kwargs,
854 ignored_params,
855 )
856 self._log_gradient_scaling(gradient_scaled_units)
857 ordered_wrap_modules = self._order_wrap_modules(model, wrap_modules)
858 self._configure_prefetch(
859 [wrap_module.module for wrap_module in ordered_wrap_modules]
860 )
861 logger.info(
862 "Applied FSDP2 to %d child units and the root module",
863 len(wrap_modules),
864 )
865 return model
868def _instantiate_fsdp2(
869 *,
870 config: Any,
871 mesh_context: MeshContext,
872 fp32_main_params: bool = False,
873) -> FSDP2Manager | None:
874 """Create an FSDP2 manager for the resolved strategy config."""
875 if config is None:
876 return None
877 if isinstance(config, FSDP2Config):
878 return FSDP2Manager(
879 config=config,
880 mesh_context=mesh_context,
881 fp32_main_params=fp32_main_params,
882 )
883 raise ValueError(f"FSDP2Manager requires FSDP2Config, got {type(config).__name__}")
886def _apply_fsdp2(
887 model: ModuleClass,
888 fsdp2_manager: Any,
889 source_shard_info: Any,
890 ignored_params: set[ParameterClass] | None = None,
891) -> ModuleClass:
892 """Apply FSDP2; Torch scheduler hooks remain outside Dynamo by design.
894 Moved from ``_transformers/infrastructure.py`` (05 §15.2.1) so the FSDP
895 application step lives with ``FSDP2Manager.parallelize``.
896 """
897 if fsdp2_manager is None:
898 return model
899 if not isinstance(fsdp2_manager, FSDP2Manager):
900 logger.warning("fsdp2_manager is not an FSDP2Manager instance")
901 return model
902 model = fsdp2_manager.parallelize(
903 model,
904 source_shard_info=source_shard_info,
905 ignored_params=ignored_params,
906 )
907 logger.info("FSDP2 wrap applied")
908 return model