Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed / _builder / source_shard.py: 86%
132 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2025-2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15# pylint: disable=missing-apache-license-header
16"""source_shard: build_source_shard_info (05 §6.7.1).
18source_shard_info is read from the ShardingPlan (rather than from DTensors — under
19production the parameters have already been unwrapped by
20_local_params_context, and only the plan retains the complete placement
21information).
23Each entry records the parameter's **complete source layout**: the placements
24on every source (non-FSDP) mesh axis, together with the source sub-mesh.
25``fully_shard`` later prefixes its own FSDP shard/replicate dimensions, so the
26final sharded parameter carries the full distributed layout (FSDP dims +
27source dims) for downstream consumers such as distributed checkpointing.
28"""
30from __future__ import annotations
32from collections.abc import Mapping
33from typing import Any, Dict, List, Optional, Tuple, TypeAlias
35from torch.nn import Module as ModuleClass
36from torch.nn import Parameter as ParameterClass
37from torch.utils.weak import WeakIdKeyDictionary
39import hyper_parallel.core.fully_shard.utils as fully_shard_utils
40from hyper_parallel import DeviceMesh, DTensor, Replicate
41from hyper_parallel.components.modules.module_family import _family_source_shard_info
42from hyper_parallel.core.dtensor.placement_types import Partial, Placement, Shard
43from hyper_parallel.core.fully_shard.hsdp_utils import get_managed_modules_parameters
44from hyper_parallel.distributed.plan import ShardingPlan
45from hyper_parallel.distributed.recipe_spec import EP, resolve_placements
47# Mesh dimensions whose sharding semantics belong to FSDP (or to other
48# parallelism concerns), never to the source layout recorded here:
49# - dp/dp_shard/dp_replicate/cp: FSDP shards weights over the whole DP+CP
50# domain (cp is folded into the fsdp shard axis at mesh build time), and
51# FSDP's reduce-scatter already covers gradient sync on these axes;
52# - pp: different stages hold different parameters (distinguished by FQN),
53# a weight is never distributed along pp;
54# - fsdp_*/edp_*: the FSDP child-mesh axes themselves.
55# Declaring a Shard placement for a weight on one of these axes would
56# double-count the sharding (Phase A shard + FSDP shard on the same ranks),
57# so build_source_shard_info rejects it fail-fast.
58FSDP_OWNED_DIMS = frozenset({
59 "dp", "dp_replicate", "dp_shard", "cp", "pp",
60 "fsdp_replicate", "fsdp_shard",
61 "edp", "edp_replicate", "edp_shard",
62})
65def _source_dim_names(mesh, fallback_dim_names):
66 """Return the non-FSDP axis names of a source mesh.
68 Falls back to the plan's axis names when the mesh carries no dimension
69 names (e.g. a mock or a plain 1-D TP mesh).
70 """
71 dim_names = getattr(mesh, "mesh_dim_names", None) if mesh is not None else None
72 if dim_names is None:
73 dim_names = tuple(fallback_dim_names or ())
74 return tuple(name for name in dim_names if name not in FSDP_OWNED_DIMS)
77def _source_sub_mesh(mesh, source_dim_names):
78 """Slice the source sub-mesh covering exactly ``source_dim_names``."""
79 if mesh is None:
80 return None
81 dim_names = getattr(mesh, "mesh_dim_names", None)
82 if dim_names is None or tuple(dim_names) == tuple(source_dim_names):
83 return mesh
84 if not source_dim_names:
85 return None
86 if len(source_dim_names) == 1:
87 return mesh[source_dim_names[0]]
88 return mesh[source_dim_names]
91def _check_fsdp_owned_axes(full_fqn, named_placement):
92 """Reject weight sharding declared on axes owned by FSDP/PP."""
93 for axis, placement in named_placement.items():
94 if axis in FSDP_OWNED_DIMS and not placement.is_replicate():
95 raise ValueError(
96 f"{full_fqn}: placement {placement} on axis {axis!r} conflicts with "
97 "FSDP ownership of that axis (weights are sharded over the DP+CP "
98 "domain by fully_shard itself); declare Replicate or remove the axis"
99 )
102def _normalize_tied_pair_placements(info, tied_pairs):
103 """Normalize tied parameters to the finer source placement."""
104 if not tied_pairs:
105 return
106 for a, b in tied_pairs:
107 if a in info and b in info:
108 pa, _ = info[a]
109 pb, _ = info[b]
110 if pa != pb and len(pa) == len(pb):
111 norm = tuple(
112 x if isinstance(x, Shard) else y
113 for x, y in zip(pa, pb)
114 )
115 info[a] = (norm, info[a][1])
116 info[b] = (norm, info[b][1])
119def build_source_shard_info(
120 plan: ShardingPlan,
121 dense_source_mesh: Any,
122 *,
123 expert_source_mesh: Any = None,
124 tied_pairs: Optional[List[Tuple[str, str]]] = None,
125) -> Dict[str, Tuple[Tuple[Placement, ...], Any]]:
126 """Build complete source-layout metadata for dense and routed-expert parameters.
128 Args:
129 plan: The finalized ShardingPlan.
130 dense_source_mesh: Dense-region source mesh. May be the full dense
131 FSDP mesh (e.g. ``(fsdp_replicate, fsdp_shard, tp)``) — FSDP-owned
132 axes are stripped; or a plain TP child mesh (used as-is).
133 expert_source_mesh: Full expert mesh (e.g. ``(edp_shard, ep)``) or the
134 EP child mesh; required when any spec marks routed experts.
135 tied_pairs: parameter pairs with shared storage (defaults to
136 plan.tied_pairs). Both ends of a tied pair must map to the same
137 placements — axis by axis, the finer sharding wins (Shard takes
138 precedence over Replicate), guaranteeing consistent TP all-reduce /
139 reduce-scatter semantics on both ends.
141 Returns:
142 ``{param_fqn: (placements_tuple, source_sub_mesh)}`` where
143 ``placements_tuple`` holds one placement per source (non-FSDP) mesh
144 axis, missing axes filled with ``Replicate()``.
145 """
146 dense_source_dims = _source_dim_names(dense_source_mesh, plan.mesh_dim_names)
147 dense_mesh = _source_sub_mesh(dense_source_mesh, dense_source_dims)
148 expert_source_dims = _source_dim_names(expert_source_mesh, ("ep",))
149 expert_mesh = _source_sub_mesh(expert_source_mesh, expert_source_dims)
151 info = {}
152 for fqn, spec in plan.modules.items():
153 for param_name, named_placement in spec.params.items():
154 full_fqn = f"{fqn}.{param_name}" if fqn else param_name
155 _check_fsdp_owned_axes(full_fqn, named_placement)
156 if (spec._ep_size > 0 # pylint: disable=protected-access
157 and isinstance((named_placement or {}).get(EP), Shard)):
158 if expert_source_mesh is None:
159 raise ValueError(
160 "Virtual-EP parameter metadata requires an EP source mesh"
161 )
162 placements = tuple(resolve_placements(named_placement, expert_source_dims))
163 info[full_fqn] = (placements, expert_mesh)
164 else:
165 placements = tuple(resolve_placements(named_placement, dense_source_dims))
166 info[full_fqn] = (placements, dense_mesh)
168 _normalize_tied_pair_placements(
169 info,
170 tied_pairs if tied_pairs is not None else plan.tied_pairs,
171 )
172 return info
175# ────────────────────────────────────────────────────────────────────────────
176# FSDP2 source-layout discovery (moved verbatim from components/distributed/
177# fsdp2.py in stage 4f, 05 §15.2.4 row 406; ``self`` became an explicit
178# ``manager`` parameter — FSDP2Manager calls these free functions).
179# ────────────────────────────────────────────────────────────────────────────
181SourceShardInfoByFQN: TypeAlias = Mapping[ # pylint: disable=invalid-name
182 str, tuple[tuple[Placement, ...], DeviceMesh]
183]
186SourceShardInfoByParam: TypeAlias = dict[ # pylint: disable=invalid-name
187 ParameterClass,
188 "fully_shard_utils.SourceShardMetaInfo",
189]
192def _build_dtensor_source_shard_info(
193 manager,
194 model: ModuleClass,
195) -> SourceShardInfoByParam | None:
196 """Build source metadata from DTensor parameters when TP is active."""
197 if manager.mesh_context.tp_size <= 1:
198 return None
200 parameters = list(model.parameters())
201 if not parameters or not all(
202 isinstance(parameter, DTensor) for parameter in parameters
203 ):
204 raise ValueError(
205 "TP is enabled but source_shard_info is missing and "
206 "not all model parameters are DTensors"
207 )
208 return {
209 parameter: fully_shard_utils.SourceShardMetaInfo( # pylint: disable=no-member
210 mesh=parameter.device_mesh,
211 placements=tuple(parameter.placements),
212 origin_is_dtensor=True,
213 )
214 for parameter in parameters
215 }
218def _build_parameter_source_shard_info(
219 parameter_fqn: str,
220 placements: tuple[Placement, ...],
221 source_mesh: DeviceMesh,
222) -> fully_shard_utils.SourceShardMetaInfo:
223 """Validate and build source metadata for one model parameter."""
224 if any(isinstance(placement, Partial) for placement in placements):
225 raise ValueError(
226 "source_shard_info does not support Partial placement: "
227 f"{parameter_fqn}"
228 )
229 return fully_shard_utils.SourceShardMetaInfo( # pylint: disable=no-member
230 mesh=source_mesh,
231 placements=tuple(placements),
232 origin_is_dtensor=False,
233 )
236def _record_parameter_source_shard_info(
237 metadata_by_parameter: SourceShardInfoByParam,
238 parameter: ParameterClass,
239 parameter_fqn: str,
240 parameter_source_shard_info: fully_shard_utils.SourceShardMetaInfo,
241) -> None:
242 """Record one layout while rejecting conflicting tied aliases."""
243 previous_source_shard_info = metadata_by_parameter.get(parameter)
244 if previous_source_shard_info is not None and (
245 previous_source_shard_info.mesh is not parameter_source_shard_info.mesh
246 or previous_source_shard_info.placements
247 != parameter_source_shard_info.placements
248 ):
249 raise ValueError(
250 "Tied parameter aliases have conflicting source layouts; "
251 f"conflict found at {parameter_fqn}"
252 )
253 metadata_by_parameter[parameter] = parameter_source_shard_info
256def _build_source_shard_info_by_param(
257 manager,
258 model: ModuleClass,
259 source_shard_info: SourceShardInfoByFQN | None,
260) -> SourceShardInfoByParam | None:
261 """Resolve stable FQNs to parameter identities after model rewrites.
263 Activation-checkpoint wrappers may add an internal child-module prefix,
264 but the supported wrapper contract removes that prefix from
265 ``named_modules``/``named_parameters``. Therefore planner metadata
266 produced before checkpointing remains resolvable here, while FSDP still
267 receives the final parameter objects that it will manage.
269 Args:
270 model: Model after sharding and checkpoint rewrites, before layer
271 compile is installed.
272 source_shard_info: Source-layout metadata produced by the TP planner.
274 Returns:
275 Parameter-keyed metadata for later ``fully_shard`` calls, or
276 ``None`` when tensor parallelism is disabled.
278 Raises:
279 ValueError: If TP metadata is missing, references an unknown FQN,
280 contains ``Partial``, or gives tied aliases conflicting layouts.
281 """
282 constructed = _family_source_shard_info(model)
283 if constructed:
284 for name, layout in constructed.items():
285 if name in (source_shard_info or {}) and source_shard_info[name] != layout:
286 raise ValueError(f"source_shard_info conflicts with the constructed family layout at {name!r}")
287 source_shard_info = {**constructed, **(source_shard_info or {})}
288 if source_shard_info is None:
289 return _build_dtensor_source_shard_info(manager, model)
290 if manager.mesh_context.tp_size > 1 and not source_shard_info:
291 raise ValueError("TP is enabled but source_shard_info is empty")
293 parameters_by_fqn = dict(model.named_parameters(remove_duplicate=False))
294 metadata_by_parameter: SourceShardInfoByParam = {}
295 for parameter_fqn, (placements, source_mesh) in source_shard_info.items():
296 parameter = parameters_by_fqn.get(parameter_fqn)
297 if parameter is None:
298 raise ValueError(
299 f"source_shard_info contains unknown parameter FQN: {parameter_fqn}"
300 )
301 parameter_source_shard_info = _build_parameter_source_shard_info(
302 parameter_fqn,
303 placements,
304 source_mesh,
305 )
306 _record_parameter_source_shard_info(
307 metadata_by_parameter,
308 parameter,
309 parameter_fqn,
310 parameter_source_shard_info,
311 )
312 return metadata_by_parameter
315def _get_default_source_shard_info(manager) -> fully_shard_utils.SourceShardMetaInfo:
316 """Build all-Replicate source metadata covering every non-FSDP mesh axis.
318 Used to complete source_shard_info for parameters the plan does not mention;
319 the axis set must match the per-parameter entries so that all parameters
320 in one ``fully_shard`` unit share the same source-mesh dimensionality.
321 """
322 world_mesh = manager.mesh_context.fsdp_non_moe_mesh or manager.mesh_context.device_mesh
323 if world_mesh is None or world_mesh.mesh_dim_names is None:
324 raise ValueError("TP metadata completion requires a named world mesh")
325 source_dim_names = tuple(
326 dim_name
327 for dim_name in world_mesh.mesh_dim_names
328 if dim_name not in FSDP_OWNED_DIMS
329 )
330 if not source_dim_names:
331 raise ValueError(
332 "TP is enabled but the world mesh has no non-FSDP source dimension"
333 )
334 source_mesh = (
335 world_mesh
336 if tuple(world_mesh.mesh_dim_names) == source_dim_names
337 else world_mesh[source_dim_names[0]]
338 if len(source_dim_names) == 1
339 else world_mesh[source_dim_names]
340 )
341 return fully_shard_utils.SourceShardMetaInfo( # pylint: disable=no-member
342 mesh=source_mesh,
343 placements=tuple(Replicate() for _ in source_dim_names),
344 origin_is_dtensor=False,
345 )
348def _build_managed_source_shard_info(
349 manager,
350 owner: ModuleClass,
351 metadata_by_parameter: SourceShardInfoByParam | None,
352 ignored_params: set[ParameterClass] | None = None,
353) -> SourceShardInfoByParam | None:
354 """Select and complete metadata for parameters managed by one wrap call.
356 Metadata must use the same parameter selection as ``fully_shard``. In
357 particular, nested FSDP units mark their parameters initialized before the
358 parent is wrapped, so ownership assigned from the pre-wrap module tree can
359 over-approximate the actual parameter set. Deriving this set through the
360 shared helper keeps source-layout metadata complete and exact.
361 """
362 if metadata_by_parameter is None:
363 return None
365 default_source_shard_info = None
366 if (
367 manager.mesh_context.fsdp_non_moe_mesh is not None
368 or manager.mesh_context.device_mesh is not None
369 ):
370 default_source_shard_info = _get_default_source_shard_info(manager)
371 managed_source_shard_info = {}
372 for parameter in get_managed_modules_parameters((owner,), ignored_params):
373 source_shard_info = metadata_by_parameter.get(parameter)
374 if source_shard_info is None:
375 if default_source_shard_info is None:
376 raise ValueError(
377 "source_shard_info is present but does not cover an FSDP-managed parameter"
378 )
379 source_shard_info = default_source_shard_info
380 managed_source_shard_info[parameter] = source_shard_info
381 return managed_source_shard_info
384def _source_infos_for_fully_shard(
385 managed_source_shard_info: SourceShardInfoByParam | None,
386) -> Mapping[ParameterClass, fully_shard_utils.SourceShardMetaInfo] | None:
387 """Return non-owning metadata only for plain source-layout parameters.
389 Validate-mode parameters remain native DTensors. The FSDP
390 state derives their source layouts from each parameter directly and
391 rejects duplicate explicit metadata.
393 The adapter keeps its construction-local mapping for post-wrap gradient
394 configuration. FSDP receives weak identity keys so its scheduler and state
395 cannot keep the replaced original parameters and their storage alive.
396 """
397 if managed_source_shard_info is None:
398 return None
399 if managed_source_shard_info and all(
400 source_shard_info.origin_is_dtensor
401 for source_shard_info in managed_source_shard_info.values()
402 ):
403 return None
404 return WeakIdKeyDictionary(managed_source_shard_info)