Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed / _builder / planner.py: 94%
582 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2025-2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15# pylint: disable=missing-apache-license-header
16"""planner: ShardingPlanner 6-phase derivation pipeline (05 §3.6 canonical).
18Phase 1 parameter role classification (ParameterClassifier + the family's
19 ModelAdapterSpec.sharding_rules from models/registry.py)
20Phase 2 communication boundary grouping (two passes: first group by owning
21 module, then merge upward depth-first — fixes the flaw in the
22 05 §3.6.6 pseudocode where "single-parameter group inference"
23 misjudges a q_proj leaf module as an mlp boundary)
24Phase 3 semantic role inference (explicit FQN patterns > structural guards
25 > parameter role combinations)
26Phase 4 template lookup to generate spec (_build_spec_from_template)
27Phase 4.5 user plan_overrides merge/insert (_merge_plan_overrides, 05 §3.6.7)
28Phase 5 _is_terminal marking only (D-14, 05 §13: compile-time chain
29 propagation/validation removed — specs are fully self-declared and
30 each module vouches for its own propagation in validate mode)
31Phase 6 special parameter handler collection (SPECIAL_HANDLERS)
33Registries:
34- family sharding rules: ``ModelAdapterSpec.sharding_rules`` providers in
35 ``models/<family>/adapter/registration.py``, resolved lazily through
36 ``models/registry.py::get_model_adapter`` (any spelling — model_type,
37 HF architecture or canonical arch name);
38- ``SPECIAL_HANDLERS``: {handler_name: callable(module, param_name, mesh)} —
39 lives in ``_builder/special_handlers.py``.
40"""
42import copy
43import logging
44import re
45from typing import Any, Dict, List, Optional, Tuple
47from torch import nn
49from hyper_parallel.components.modules.module_family import FamilyShardingContext, _family_parameter_shapes
50from hyper_parallel.distributed._builder.family_layout import collect_owned_layouts, validate_owned_layouts
51from hyper_parallel.models.registry import resolve_model_adapter
52from hyper_parallel.core.dtensor.device_mesh import DeviceMesh
53from hyper_parallel.core.dtensor.placement_types import (
54 Partial,
55 Replicate,
56 Shard,
57)
58from hyper_parallel.distributed.plan import ShardingPlan
59from hyper_parallel.distributed.recipe_spec import (
60 DP,
61 EP,
62 TP,
63 ModuleShardingSpec,
64)
65from hyper_parallel.distributed.tensor_parallel.param_role import (
66 ParameterClassifier,
67 ParamRole,
68 _match_any,
69)
70from hyper_parallel.distributed._builder.default_templates import (
71 ShardingTemplate,
72 TEMPLATES,
73 _build_spec_from_template,
74 _finalize_tp_local_attr_plans,
75 _moe_expert_tp_placement,
76 _multi_dim,
77)
78from hyper_parallel.distributed._builder.function_module import FunctionModule
79from hyper_parallel.distributed._builder.rule_resolver import (
80 _last_segment,
81 _merge_plan_overrides,
82 _normalize_contract_fields,
83)
84from hyper_parallel.distributed._builder.special_handlers import (
85 _SPECIAL_HANDLER_PATTERNS,
86 _collect_special_handlers,
87)
89logger = logging.getLogger(__name__)
91# Per-family naming-rule overrides no longer live here: each family declares
92# them via ``ModelAdapterSpec.sharding_rules`` in
93# ``models/<family>/adapter/registration.py`` (DeepSeek MLA, Qwen2-MoE
94# shared_expert_gate), discovered through ``models/registry.py``. The
95# planner core stays family-agnostic (05 §15.9 step 1).
98# Leaf-segment guard for projection/container segment names: these segment
99# names are not boundary containers themselves; inference returns unknown
100# and continues upward.
101# NOTE (accuracy fix F3): "shared_experts" was REMOVED from this guard —
102# the shared expert (singular or plural spelling) must surface as its own
103# nested "mlp" boundary (its boundary exit owns the RowWise Partial TP
104# reduction, accuracy_problem.md 10.3 Option A); keeping it in the guard would
105# merge it into the parent MoE boundary and silently leave the TP reduction
106# without an owner.
107_LEAF_SEGMENT_GUARD = frozenset({
108 "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj",
109 "qkv_proj", "fused_qkv", "linear_qkv", "gate_up_proj", "query_key_value",
110 "experts", "gate", "linear", "proj",
111 "fc1", "fc2", "w1", "w2", "w3", "w13", "dense", "dense_h_to_4h", "dense_4h_to_h",
112})
114_ATTN_PATTERNS = ("attn", "attention")
115_MLP_PATTERNS = ("mlp", "ffn", "feed_forward")
116_MOE_CONTAINER_PATTERNS = ("mlp", "moe", "moe_block", "moe_layer")
119def _get_architecture(model) -> str:
120 """Detect the canonical architecture name:
121 config.architectures[0] > config.model_type > class name;
122 lowercased with ForCausalLM-style suffixes stripped."""
123 cfg = getattr(model, "config", None)
124 arch_str = None
125 archs = getattr(cfg, "architectures", None)
126 if archs:
127 arch_str = archs[0]
128 if not arch_str:
129 arch_str = getattr(cfg, "model_type", None)
130 if not arch_str:
131 arch_str = type(model).__name__
133 s = arch_str.lower()
134 for suffix in ("forcausallm", "forconditionalgeneration",
135 "forsequenceclassification", "forimagetexttotext"):
136 if s.endswith(suffix):
137 s = s[: -len(suffix)]
138 return s
141class ShardingPlanner:
142 """Automatically derive a ShardingPlan from any HF-style model (05 §3.6.6).
144 ``plan_overrides``: {module_fqn | fqn_glob: ModuleShardingSpec} — the
145 SINGLE injection/override interface (05 §3.6.7 + the unification rework):
147 - **merge mode** (key hits a planner-derived boundary): empty contract
148 fields (``params``/``in_src``/``in_dst``/``out_src``/``out_dst``/
149 ``out_names``) INHERIT the derived values, non-empty ones replace
150 (field granularity); the string sentinels ``"auto"`` (explicit
151 inherit — self-documenting "derive per template") and ``"none"``
152 (explicit clear) are resolved at merge time and never reach the plan
153 output; the injection fields (``local_compute_fn`` /
154 ``inner_target`` / ``inner_wrapper`` non-None, ``region_dispatch=False``)
155 always win; internal flags (``_ep_size`` / ``_ep_stack`` /
156 ``_needs_cp_attn``) always inherit. This is how
157 CP/EP compute injection is declared — no need to re-declare contracts;
158 - **insert mode** (exact key misses every derived boundary): the spec is
159 inserted as-is and must be fully self-declared — an override with
160 empty params AND empty contracts fails fast ("no template matched");
161 - **glob keys** (containing ``*``/``?``/``[``): merge-applied to every
162 matching boundary. A glob with a concrete parameter or I/O contract
163 also expands against the final model tree and inserts unmatched modules
164 with the same validation as exact insert mode. Partial globs carrying
165 only injection/attribute changes remain merge-only and fail if they
166 target a model module for which no boundary was derived.
168 The YAML transport (``hyper_parallel.trainer.config.PlanOverride``) is
169 converted to this dict by ``entries_to_plan_overrides()`` on the trainer
170 side BEFORE the
171 planner is constructed — the planner itself has exactly one override
172 interface.
174 ``derive=False`` turns the planner into a pure declaration assembler:
175 template derivation (Phases 2-4) is skipped and the plan contains ONLY
176 the ``plan_overrides`` specs — every key is insert mode and must be
177 fully self-declared. Use it for subtrees where automatic derivation is
178 semantically wrong, e.g. the multimodal encoder_dp ViT bridge (each
179 rank encodes a different image shard, so any derived TP collective
180 inside the ViT would silently mix samples). This replaces the error-
181 prone post-hoc pruning idiom ``plan.modules = {"": plan.modules[""]}``.
182 """
184 def __init__(
185 self,
186 plan_overrides: Optional[Dict[str, ModuleShardingSpec]] = None,
187 *,
188 derive: bool = True,
189 allow_uncovered_params: bool = False,
190 ) -> None:
191 """Create a ShardingPlanner.
193 Args:
194 plan_overrides: ``{module_fqn | fqn_glob: ModuleShardingSpec}`` —
195 the single injection/override interface (05 §3.6.7); see the
196 class docstring for merge/insert/glob semantics.
197 derive: When ``False``, template derivation (Phases 2-4) is
198 skipped and the plan contains ONLY the ``plan_overrides``
199 specs (every key is insert mode, fully self-declared).
200 allow_uncovered_params: When ``True``, downgrade the F4b
201 "every trainable parameter must be covered by the plan"
202 hard error to a warning (exploratory debugging only).
203 """
204 self._classifier = ParameterClassifier()
205 self._templates = TEMPLATES
206 self._special_handler_patterns = dict(_SPECIAL_HANDLER_PATTERNS)
207 self._plan_overrides = dict(plan_overrides or {})
208 # derive=False: skip template derivation entirely — the plan contains
209 # ONLY the plan_overrides specs (every key is insert mode and must be
210 # fully self-declared). This replaces post-hoc pruning of
211 # ``plan.modules`` (e.g. the multimodal encoder_dp ViT bridge, where
212 # each rank encodes different images and ANY derived TP collective
213 # inside the ViT subtree would be a math error).
214 self._derive = derive
215 # F4b (accuracy_fix_plan.md §2): by default every trainable parameter
216 # must be covered by the plan (fail-fast). Set True ONLY for
217 # exploratory debugging to downgrade the hard error to a warning.
218 self._allow_uncovered_params = allow_uncovered_params
220 # ── Main entry point ────────────────────────────────────────────────
222 def _derive_boundary_specs(
223 self,
224 plan: ShardingPlan,
225 boundary_groups,
226 *,
227 sequence_parallel: bool,
228 loss_parallel: bool,
229 mesh_dim_names: Tuple[str, ...],
230 arch: str,
231 ep_extend: int,
232 mesh: DeviceMesh,
233 model: Any,
234 param_ndims: Dict[str, int],
235 ) -> None:
236 """Materialize template-derived boundary specs into a plan."""
237 if not self._derive:
238 logger.info(
239 "derive=False: template derivation skipped — the plan will contain only plan_overrides "
240 "specs (all insert mode, fully self-declared)"
241 )
242 return
243 for boundary_fqn, group in boundary_groups.items():
244 boundary_type = self._infer_boundary_type(boundary_fqn, group)
245 if self._templates.get(boundary_type) is None:
246 logger.warning("No template for boundary_type=%s at %s", boundary_type, boundary_fqn)
247 continue
248 spec = _build_spec_from_template(
249 self._templates,
250 boundary_fqn,
251 group,
252 self._templates.get(boundary_type),
253 sequence_parallel,
254 loss_parallel,
255 mesh_dim_names,
256 param_ndims=param_ndims,
257 )
258 if spec is None:
259 continue
260 if boundary_type == "moe_mlp":
261 self._mark_hf_native_moe(
262 spec, group, boundary_fqn, self._templates.get(boundary_type), mesh_dim_names, arch,
263 ep_extend=ep_extend, mesh=mesh, model=model, param_ndims=param_ndims,
264 )
265 plan.modules[boundary_fqn] = spec
267 def _finalize_boundary_specs(
268 self,
269 plan: ShardingPlan,
270 model: Any,
271 *,
272 tp_size: int,
273 ep_size: int,
274 mesh: DeviceMesh,
275 mesh_dim_names: Tuple[str, ...],
276 ) -> None:
277 """Normalize overrides and finish placement-dependent boundary metadata."""
278 _merge_plan_overrides(self._plan_overrides, plan, model, derive=self._derive)
279 _normalize_contract_fields(plan)
280 self._mark_virtual_ep_parameters(plan, ep_size, mesh)
281 _finalize_tp_local_attr_plans(plan, model, tp_size=tp_size, mesh_dim_names=mesh_dim_names)
282 self._finalize_deferred_biases(plan, model, mesh_dim_names)
283 self._finalize_fused_expert_tp_guard(plan, tp_size=tp_size)
285 @staticmethod
286 def _log_explanation(plan: ShardingPlan, enabled: bool) -> None:
287 """Log the plan explanation when requested."""
288 if enabled:
289 logger.info("ShardingPlan explain:\n%s", plan.explain())
291 @staticmethod
292 def _resolve_isolated_roots(model: Any, arch: str) -> tuple[str, ...]:
293 """Find subtrees whose activation layout is owned by a separate runtime."""
294 from hyper_parallel.models.registry import get_model_adapter # pylint: disable=C0415
296 adapter = (model._hp_model_adapter_spec if hasattr(model, "_hp_model_adapter_spec") # pylint: disable=protected-access
297 else get_model_adapter(arch))
298 if adapter is None or adapter.encoder_parallel is None:
299 return ()
300 spec = adapter.encoder_parallel()
301 names = {name for name, _ in model.named_modules()}
302 return tuple(root for root in spec.isolated_subtrees if root in names)
304 def _classify_boundary_groups(self, model: Any, arch: str, isolated_roots=(), excluded_names=()):
305 """Classify model parameters and group them into candidate boundaries."""
306 param_roles = {name: role for name, role in self._classify_all_params(model, arch).items()
307 if name not in excluded_names}
308 regular_roles = {
309 name: role for name, role in param_roles.items()
310 if not any(name.startswith(root + ".") for root in isolated_roots)
311 }
312 return param_roles, self._group_by_boundary(regular_roles)
314 def plan(
315 self,
316 model: Any,
317 mesh: DeviceMesh,
318 *,
319 tp_size: int = 1,
320 cp_size: int = 1,
321 ep_size: int = 1,
322 sequence_parallel: bool = True,
323 loss_parallel: bool = False,
324 explain: bool = False,
325 ) -> ShardingPlan:
326 """Derive a ShardingPlan from *model* and *mesh* (6-phase pipeline).
328 Args:
329 model: Any HuggingFace ``PreTrainedModel``.
330 mesh: A :class:`~hyper_parallel.core.dtensor.device_mesh.DeviceMesh`
331 whose ``mesh_dim_names`` declare the physical topology axes (e.g.
332 ``("dp", "tp")`` or ``("dp_replicate", "tp", "cp")``). The
333 ``tp`` / ``cp`` axes are managed by DTensor; ``dp*`` / ``pp``
334 axes belong to FSDP2 / pipeline-parallel runtimes and are
335 filtered out.
336 tp_size: Tensor-parallel group size. Must be **equal** to
337 ``mesh[\"tp\"].size()`` when ``mesh_dim_names`` contains ``"tp"``
338 and tp_size > 1; a mismatch raises :class:`ValueError`
339 immediately (fail-first).
340 cp_size: Context-parallel group size. Same fail-first contract as
341 *tp_size* with respect to ``mesh[\"cp\"]``.
342 ep_size: Expert-parallel group size.
344 - **D-10 TP-extend-EP** (the default MoE path, 05 §6.4.8): the
345 mesh does **not** have an ``"ep"`` axis; *ep_size* is the
346 extended EP group size, and the expert mesh ``(edp, ep)`` is
347 derived by flattening the full dense region
348 (``dp × tp × cp``). *ep_size* is **not** validated against
349 the mesh in this case.
350 - **Old-style EP** (when ``mesh_dim_names`` contains ``"ep"``):
351 *ep_size* must equal ``mesh[\"ep\"].size()``, validated
352 fail-first.
353 sequence_parallel: When ``True`` (default), sequence dimension is
354 sharded across TP (Shard(1) on the ``tp`` axis of activations).
355 loss_parallel: When ``True``, lm_head output stays Shard(-1);
356 otherwise gathered to Replicate (cross-entropy compatibility).
357 explain: When ``True``, log the plan introspection report
358 (:meth:`ShardingPlan.explain`) at INFO level after planning —
359 per-boundary param sharding, compiled communication plans,
360 and injection resolutions. The same report is available
361 standalone via ``plan.explain(fqn=None)``.
363 Returns:
364 :class:`ShardingPlan`: module FQN → :class:`ModuleShardingSpec`.
366 Raises:
367 ValueError: If *tp_size* / *cp_size* / *ep_size* do not match the
368 corresponding mesh dimensions (fail-first), if model-level
369 constraints are violated (head divisibility, expert count, etc.),
370 or if any ``plan_overrides`` spec declares a DP placement
371 (05 §3.1.1 coordinate-system convention: the plan is always a
372 single dp slice).
373 """
374 arch = _get_architecture(model)
375 self._check_overrides_no_dp() # fail-first: the plan's coordinate system = a single dp slice
376 mesh_dim_names = self._build_mesh_dim_names(mesh, tp_size, cp_size, ep_size)
377 plan, param_roles, param_metadata = self._build_plan(
378 model,
379 mesh,
380 mesh_dim_names,
381 arch,
382 tp_size=tp_size,
383 cp_size=cp_size,
384 ep_size=ep_size,
385 sequence_parallel=sequence_parallel,
386 loss_parallel=loss_parallel,
387 )
389 # D-14 invariants, Phase 5/6, F4 lints, DX guard, tied weights
390 plan = self._finalize_plan(
391 plan,
392 model,
393 param_roles,
394 param_metadata,
395 tp_size=tp_size,
396 cp_size=cp_size,
397 ep_size=ep_size,
398 )
400 self._log_explanation(plan, explain)
402 return plan
404 def _build_plan(
405 self,
406 model: Any,
407 mesh: DeviceMesh,
408 mesh_dim_names: Tuple[str, ...],
409 arch: str,
410 *,
411 tp_size: int,
412 cp_size: int,
413 ep_size: int,
414 sequence_parallel: bool,
415 loss_parallel: bool,
416 ) -> Tuple[ShardingPlan, Dict[str, ParamRole], Tuple[Dict[str, int], Dict[str, Tuple[int, ...]]]]:
417 """Run Phases 1-4.5: classify, derive, and merge boundary specs."""
418 # D-10 TP-extend-EP (05 §6.4.8): ep_size is the extended EP group
419 # size (the a2a communication domain, extended from the TP group to
420 # neighboring dp/cp ranks; expert weights are sharded only along
421 # the expert dim; no separate etp configuration). Validation
422 # happens when _mark_hf_native_moe actually matches an HF-native
423 # MoE (pre-stacked EP-aware modules use their own dispatcher and
424 # are not subject to this constraint)
425 # Phase 1: parameter role classification
426 layout_context = FamilyShardingContext(
427 mesh_dim_names, {"tp": tp_size, "cp": cp_size, "ep": ep_size},
428 _family_parameter_shapes(model),
429 sequence_parallel, loss_parallel,
430 )
431 owned = collect_owned_layouts(model, layout_context, resolve_model_adapter(model))
432 isolated_roots = self._resolve_isolated_roots(model, arch) if self._derive else ()
433 param_roles, boundary_groups = self._classify_boundary_groups(
434 model, arch, isolated_roots, excluded_names=owned.owned_names,
435 )
437 # Phase 3+4: semantic inference + template-fills I/O
438 # F4a/F4b: full shapes + requires_grad for the plan-time lints
439 # (meta tensors carry shapes too — the zero-memory path is unaffected)
440 param_metadata = (
441 {name: p.ndim for name, p in model.named_parameters()},
442 dict(layout_context.parameter_shapes),
443 )
444 plan = ShardingPlan(
445 mesh_dim_names=mesh_dim_names,
446 sequence_parallel=sequence_parallel,
447 loss_parallel=loss_parallel,
448 )
449 self._derive_boundary_specs(
450 plan,
451 boundary_groups,
452 sequence_parallel=sequence_parallel,
453 loss_parallel=loss_parallel,
454 mesh_dim_names=mesh_dim_names,
455 arch=arch,
456 ep_extend=ep_size if ep_size > 1 else 0,
457 mesh=mesh,
458 model=model,
459 param_ndims=param_metadata[0],
460 )
462 for root in isolated_roots:
463 module = model.get_submodule(root)
464 params = {name: {TP: Replicate()} for name, parameter in module.named_parameters()
465 if parameter.requires_grad}
466 plan.modules[root] = ModuleShardingSpec(params=params, in_src={}, in_dst={}, is_boundary=False)
468 plan.modules.update(owned.specs)
469 plan.parallel_modules.update(owned.parallel)
470 plan.owned_parameter_names.update(owned.owned_names)
471 plan.tied_pairs.extend(owned.aliases)
472 plan.parameter_aliases.extend(owned.aliases)
474 # Phase 4.5: merge derived matches and expand fully declared globs
475 # against the final model tree to insert new boundaries.
476 self._finalize_boundary_specs(
477 plan,
478 model,
479 tp_size=tp_size,
480 ep_size=ep_size,
481 mesh=mesh,
482 mesh_dim_names=mesh_dim_names,
483 )
484 validate_owned_layouts(model, plan, owned, layout_context)
485 return plan, param_roles, param_metadata
487 def _finalize_plan(
488 self,
489 plan: ShardingPlan,
490 model: Any,
491 param_roles: Dict[str, ParamRole],
492 param_metadata: Tuple[Dict[str, int], Dict[str, Tuple[int, ...]]],
493 *,
494 tp_size: int,
495 cp_size: int,
496 ep_size: int,
497 ) -> ShardingPlan:
498 """Run the D-14 invariants, Phase 5/6, the F4 lints, and tied pairs."""
499 # D-14 invariants (05 §13.2/§13.3): full self-declaration + param
500 # uniqueness (the only nesting check that remains)
501 self._check_full_declaration(plan)
502 self._check_param_uniqueness(plan)
504 # Phase 5: _is_terminal marking (D-14: chain propagation removed)
505 plan = self._mark_terminal(plan, model)
507 # Phase 6: special parameter handling
508 plan.special_handlers = _collect_special_handlers(param_roles, self._special_handler_patterns)
510 # F4 plan-time lints (accuracy_fix_plan.md §2 — after Phase 4.5
511 # overrides merge and Phase 6, so hand-written specs and special
512 # handlers are all accounted for):
513 # F4a: every Shard(dim) must divide the parameter shape — an empty
514 # shard at apply time becomes a plan-time teaching error;
515 # F4b: every trainable parameter must be covered by the plan.
516 self._check_shard_divisibility(
517 plan, param_metadata[1], tp_size=tp_size, cp_size=cp_size,
518 ep_size=ep_size)
519 self._check_all_trainable_params_covered(plan, model)
521 # DX guard: FunctionModule instances not covered by any spec run
522 # without any boundary communication — warn instead of silently passing
523 self._warn_uncovered_function_modules(model, plan)
525 # tied-weight detection (embed <-> lm_head sharing storage)
526 plan.tied_pairs = list(dict.fromkeys([*plan.tied_pairs, *self._detect_tied_pairs(model)]))
527 return plan
529 # ── Architecture detection ──────────────────────────────────────────
531 @staticmethod
532 def _get_architecture(model) -> str:
533 """Compatibility wrapper for the module-level architecture helper."""
534 return _get_architecture(model)
536 @staticmethod
537 def _validate_dtensor_axes(
538 mesh, tp_size: int, cp_size: int, ep_size: int,
539 ) -> None:
540 """Fail-first: validate passed tp/cp/ep sizes against the mesh dimensions.
542 The mesh *must* declare ``mesh_dim_names`` (and the corresponding
543 ``mesh_shape``) for validation to proceed; a mesh without names skips
544 validation (backward-compatible fallback).
546 Degenerate single-rank meshes (all dimensions size 1, common in
547 compile-time unit tests) also skip validation — on a single rank every
548 DTensor placement is a no-op.
550 Rules:
551 - tp_size > 1 → mesh must contain a "tp" axis whose size equals tp_size.
552 - cp_size > 1 → mesh must contain a "cp" axis whose size equals cp_size.
553 - ep_size > 1 is validated only when the mesh has an explicit "ep"
554 dimension (old-style EP). In D-10 TP-extend-EP the ep group is
555 *derived* from the dense region, not a native mesh axis, so the
556 absence of "ep" in mesh_dim_names is expected and not an error.
557 """
558 mesh_names = tuple(getattr(mesh, "mesh_dim_names", ()) or ())
559 mesh_shape = tuple(getattr(mesh, "mesh_shape", ()) or ())
560 if not mesh_names or not mesh_shape:
561 return # cannot validate without mesh metadata
563 # Degenerate single-rank mesh (compile-time test fixtures): skip
564 if all(sz == 1 for sz in mesh_shape):
565 return
567 name_to_size = dict(zip(mesh_names, mesh_shape))
569 for ax, size in (("tp", tp_size), ("cp", cp_size)):
570 if size > 1:
571 if ax not in name_to_size:
572 raise ValueError(
573 f"{ax}_size={size} > 1, but the mesh has no '{ax}' dimension. "
574 f"Mesh dimensions: {list(name_to_size.keys())}. "
575 f"Either add '{ax}' to the mesh's mesh_dim_names, or set "
576 f"{ax}_size=1."
577 )
578 mesh_sz = name_to_size[ax]
579 if size != mesh_sz:
580 raise ValueError(
581 f"{ax}_size ({size}) does not match mesh['{ax}'] size "
582 f"({mesh_sz}). They must be equal."
583 )
585 # ep_size: only validate when "ep" is a declared mesh axis (old-style
586 # EP). D-10 TP-extend-EP does not put "ep" in mesh_dim_names — the
587 # ep group is derived from the dense region by _expert_mesh_layout.
588 if ep_size > 1 and "ep" in name_to_size:
589 mesh_sz = name_to_size["ep"]
590 if ep_size != mesh_sz:
591 raise ValueError(
592 f"ep_size ({ep_size}) does not match mesh['ep'] size "
593 f"({mesh_sz}). They must be equal."
594 )
596 def _build_mesh_dim_names(
597 self, mesh, tp_size: int, cp_size: int, ep_size: int,
598 ) -> Tuple[str, ...]:
599 """Filter tp/cp/ep with mesh.mesh_dim_names as the authoritative
600 order; fall back to (tp,cp,ep) when undeclared; drop size=1 axes.
602 dp* axes are ALWAYS stripped (05 §3 coordinate-system convention): the plan's
603 coordinate system is a single dp slice — dp semantics live in the
604 data pipeline (data assignment) and FSDP (weight/grad domain),
605 never in placements. Overrides declaring DP placements are
606 rejected fail-first by :meth:`_check_overrides_no_dp`.
608 Raises :class:`ValueError` when *tp_size* / *cp_size* do not match the
609 corresponding mesh dimension sizes (fail-first before any planning work).
610 """
611 self._validate_dtensor_axes(mesh, tp_size, cp_size, ep_size)
612 mesh_names = tuple(getattr(mesh, "mesh_dim_names", ()) or ())
613 dtensor_axes = ("tp", "cp", "ep")
614 active = {ax for ax, sz in (("tp", tp_size), ("cp", cp_size), ("ep", ep_size))
615 if sz and sz > 1}
616 if mesh_names:
617 return tuple(n for n in mesh_names if n in dtensor_axes and n in active)
618 return tuple(ax for ax in dtensor_axes if ax in active)
620 def _check_overrides_no_dp(self) -> None:
621 """Fail-first: plan_overrides must not declare DP placements.
623 The plan's coordinate system is a single dp slice (05 §3 coordinate-system convention):
624 dp placements never drive boundary communication, and parameter dp
625 layout is owned by FSDP *after* the plan — a DP key here would be
626 silently dropped at resolve time, so reject it with a teaching
627 message instead of letting the misreading happen.
628 """
630 def _declares_dp(node) -> bool:
631 return isinstance(node, dict) and any(
632 k == DP or _declares_dp(v) for k, v in node.items())
634 for fqn, spec in self._plan_overrides.items():
635 if not isinstance(spec, ModuleShardingSpec):
636 continue # wrong types are reported as TypeError by _merge_plan_overrides
637 for field in ("params", "in_src", "in_dst", "out_src", "out_dst"):
638 if _declares_dp(getattr(spec, field)):
639 raise ValueError(
640 f'plan_overrides["{fqn}"].{field}: declaring a DP '
641 "placement is not allowed. The plan's coordinate "
642 "system is a single dp slice (tp/cp/ep, 05 §3 "
643 "coordinate-system convention) — dp data sharding is "
644 "expressed by the data pipeline, parameter/gradient "
645 "sharding by FSDP; for scenarios such as multimodal "
646 "encoder_dp the dp semantics live in vit_mesh + data "
647 "assignment + fully_shard, and the I/O contract only "
648 "needs to declare tp/cp/ep."
649 )
651 # ── Phase 1 ─────────────────────────────────────────────────────────
653 @staticmethod
654 def _warn_uncovered_function_modules(model, plan) -> None:
655 """DX guard: a FunctionModule without a boundary spec gets NO
656 redistribution communication (it is invisible to the planner's
657 derivation) — surface it instead of silently passing."""
658 uncovered = [fqn for fqn, m in model.named_modules()
659 if isinstance(m, FunctionModule) and fqn not in plan.modules]
660 if uncovered:
661 logger.warning(
662 "FunctionModule %s has no boundary spec — no communication "
663 "will be inserted around it. Declare one via "
664 "plan_overrides (params={}, region_dispatch=False).",
665 uncovered,
666 )
668 def _classify_all_params(self, model, arch: str) -> Dict[str, ParamRole]:
669 """Phase 1 entry: default naming rules plus the family's declared
670 sharding rules (``ModelAdapterSpec.sharding_rules``, resolved through
671 the models registry — the planner core carries no per-family
672 knowledge)."""
673 from hyper_parallel.models.registry import ( # pylint: disable=import-outside-toplevel
674 get_model_adapter,
675 )
676 spec = (model._hp_model_adapter_spec if hasattr(model, "_hp_model_adapter_spec") # pylint: disable=protected-access
677 else get_model_adapter(arch))
678 rules = []
679 if spec is not None and spec.sharding_rules is not None:
680 rules = list(spec.sharding_rules())
681 if not rules:
682 return self._classifier.classify(model, arch)
683 classifier = ParameterClassifier(arch_overrides={arch: rules})
684 return classifier.classify(model, arch)
686 # ── Phase 2 ─────────────────────────────────────────────────────────
688 def _group_by_boundary(
689 self, param_roles: Dict[str, ParamRole],
690 ) -> Dict[str, List[Tuple[str, ParamRole]]]:
691 """Single deepest-first sweep over the module hierarchy:
693 Pass 1: group by owning module FQN (strip the leaf parameter name).
694 Pass 2: materialize ALL ancestor modules upfront and sweep them in
695 strictly decreasing depth — every shallower ancestor is
696 therefore inferred only after ALL its descendants' parameters
697 have merged into it. On ``unknown`` the whole group's
698 parameters merge upward into the parent module. If still
699 unknown after backtracking to the root, attribute the group
700 to the parameter's own module (no template will match later
701 → warning and skip).
703 (Fix for the merge-ordering hazard of the former tail-enqueue work
704 queue: a parent enqueued by an early child — e.g. the MoE gate —
705 was inferred before later descendants — e.g. the experts subtree —
706 had merged into it, and the late parameters were then silently
707 dropped into an already-consumed module. Accuracy fix F3 made this
708 observable: a MOE_GATE-only ``mlp`` group must merge upward, which
709 stranded the experts params entirely.)
710 """
711 # Pass 1
712 own: Dict[str, List[Tuple[str, ParamRole]]] = {}
713 for fqn, role in param_roles.items():
714 module_fqn = ".".join(fqn.split(".")[:-1])
715 own.setdefault(module_fqn, []).append((fqn, role))
717 # Pass 2
718 all_modules = set(own)
719 for mfqn in own:
720 parts = mfqn.split(".")
721 for i in range(1, len(parts)):
722 all_modules.add(".".join(parts[:i]))
723 merged: Dict[str, List[Tuple[str, ParamRole]]] = {
724 mfqn: list(own.get(mfqn, [])) for mfqn in all_modules
725 }
726 groups: Dict[str, List[Tuple[str, ParamRole]]] = {}
727 for mfqn in sorted(all_modules, key=lambda f: f.count("."),
728 reverse=True):
729 params = merged[mfqn]
730 if not params:
731 continue
732 if self._infer_boundary_type(mfqn, params) != "unknown":
733 groups[mfqn] = params
734 else:
735 parent = mfqn.rsplit(".", 1)[0] if "." in mfqn else ""
736 if parent:
737 merged[parent].extend(params)
738 else:
739 # Still unknown after backtracking to the root:
740 # attribute each parameter to its own module (no template
741 # will match later → skipped). Unrelated unknown
742 # subtrees may have merged at the root, so anchoring the
743 # whole group to the first parameter would produce invalid
744 # relative names for the remaining parameters.
745 for param_fqn, role in params:
746 origin = param_fqn.rpartition(".")[0]
747 groups.setdefault(origin, []).append((param_fqn, role))
748 return groups
750 # ── Phase 3 ─────────────────────────────────────────────────────────
752 @staticmethod
753 def _explicit_boundary_type(fqn_lower: str, segment: str) -> Optional[str]:
754 """Return an explicit FQN-based boundary type when one matches."""
755 if _match_any(
756 fqn_lower,
757 ["embed_tokens", "wte", ".embed.", "tok_embeddings", "embed_in", "word_embeddings"],
758 ):
759 return "embed"
760 if _match_any(fqn_lower, ["lm_head", "embed_out", "output_layer"]):
761 return "lm_head"
762 if _match_any(fqn_lower, ["norm", "layernorm", "rmsnorm", "ln_"]):
763 return "norm"
764 if _match_any(segment, ["router"]):
765 return "moe_gate"
766 return None
768 @staticmethod
769 def _moe_boundary_type(fqn: str, roles: set[ParamRole]) -> Optional[str]:
770 """Return the boundary type implied by MoE-specific parameter roles."""
771 if ParamRole.MOE_EXPERT in roles:
772 return "moe_mlp" if _match_any(_last_segment(fqn), list(_MOE_CONTAINER_PATTERNS)) else "unknown"
773 if ParamRole.SHARED_EXPERT in roles:
774 return "mlp"
775 if ParamRole.MOE_GATE in roles:
776 return "unknown"
777 return None
779 @staticmethod
780 def _dense_boundary_type(fqn_lower: str, group: List[Tuple[str, ParamRole]]) -> str:
781 """Infer an attention or MLP boundary from dense parameter roles."""
782 has_colwise = any(
783 role in (ParamRole.COLWISE, ParamRole.FUSED_QKV, ParamRole.FUSED_GATE_UP)
784 for _, role in group
785 )
786 has_rowwise = any(role == ParamRole.ROWWISE for _, role in group)
787 if has_colwise and has_rowwise:
788 if _match_any(fqn_lower, list(_ATTN_PATTERNS)):
789 return "attention"
790 if _match_any(fqn_lower, list(_MLP_PATTERNS)):
791 return "mlp"
792 return "attention"
793 if has_colwise and _match_any(fqn_lower, list(_MLP_PATTERNS)):
794 return "mlp"
795 return "unknown"
797 def _infer_boundary_type(self, fqn: str, group: List[Tuple[str, ParamRole]]) -> str:
798 """Identify the semantic role from the module FQN + the group's
799 parameter roles.
801 Priority: explicit FQN patterns > leaf-segment guard > MoE roles >
802 parameter role combinations > default.
803 """
804 fqn_lower = fqn.lower()
805 seg = _last_segment(fqn)
807 explicit_type = self._explicit_boundary_type(fqn_lower, seg)
808 if explicit_type is not None:
809 return explicit_type
811 # 2. Leaf-segment guard: projection/expert leaf modules are not
812 # boundary containers themselves
813 if seg in _LEAF_SEGMENT_GUARD:
814 return "unknown"
815 # Numeric-segment guard: HF per-expert containers (experts.0..N) are
816 # not boundaries; parameters must aggregate upward into the moe
817 # container (D-09, 05 §6.4.7)
818 if seg.isdigit():
819 return "unknown"
821 # 3. MoE roles (accuracy fix F3, accuracy_fix_plan.md §2):
822 # structural lint — a "moe_mlp" boundary must contain routed experts
823 # (MOE_EXPERT); a MOE_GATE-only group (e.g. a scalar gate Linear
824 # misclassified as router) must NOT anchor a MoE boundary. Container
825 # identity is judged on the module's OWN last segment, not a
826 # substring of the whole FQN (a parent path segment like "mlp" must
827 # not qualify a leaf Linear such as shared_expert_gate).
828 roles = {r for _, r in group}
829 moe_type = self._moe_boundary_type(fqn, roles)
830 return moe_type if moe_type is not None else self._dense_boundary_type(fqn_lower, group)
832 # ── Phase 4 post-processing: MoE EP marking (D-09/D-10, 05 §6.4.7/§6.4.8) ──
834 @staticmethod
835 def _validate_ep_domain(ep_extend, mesh) -> None:
836 """Validate that virtual EP evenly partitions the dense rank domain.
838 The dense region = all ranks of the non-pp mesh axes
839 (dp_replicate × dp_cp × tp).
840 """
841 names = tuple(getattr(mesh, "mesh_dim_names", ()) or ())
842 shape = tuple(getattr(mesh, "mesh_shape", ()) or ())
843 domain = 1
844 for name, size in zip(names, shape):
845 if name == "pp" and size > 1:
846 raise NotImplementedError(
847 "virtual EP v1 does not support pp>1 "
848 "(split the mesh by stage before calling)"
849 )
850 if name != "pp":
851 domain *= size
852 if ep_extend > domain or domain % ep_extend != 0:
853 raise ValueError(
854 f"ep_size ({ep_extend}) must not exceed and must divide "
855 f"the dense region (dp_replicate × dp_cp × tp = {domain})"
856 )
858 @classmethod
859 def _validate_ep_extend(cls, ep_extend, mesh, model) -> None:
860 """Validate virtual EP plus the routed-expert count constraint."""
861 cls._validate_ep_domain(ep_extend, mesh)
862 num_experts = (getattr(getattr(model, "config", None), "num_experts", None)
863 or getattr(getattr(model, "config", None), "n_routed_experts", 0))
864 if num_experts and num_experts % ep_extend != 0:
865 raise ValueError(
866 f"num_experts ({num_experts}) must be divisible by ep_size ({ep_extend})"
867 )
869 @classmethod
870 def _mark_virtual_ep_parameters(
871 cls,
872 plan: ShardingPlan,
873 ep_size: int,
874 mesh: DeviceMesh,
875 ) -> None:
876 """Map explicit virtual-EP parameters to the derived expert mesh.
878 This deliberately keys off placement semantics rather than parameter
879 names, so routed experts and sparse lookup tables use one mechanism.
880 The boundary still has to inject its matching all-to-all computation.
881 """
882 if ep_size <= 1 or "ep" in tuple(getattr(mesh, "mesh_dim_names", ()) or ()):
883 return
884 marked = []
885 for fqn, spec in plan.modules.items():
886 if getattr(spec, "_ep_size", 0): # pylint: disable=protected-access
887 continue
888 ep_params = [
889 name
890 for name, named in (spec.params or {}).items()
891 if isinstance((named or {}).get(EP), Shard)
892 ]
893 if ep_params:
894 spec._ep_size = ep_size # pylint: disable=protected-access
895 marked.append((fqn, ep_params))
896 if not marked:
897 return
898 cls._validate_ep_domain(ep_size, mesh)
899 logger.info(
900 "virtual EP: mapped explicitly EP-sharded parameters to the "
901 "derived expert mesh: %s",
902 marked,
903 )
905 # per-expert parameter pattern: experts.<idx>.<proj>.weight (legacy HF /
906 # in-house MoE layouts).
907 _PER_EXPERT_RE = re.compile(r"^experts\.(\d+)\.([^.]+)\.weight$")
908 # batched parameter pattern: experts.<attr> (a single attribute with no
909 # numeric segment, the layout after the HF 2025 refactor —
910 # gate_up_proj [E, 2I, H] / down_proj [E, H, I], natively stacked with
911 # no stacking needed; the automodel names gate_and_up_projs/down_projs
912 # are isomorphic).
913 _BATCHED_EXPERT_RE = re.compile(
914 r"^experts\.(gate_up_proj|gate_and_up_projs|down_proj|down_projs"
915 r"|gate_proj|up_proj)$")
916 # Fused (gate|up merged into ONE parameter, chunk-split inside forward)
917 # expert weight pattern — covers both the batched 3D layout
918 # (``experts.gate_up_proj [E, 2I, H]``) and the per-expert 2D layout
919 # (``experts.<idx>.gate_up_proj.weight [2I, H]``). Such weights are
920 # incompatible with contiguous-block TP Shard on the fused dim; see
921 # _finalize_fused_expert_tp_guard.
922 _FUSED_EXPERT_WEIGHT_RE = re.compile(
923 r"^experts\.(?:\d+\.)?(?:gate_up_proj|gate_and_up_projs"
924 r"|fused_gate_up|w13)(?:\.weight)?$")
926 def _detect_expert_layout(self, group, boundary_fqn: str, param_ndims):
927 """Return expert parameters grouped by per-expert and batched layouts."""
928 expert_params = [fqn for fqn, role in group if role == ParamRole.MOE_EXPERT]
929 if not expert_params:
930 return None
931 stacks: Dict[str, List[Tuple[int, str]]] = {}
932 batched: List[str] = []
933 for param_fqn in expert_params:
934 rel = param_fqn[len(boundary_fqn) + 1:]
935 if "bias" in rel.lower():
936 logger.warning(
937 "%s: MoE expert has bias (%s); not supported in v1, skipping EP marking",
938 boundary_fqn, rel,
939 )
940 return None
941 match = self._PER_EXPERT_RE.match(rel)
942 if match is not None:
943 stacks.setdefault(match.group(2), []).append((int(match.group(1)), rel))
944 elif self._BATCHED_EXPERT_RE.match(rel) is not None and (param_ndims or {}).get(param_fqn, 2) >= 3:
945 batched.append(rel)
946 if stacks and batched:
947 logger.warning(
948 "%s: mixed per-expert and batched layouts (%s ...); skipping EP marking",
949 boundary_fqn, batched[0],
950 )
951 return None
952 return expert_params, stacks, batched
954 @staticmethod
955 def _mark_old_style_experts(
956 spec: ModuleShardingSpec,
957 stacks: Dict[str, List[Tuple[int, str]]],
958 template: ShardingTemplate,
959 ) -> None:
960 """Stack per-expert parameters while retaining explicit TP and EP axes."""
961 for projection, items in stacks.items():
962 items.sort()
963 sources = [rel for _, rel in items]
964 stacked = f"experts.{projection}"
965 tp_placement = _moe_expert_tp_placement(stacked, ndim=3, template=template)
966 for rel in sources:
967 spec.params.pop(rel, None)
968 spec.params[stacked] = _multi_dim(
969 tp=tp_placement, cp=Replicate(), ep=template.moe_expert_placement
970 )
971 spec._ep_stack[stacked] = sources # pylint: disable=protected-access # planner owns the spec DSL internals
973 @staticmethod
974 def _mark_extended_expert_params(
975 spec: ModuleShardingSpec,
976 expert_params: List[str],
977 stacks: Dict[str, List[Tuple[int, str]]],
978 batched: List[str],
979 boundary_fqn: str,
980 template: ShardingTemplate,
981 ) -> None:
982 """Mark expert parameters for TP-extended EP sharding."""
983 if stacks:
984 for projection, items in stacks.items():
985 items.sort()
986 sources = [rel for _, rel in items]
987 stacked = f"experts.{projection}"
988 for rel in sources:
989 spec.params.pop(rel, None)
990 spec.params[stacked] = _multi_dim(
991 tp=None, cp=Replicate(), ep=template.moe_expert_placement
992 )
993 # The planner owns the spec DSL internals.
994 spec._ep_stack[stacked] = sources # pylint: disable=protected-access
995 spec.region_dispatch = None
996 return
997 if batched:
998 for rel in batched:
999 spec.params[rel] = _multi_dim(
1000 tp=None, cp=Replicate(), ep=template.moe_expert_placement
1001 )
1002 spec.region_dispatch = None
1003 return
1004 for param_fqn in expert_params:
1005 rel = param_fqn[len(boundary_fqn) + 1:]
1006 spec.params[rel] = _multi_dim(
1007 tp=None, cp=Replicate(), ep=template.moe_expert_placement
1008 )
1010 @staticmethod
1011 def _set_extended_ep_contract(spec: ModuleShardingSpec, ep_extend: int) -> None:
1012 """Set the identity boundary contract used by TP-extended EP."""
1013 identity = copy.deepcopy(spec.in_src)
1014 spec.in_dst = copy.deepcopy(identity)
1015 out_layout = copy.deepcopy(next(iter(identity.values())))
1016 spec.out_src = {"output": copy.deepcopy(out_layout)}
1017 spec.out_dst = {"output": copy.deepcopy(out_layout)}
1018 spec._ep_size = ep_extend # pylint: disable=protected-access # planner owns the spec DSL internals
1020 def _mark_hf_native_moe(
1021 self, spec: ModuleShardingSpec, group, boundary_fqn: str,
1022 template: ShardingTemplate, mesh_dim_names: Tuple[str, ...], arch: str,
1023 *, ep_extend: int = 0, mesh=None, model=None, param_ndims=None,
1024 ) -> None:
1025 """Post-process moe_mlp spec for EP (05 §6.4.7/§6.4.8).
1027 The EP **mode** is determined solely by whether the mesh includes an
1028 explicit ``"ep"`` axis — NOT by parameter naming:
1030 * **Old-style EP** (``"ep" in mesh_dim_names``): the mesh already has
1031 an ``"ep"`` axis. :meth:`_build_spec_from_template` produced
1032 ``{TP: Shard(…), EP: Shard(0)}`` — the correct dual-axis
1033 sharding. This method only performs **stacking** when the expert
1034 layout is per-expert 2D (``experts.<idx>.<proj>.weight``); the
1035 stacked entry keeps both TP and EP keys. Batched 3D and custom
1036 (``w1``/``w2``/``w3``) layouts are already ready — nothing to do.
1037 No ``_ep_size`` is set; communication is handled by the module's
1038 own dispatcher or external ``_attach_ep``.
1040 * **D-10 TP-extend-EP** (``"ep" not in mesh_dim_names``, the
1041 default for HF-native models): the ep group is *derived* from the
1042 dense region. Expert weights are rewritten to
1043 ``{EP: Shard(0)}`` (no TP key), the boundary contract is changed
1044 to SP-in identity, and ``_ep_size`` is set.
1046 The expert **layout** (per-expert / batched / custom) only affects
1047 the *stacking strategy* — orthogonal to the EP mode:
1049 - **per-expert**: ``experts.<idx>.<proj>.weight`` → stack into
1050 ``experts.<proj>`` 3D before sharding;
1051 - **batched** (D-11): ``experts.gate_up_proj`` etc., natively 3D;
1052 - **custom**: ``experts.w1`` / ``w2`` / ``w3``, already 3D,
1053 pre-stacked by the module author.
1055 In D-10 mode all three layouts are supported; in old-style EP mode
1056 only per-expert needs stacking (batched and custom are already 3D).
1057 """
1058 # ``arch`` is part of the dispatch-interface signature shared with the
1059 # other boundary post-processors; the EP mode is decided by the mesh
1060 # and layout alone, so it is intentionally unused here.
1061 _ = arch
1062 if not ep_extend:
1063 return
1064 layout = self._detect_expert_layout(group, boundary_fqn, param_ndims)
1065 if layout is None:
1066 return
1067 expert_params, stacks, batched = layout
1068 if "ep" in mesh_dim_names:
1069 if stacks:
1070 self._mark_old_style_experts(spec, stacks, template)
1071 return
1072 self._validate_ep_extend(ep_extend, mesh, model)
1073 self._mark_extended_expert_params(
1074 spec, expert_params, stacks, batched, boundary_fqn, template
1075 )
1076 self._set_extended_ep_contract(spec, ep_extend)
1079 @staticmethod
1080 def _finalize_fused_expert_tp_guard(plan: ShardingPlan, *, tp_size: int) -> None:
1081 """Fail fast on fused expert weights TP-sharded without D-10 EP.
1083 A fused expert weight (gate/up merged into ONE parameter — e.g.
1084 ``experts.gate_up_proj [E, 2I, H]`` — and chunk-split inside the
1085 module's forward) is incompatible with the framework's
1086 contiguous-block TP ``Shard`` on the fused dim: after Phase A each
1087 rank holds a contiguous slice of the fused dim (tp=2 → rank0 all
1088 gate, rank1 all up), so the runtime ``chunk`` result is NOT
1089 equivalent to the non-TP semantics (Megatron solves this with
1090 chunk-aware sharding, which the framework does not support). The
1091 error is silent: the MoE boundary is local-region black-box hosted
1092 in BOTH modes, so production and validate run the same (wrong)
1093 local forward and the dual-mode comparison cannot expose it.
1095 Runs AFTER all plan_overrides are merged (next to
1096 ``_finalize_deferred_biases``) and anchors detection on the FINAL
1097 spec declarations, so an explicit TP-Replicate override on the
1098 fused params legitimately clears the hazard. D-10 TP-extend-EP
1099 (``_ep_size > 0``) removes the TP key from expert weights entirely
1100 (each rank holds complete experts), so it is exempt.
1101 """
1102 if tp_size <= 1:
1103 return
1104 offenders = []
1105 for fqn, spec in plan.modules.items():
1106 if getattr(spec, "_ep_size", 0): # pylint: disable=protected-access
1107 # D-10 TP-extend-EP: expert weights carry no TP shard — the
1108 # fused dim stays complete per rank, chunk is correct.
1109 continue
1110 for param_name, named in (spec.params or {}).items():
1111 if not isinstance(named, dict):
1112 continue
1113 if ShardingPlanner._FUSED_EXPERT_WEIGHT_RE.match(param_name) is None:
1114 continue
1115 if isinstance(named.get(TP), Shard):
1116 offenders.append((fqn, param_name, named[TP]))
1117 if not offenders:
1118 return
1119 listing = "\n".join(
1120 f" - {fqn}.{param_name} → {{TP: {placement}, ...}}"
1121 for fqn, param_name, placement in offenders[:8])
1122 more = (f"\n ... and {len(offenders) - 8} more"
1123 if len(offenders) > 8 else "")
1124 boundary_fqn, example_param, _ = offenders[0]
1125 raise ValueError(
1126 f"Detected {len(offenders)} fused MoE expert weight(s) (gate/up "
1127 "merged into a single parameter, chunk-split inside forward) "
1128 "declared with TP sharding:\n"
1129 f"{listing}{more}\n"
1130 "This pattern is incompatible with the framework's contiguous-"
1131 "block TP Shard: after Phase A shards the fused dim contiguously, "
1132 "each rank holds a slice whose gate/up layout is misordered, so "
1133 "the runtime chunk is NOT equivalent to the non-TP semantics "
1134 "(Megatron handles this with chunk-aware sharding, which the "
1135 "framework does not support). Worse, the error is silent: the "
1136 "MoE boundary is local-region black-box hosted in BOTH modes, so "
1137 "production and validate run the same (wrong) local forward and "
1138 "the dual-mode comparison cannot expose the accuracy corruption. "
1139 "Choose one of:\n"
1140 " (1) Enable ep_size (recommended): "
1141 "ShardingPlanner().plan(..., ep_size=N) with N>1, N dividing the "
1142 "dense region dp*cp*tp and dividing num_experts. Enabling ep_size "
1143 "takes the TP-extend-EP path (design revision D-10): the dense "
1144 "region is re-partitioned into the derived expert mesh "
1145 "(edp_shard, ep), and expert weights are sharded ONLY as "
1146 "{EP: Shard(0)} along the expert dim with NO TP key — every rank "
1147 "holds complete experts, so the gate/up produced by the runtime "
1148 "chunk matches the non-parallel semantics. Note: the MoE "
1149 "boundary must then also declare local_compute_fn explicitly "
1150 "(EP compute injection; the apply-time preflight checks this and "
1151 "prints a YAML example).\n"
1152 " (2) Keep ep=1: override the fused weight to TP Replicate via "
1153 "plan_overrides (it then opts out of TP sharding and saves no "
1154 "memory for that weight), e.g.:\n"
1155 " ShardingPlanner(plan_overrides={\n"
1156 f" {boundary_fqn!r}: ModuleShardingSpec(\n"
1157 " params={\n"
1158 f" {example_param!r}: "
1159 "{TP: Replicate(), CP: Replicate()},\n"
1160 " # ...declare ALL params of this boundary too "
1161 "(experts.down_proj / gate.weight etc.) — params merge replaces "
1162 "the field wholesale (no per-key merge)\n"
1163 " },\n"
1164 " # out_src TP must ALSO be changed from the template-"
1165 "derived Partial to Replicate: with replicated weights each "
1166 "rank's MoE output is already complete (not Partial); keeping "
1167 "Partial would make the boundary exit reduction count the "
1168 "output tp_size times\n"
1169 " out_src={'output': {TP: Replicate(), CP: Replicate()}},"
1170 " # keep the template-derived CP value (Shard(1) when "
1171 "sequence_parallel is on)\n"
1172 " ),\n"
1173 " })\n"
1174 " (3) Switch to separate w1/w3 expert weights (gate_proj and "
1175 "up_proj as independent parameters) — natively supported, no "
1176 "override needed.")
1178 @staticmethod
1179 def _should_defer_rowwise_bias(
1180 module_fqn: str,
1181 module: nn.Module,
1182 owner_path: str,
1183 param_name: str,
1184 bias_tp,
1185 out_src,
1186 partial_outputs: List[str],
1187 ) -> bool:
1188 """Validate a rowwise bias and return whether it must be deferred."""
1189 if not partial_outputs:
1190 return False
1191 if len(out_src) != 1:
1192 raise ValueError(
1193 f"boundary {module_fqn!r}: rowwise bias deferral (D-22) v1 only supports single-output "
1194 f"boundaries — this boundary's out_src declares {len(out_src)} outputs with a TP Partial "
1195 f"reduction, so the framework cannot attribute {param_name!r} to a unique output. Take over "
1196 "the region with local_compute_fn (add the bias yourself after the reduction)"
1197 )
1198 if bias_tp is not None and not isinstance(bias_tp, Replicate):
1199 raise ValueError(
1200 f"boundary {module_fqn!r}: the bias of rowwise Linear {owner_path!r} declares a non-Replicate "
1201 f"TP placement ({bias_tp!r}) — D-22 deferred addition requires the bias to stay Replicate "
1202 f"(added exactly once as a whole after the TP reduction). Remove {param_name!r} from "
1203 "spec.params, or change it to {TP: replicate()}"
1204 )
1205 owner = module.get_submodule(owner_path) if owner_path else module
1206 if isinstance(owner, nn.Linear):
1207 return True
1208 logger.warning(
1209 "boundary %s: rowwise Linear %r carries a bias and the boundary out_src is TP Partial — but "
1210 "the owner type is %s (not nn.Linear), so the framework does not touch its forward semantics: "
1211 "the bias will be counted multiple times by the Partial reduction (production output = correct "
1212 "value + tp_size × bias). Move the bias after the boundary communication, switch to nn.Linear, "
1213 "or take over the region with local_compute_fn",
1214 module_fqn, owner_path, type(owner).__name__,
1215 )
1216 return False
1218 @staticmethod
1219 def _validate_colwise_bias(
1220 module_fqn: str,
1221 weight_path: str,
1222 param_name: str,
1223 param,
1224 bias_named,
1225 bias_tp,
1226 shard_dim: int,
1227 ) -> None:
1228 """Require a colwise bias to follow its weight's output shard."""
1229 bias_dim = None
1230 if isinstance(bias_tp, Shard):
1231 bias_dim = bias_tp.dim if bias_tp.dim >= 0 else bias_tp.dim + param.ndim
1232 if bias_dim == shard_dim:
1233 return
1234 declared = repr(bias_tp) if bias_named else "undeclared"
1235 raise ValueError(
1236 f"boundary {module_fqn!r}: {weight_path!r} is sharded along the output dim as Shard({shard_dim}), "
1237 f"but {param_name!r} is not sharded along the output channels the same way ({declared}) — "
1238 f"template mismatch (typical: lm_head.bias). Declare it explicitly via plan_overrides as "
1239 f"{{'{param_name}': {{TP: shard({shard_dim})}}}}, or remove the bias"
1240 )
1242 @staticmethod
1243 def _bias_owner_path(param_name: str) -> Optional[str]:
1244 """Return a bias owner's module path, or ``None`` for other params."""
1245 if param_name == "bias":
1246 return ""
1247 if param_name.endswith(".bias"):
1248 return param_name[: -len(".bias")]
1249 return None
1251 @staticmethod
1252 def _collect_deferred_biases(module_fqn: str, spec: ModuleShardingSpec, module: nn.Module) -> List[str]:
1253 """Validate one boundary's biases and collect rowwise deferred names."""
1254 named_params = dict(module.named_parameters())
1255 out_src = spec.out_src or {}
1256 partial_outputs = [
1257 out_name for out_name, named in out_src.items() if isinstance(named.get(TP), Partial)
1258 ]
1259 deferred: List[str] = []
1260 for param_name, param in named_params.items():
1261 owner_path = ShardingPlanner._bias_owner_path(param_name)
1262 if owner_path is None:
1263 continue
1264 weight_path = f"{owner_path}.weight" if owner_path else "weight"
1265 weight = named_params.get(weight_path)
1266 tp_placement = (spec.params or {}).get(weight_path)
1267 if weight is None or tp_placement is None:
1268 continue
1269 tp_placement = tp_placement.get(TP)
1270 if not isinstance(tp_placement, Shard):
1271 continue
1272 shard_dim = tp_placement.dim if tp_placement.dim >= 0 else tp_placement.dim + weight.ndim
1273 bias_tp = spec.params.get(param_name)
1274 bias_tp = bias_tp.get(TP) if bias_tp else None
1275 if shard_dim == weight.ndim - 1:
1276 if ShardingPlanner._should_defer_rowwise_bias(
1277 module_fqn, module, owner_path, param_name, bias_tp, out_src, partial_outputs
1278 ):
1279 deferred.append(param_name)
1280 else:
1281 ShardingPlanner._validate_colwise_bias(
1282 module_fqn, weight_path, param_name, param, spec.params.get(param_name), bias_tp, shard_dim
1283 )
1284 return deferred
1286 @staticmethod
1287 def _finalize_deferred_biases(
1288 plan: ShardingPlan, model, mesh_dim_names: Tuple[str, ...],
1289 ) -> None:
1290 """D-22: decide which Linear biases are deferred to after the boundary
1291 exit TP reduction, and validate every bias against its sibling weight's
1292 sharding direction.
1294 Runs AFTER all plan_overrides are merged (next to
1295 ``_finalize_tp_local_attr_plans``) and anchors detection on the FINAL
1296 spec declarations plus the model structure — never on ParamRole — so
1297 derived / merge / insert / derive=False specs (and family sharding-rule
1298 naming like ``wo``/``c_proj``) all share one code path.
1300 Per boundary, for every physical ``X.bias`` parameter (scanned from
1301 the module, regardless of whether spec.params declares it — a
1302 physically present bias is fused by F.linear either way):
1304 - sibling ``X.weight`` TP placement is ``Shard(weight.ndim - 1)``
1305 (contraction-dim = rowwise) AND the boundary out_src reduces on TP
1306 (Partial) → the fused bias would be counted once per TP rank by the
1307 exit reduction → defer it (bias stays Replicate, added exactly once
1308 after the reduction). Fail-fast guards: a declared non-Replicate
1309 bias placement is rejected; a non-nn.Linear owner gets a WARNING
1310 and is skipped (its forward semantics are not touched); a
1311 multi-output boundary cannot attribute the bias to one output →
1312 fail-fast (use local_compute_fn).
1313 - sibling ``X.weight`` TP placement is an output-dim ``Shard(d)``
1314 (colwise / lm_head / embed) → the bias must follow the same output
1315 shard; Replicate / undeclared / wrong-dim → plan-time "template
1316 mismatch"
1317 error (typical: lm_head.bias), instead of a remote runtime
1318 broadcast crash.
1319 - out_src without a TP Partial (no boundary reduction, or the user
1320 reduces inside the region) → the bias is already added exactly once
1321 → no defer, no check.
1322 """
1323 has_tp = "tp" in mesh_dim_names
1324 modules = dict(model.named_modules())
1325 for module_fqn, spec in plan.modules.items():
1326 spec._deferred_bias_params = () # pylint: disable=protected-access
1327 if not has_tp or not spec.is_boundary:
1328 continue
1329 module = modules.get(module_fqn)
1330 if module is None:
1331 continue
1332 deferred = ShardingPlanner._collect_deferred_biases(module_fqn, spec, module)
1333 spec._deferred_bias_params = tuple(deferred) # pylint: disable=protected-access
1334 if deferred:
1335 logger.info(
1336 "boundary %s: deferred bias (D-22, added exactly once "
1337 "after the TP reduction): %s",
1338 module_fqn, list(deferred))
1340 @staticmethod
1341 def _check_param_uniqueness(plan: ShardingPlan) -> None:
1342 """D-14 invariant 1 (05 §13.3): every parameter is sharded by exactly
1343 one boundary. spec.params keys are resolved to full parameter FQNs
1344 (relative to the boundary module); any parameter declared by ≥2 specs
1345 fails fast (double sharding corrupts silently under production)."""
1346 seen: Dict[str, str] = {} # full param fqn -> first declaring boundary
1347 for fqn, spec in plan.modules.items():
1348 prefix = fqn + "." if fqn else ""
1349 for pname in spec.params:
1350 full = prefix + pname
1351 if full in seen:
1352 raise ValueError(
1353 f"param {full!r} is declared by two boundaries: "
1354 f"{seen[full]!r} and {fqn!r} — each parameter must be "
1355 f"sharded by exactly one boundary (D-14, 05 §13.3); "
1356 f"drop it from one of the specs (an outer boundary "
1357 f"may only declare params of its own/intermediate "
1358 f"modules, never of a nested boundary's subtree)"
1359 )
1360 seen[full] = fqn
1362 @staticmethod
1363 def _check_full_declaration(plan: ShardingPlan) -> None:
1364 """D-14 (05 §13.2): chain fill is removed — every boundary spec must
1365 fully declare its I/O contract. A non-empty in_dst with an empty
1366 in_src fails fast (the previous Scenario-1 fill no longer exists)."""
1367 for fqn, spec in plan.modules.items():
1368 if spec.is_boundary and spec.in_dst and not spec.in_src:
1369 raise ValueError(
1370 f"boundary {fqn!r} declares in_dst but an empty in_src — "
1371 f"chain fill was removed (D-14, 05 §13.2); declare in_src "
1372 f"explicitly (keys must mirror in_dst)"
1373 )
1375 # ── F4 plan-time lints (accuracy_fix_plan.md §2) ─────────────────────
1377 @staticmethod
1378 def _check_param_shard_divisibility(
1379 fqn: str,
1380 prefix: str,
1381 pname: str,
1382 placement: Dict[str, Any],
1383 param_shapes: Dict[str, Tuple[int, ...]],
1384 ep_stack: Dict[str, List[str]],
1385 axis_sizes: Dict[str, int],
1386 ) -> None:
1387 """Check every sharded mesh axis for one declared parameter."""
1388 full = prefix + pname
1389 shape = param_shapes.get(full)
1390 if shape is None and pname in ep_stack:
1391 sources = ep_stack[pname]
1392 shape = param_shapes.get(prefix + sources[0]) if sources else None
1393 if shape is not None:
1394 shape = (len(sources), *shape)
1395 if shape is None:
1396 return # created later (inner_target factory) — nothing to check
1397 for axis, p in (placement or {}).items():
1398 if not isinstance(p, Shard):
1399 continue
1400 size = axis_sizes.get(axis, 1)
1401 if size <= 1:
1402 continue
1403 axis_name = getattr(axis, "value", axis) # MeshAxisName → "tp"
1404 dim = p.dim + len(shape) if p.dim < 0 else p.dim
1405 if dim >= len(shape):
1406 raise ValueError(
1407 f"plan-time shard check failed: {full!r} has shape "
1408 f"{tuple(shape)} but boundary {fqn!r} declares "
1409 f"{{{axis_name}: Shard({p.dim})}} — dim {p.dim} is out "
1410 f"of range for a {len(shape)}D parameter; fix the "
1411 f"plan_overrides declaration"
1412 )
1413 if shape[dim] % size != 0:
1414 raise ValueError(
1415 f"plan-time shard check failed: {full!r} has shape "
1416 f"{tuple(shape)} but boundary {fqn!r} declares "
1417 f"{{{axis_name}: Shard({p.dim})}} — shape[{dim}]={shape[dim]} "
1418 f"is not divisible by {axis_name} size {size} (it would "
1419 f"produce empty shards at apply time). This is most "
1420 f"often a parameter-classification error (e.g. a "
1421 f"replicated/gate parameter misclassified into a "
1422 f"sharded role — see accuracy_fix_plan.md §2): fix "
1423 f"the naming rule / family sharding_rules entry, or correct "
1424 f"the plan_overrides declaration"
1425 )
1427 @staticmethod
1428 def _check_shard_divisibility(
1429 plan: ShardingPlan,
1430 param_shapes: Dict[str, Tuple[int, ...]],
1431 *,
1432 tp_size: int,
1433 cp_size: int,
1434 ep_size: int,
1435 ) -> None:
1436 """F4a: every Shard(dim) in every spec.params must divide the
1437 parameter's shape along that dim — fail at plan time with a teaching
1438 error instead of producing an empty shard at apply time
1439 (accuracy_problem.md 10.1: a (1, 64) scalar-gate weight misclassified
1440 SHARED_EXPERT was Shard(0)'d over tp=2 into a (0, 64) empty shard,
1441 surfacing only much later at weight reconstruction).
1443 Runs after the Phase 4.5 override merge, so hand-written specs are
1444 checked too. Stacked expert params (D-09/D-10 ``_ep_stack``) resolve
1445 their shape as ``(num_experts, *source_shape)``; D-10 experts use
1446 ``spec._ep_size`` as the EP axis size. Params not resolvable at plan
1447 time (e.g. created later by an inner_target factory) are skipped.
1448 """
1449 for fqn, spec in plan.modules.items():
1450 prefix = fqn + "." if fqn else ""
1451 ep_stack = getattr(spec, "_ep_stack", None) or {}
1452 # D-10: the EP axis lives on the derived expert mesh whose size
1453 # is spec._ep_size; old-style EP uses the mesh "ep" axis (ep_size).
1454 axis_sizes = {
1455 "tp": tp_size,
1456 "cp": cp_size,
1457 "ep": getattr(spec, "_ep_size", 0) or ep_size,
1458 }
1459 for pname, placement in (spec.params or {}).items():
1460 ShardingPlanner._check_param_shard_divisibility(
1461 fqn, prefix, pname, placement, param_shapes, ep_stack, axis_sizes
1462 )
1464 @staticmethod
1465 def _declared_parameter_names(plan: ShardingPlan) -> set[str]:
1466 """Collect full names covered by placements and special handlers."""
1467 covered = set()
1468 for fqn, spec in plan.modules.items():
1469 prefix = fqn + "." if fqn else ""
1470 for pname in (spec.params or {}):
1471 covered.add(prefix + pname)
1472 for sources in (getattr(spec, "_ep_stack", None) or {}).values():
1473 covered.update(prefix + source for source in sources)
1474 covered.update(plan.special_handlers or {})
1475 return covered
1477 def _check_all_trainable_params_covered(self, plan: ShardingPlan, model) -> None:
1478 """F4b: every ``requires_grad=True`` parameter must appear in some
1479 spec.params (resolved to a full FQN) or in special_handlers —
1480 otherwise its gradient-sync semantics would be decided SILENTLY by
1481 the consumer-side default (the same class of shape-legal/silent-wrong
1482 hazard as accuracy_problem.md 10.1/10.2), so the plan must be a
1483 complete declaration of every trainable parameter's layout.
1485 Covered = spec.params ∪ _ep_stack sources ∪ special_handlers (a
1486 param explicitly declared via plan_overrides therefore counts as
1487 covered). Remediation paths, listed per parameter in the error:
1488 ① classify it explicitly (naming rule / the family's sharding_rules, e.g.
1489 shared_expert_gate → REPLICATED);
1490 ② declare it in a spec via plan_overrides (e.g. all-Replicate);
1491 ③ mark it SPECIAL (special_handlers) or freeze it
1492 (requires_grad=False).
1493 Escape hatch for exploratory debugging:
1494 ``ShardingPlanner(allow_uncovered_params=True)`` downgrades to a
1495 warning. Skipped entirely under derive=False (pure declaration
1496 assembler for subtrees — coverage is the caller's responsibility).
1497 """
1498 if not self._derive:
1499 return
1500 covered = self._declared_parameter_names(plan)
1501 covered.update(plan.owned_parameter_names)
1502 uncovered = [
1503 name for name, p in model.named_parameters()
1504 if p.requires_grad and name not in covered
1505 ]
1506 if not uncovered:
1507 return
1508 lines = "\n".join(f" - {n}" for n in uncovered[:20])
1509 more = (f" ... and {len(uncovered) - 20} more\n"
1510 if len(uncovered) > 20 else "")
1511 msg = (
1512 f"plan-time coverage check failed: {len(uncovered)} trainable "
1513 f"parameter(s) are not covered by any spec.params / "
1514 f"special_handlers:\n{lines}\n{more}"
1515 "An uncovered parameter is never sharded (kept replicated) AND is "
1516 "absent from source_shard_info — its gradient-sync semantics would be "
1517 "decided silently by the consumer-side default. Resolve each "
1518 "parameter explicitly: ① add a naming rule / a family sharding_rules "
1519 "entry "
1520 "(e.g. shared_expert_gate → REPLICATED); ② declare it in a spec "
1521 "via plan_overrides; ③ mark it SPECIAL or freeze it "
1522 "(requires_grad=False). For exploratory debugging only, construct "
1523 "ShardingPlanner(allow_uncovered_params=True) to downgrade this "
1524 "error to a warning (accuracy_fix_plan.md §2 F4b)"
1525 )
1526 if self._allow_uncovered_params:
1527 logger.warning("%s", msg)
1528 else:
1529 raise ValueError(msg)
1531 # ── Phase 5 ─────────────────────────────────────────────────────────
1533 def _mark_terminal(self, plan: ShardingPlan, model) -> ShardingPlan:
1534 """D-14 (05 §13.2): compile-time chain propagation/validation is
1535 removed; only ``_is_terminal`` marking is kept — the last boundary
1536 in forward order (each module vouches for its own propagation in
1537 validate mode; adjacent-contract checks no longer exist)."""
1538 boundaries = [fqn for fqn, spec in plan.modules.items() if spec.is_boundary]
1539 sorted_fqns = self._topological_sort_by_forward_order(boundaries, model)
1540 terminal = sorted_fqns[-1] if sorted_fqns else None
1541 for fqn, spec in plan.modules.items():
1542 # The planner owns the spec DSL internals.
1543 spec._is_terminal = fqn == terminal # pylint: disable=protected-access
1544 return plan
1546 @staticmethod
1547 def _topological_sort_by_forward_order(fqns: List[str], model) -> List[str]:
1548 """Sort by named_modules registration order; unmatched FQNs are
1549 appended at the end with a warning."""
1550 fqn_set = set(fqns)
1551 ordered: List[str] = []
1552 seen: set = set()
1553 for name, _ in model.named_modules():
1554 if name in fqn_set and name not in seen:
1555 ordered.append(name)
1556 seen.add(name)
1557 missing = fqn_set - seen
1558 if missing:
1559 logger.warning(
1560 "_topological_sort_by_forward_order: %d FQNs not found in "
1561 "named_modules; appended at the end: %s",
1562 len(missing), sorted(missing)[:5],
1563 )
1564 ordered.extend(sorted(missing))
1565 return ordered
1567 # ── tied weights ────────────────────────────────────────────────────
1569 @staticmethod
1570 def _detect_tied_pairs(model) -> List[Tuple[str, str]]:
1571 """Detect the embed_tokens.weight <-> lm_head.weight tied pair.
1573 With HF tie_word_embeddings both ends share storage; in PP
1574 scenarios the two ends cannot be detected across stages, so the
1575 user must declare plan.tied_pairs explicitly (05
1576 detect_tied_weights comment).
1577 """
1578 if not getattr(getattr(model, "config", None), "tie_word_embeddings", False):
1579 return []
1580 embed_fqn = lm_head_fqn = None
1581 # remove_duplicate=False: under the default deduplication of
1582 # named_parameters a tied parameter appears only once.
1583 for name, _ in model.named_parameters(remove_duplicate=False):
1584 if name.endswith("embed_tokens.weight"):
1585 embed_fqn = name
1586 elif name.endswith("lm_head.weight"):
1587 lm_head_fqn = name
1588 if embed_fqn and lm_head_fqn:
1589 return [(embed_fqn, lm_head_fqn)]
1590 return []
1593def _validate_tp_model_compatibility(config: Any, tp_size: int, ep_size: int) -> None:
1594 """Check model head and expert width divisibility on the TP axis."""
1595 heads = getattr(config, "num_attention_heads", None)
1596 if heads is not None and heads % tp_size != 0:
1597 raise ValueError(
1598 f"num_attention_heads ({heads}) must be divisible by TP ({tp_size})"
1599 )
1600 kv_heads = getattr(config, "num_key_value_heads", None)
1601 if kv_heads is not None and kv_heads % tp_size != 0:
1602 raise ValueError(
1603 f"num_key_value_heads ({kv_heads}) must be divisible by TP ({tp_size})"
1604 )
1605 moe_inter = getattr(config, "moe_intermediate_size", None)
1606 if ep_size > 1 and moe_inter is not None and moe_inter % tp_size != 0:
1607 raise ValueError(
1608 f"moe_intermediate_size ({moe_inter}) must be divisible by TP ({tp_size})"
1609 )
1612def _validate_ep_model_compatibility(config: Any, ep_size: int) -> None:
1613 """Check that the model has a divisible routed-expert count."""
1614 num_experts = (getattr(config, "num_experts", None)
1615 or getattr(config, "n_routed_experts", None) or 0)
1616 if num_experts <= 0:
1617 raise ValueError("EP>1 requires MoE model (num_experts > 0)")
1618 if num_experts % ep_size != 0:
1619 raise ValueError(
1620 f"num_experts ({num_experts}) must be divisible by EP ({ep_size})"
1621 )
1624def validate_model_compatibility(
1625 model: Any, *, tp_size: int = 1, cp_size: int = 1, ep_size: int = 1,
1626 seq_len: Optional[int] = None,
1627) -> None:
1628 """Validate model config against requested TP, CP, and EP degrees.
1630 Args:
1631 model: Model whose configuration declares attention and expert sizes.
1632 tp_size: Requested decoder tensor-parallel degree.
1633 cp_size: Requested decoder context-parallel degree.
1634 ep_size: Requested expert-parallel degree.
1635 seq_len: Optional global decoder token length.
1636 """
1637 config = getattr(model, "config", None)
1638 if config is None:
1639 return
1641 if tp_size > 1:
1642 _validate_tp_model_compatibility(config, tp_size, ep_size)
1644 if cp_size > 1 and seq_len is not None and seq_len % (cp_size * 2) != 0:
1645 raise ValueError(
1646 f"seq_len ({seq_len}) must be divisible by 2*cp ({2 * cp_size})"
1647 )
1649 if ep_size > 1:
1650 _validate_ep_model_compatibility(config, ep_size)