Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / mxfp8_grouped_swiglu.py: 83%
75 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""MXFP8 local computation with optional fusion and the GroupedExperts contract."""
17from __future__ import annotations
19import math
20from collections.abc import Mapping
21from typing import Any
23import torch
24from torch import nn
25from torch.nn import functional as F
26from transformers.activations import ACT2FN
27from transformers.models.deepseek_v4.modeling_deepseek_v4 import DeepseekV4Experts
29from hyper_parallel.components.modules.grouped_experts import GroupedExperts
30from hyper_parallel.components.quantization.functional.mxfp8_fused_grouped_swiglu import (
31 fused_grouped_swiglu,
32)
33from hyper_parallel.components.quantization.functional.mxfp8_projection import mxfp8_projection
34from hyper_parallel.components.quantization.ops.npu_mxfp8 import (
35 validate_npu_gmm_runtime, validate_npu_swiglu_runtime,
36)
37from hyper_parallel.components.quantization.quantizers.mxfp8 import MXFP8Quantizer
38from hyper_parallel.models.replacement import module_replacement
41@module_replacement
42class MXFP8GroupedSwiGLU(GroupedExperts):
43 """Retain routing and checkpoint conversion; replace only local expert math.
45 Persistent weights remain [E,H,2I] and [E,I,H]. Counts name tokens per
46 local expert, including zero-token experts. Output retains expert-major
47 order. Scores are accepted by the inherited protocol; outer routing applies
48 them exactly once after GMM2, with its original dtype policy.
50 Note:
51 EP recipes must explicitly set ``use_grouped_gemm: true`` to call this
52 module's MXFP8 computation. EP binding rejects a missing or false option
53 because the eager path does not support this module's weight layout.
54 """
56 requires_ep_grouped_gemm = True
58 def __init__(self, *, module: nn.Module, module_fqn: str = "",
59 context: Mapping[str, Any] | None = None) -> None:
60 """Validate the supported structure and initialize a non-fused variant."""
61 context = context or {}
62 self.module_fqn = module_fqn
63 prefix = f"{module_fqn or '<root>'}: MXFP8GroupedSwiGLU"
64 if any(context.get(axis) for axis in ("tp", "cp", "pp")):
65 raise NotImplementedError(f"{prefix} requires TP=CP=PP=1")
66 try:
67 super().__init__(module=module, module_fqn=module_fqn, context=context)
68 except (ValueError, TypeError, KeyError) as exc:
69 raise ValueError(f"{prefix}: {exc}") from exc
70 self._validate_source_contract(module, prefix)
71 try:
72 validate_npu_gmm_runtime()
73 except RuntimeError as exc:
74 raise RuntimeError(f"{prefix}: {exc}") from exc
75 self.quantizer = MXFP8Quantizer()
77 def _validate_source_contract(self, module: nn.Module, prefix: str) -> None:
78 """Check the initial layout and activation before any device computation."""
79 hidden_act = (getattr(self.config, "hidden_act", None)
80 or getattr(self.config, "hidden_activation", None)
81 or getattr(self.config, "mlp_hidden_act", "silu"))
82 source_activation = getattr(module, "act_fn", None)
83 if (not self.has_gate or self.add_bias or hidden_act not in ("silu", "swiglu")
84 or (source_activation is not None
85 and not isinstance(source_activation, (nn.SiLU, type(ACT2FN["silu"]))))):
86 raise ValueError(f"{prefix} requires standard bias-free SwiGLU")
87 if self.use_2d_experts or not self.router_gating_in_fp32:
88 raise ValueError(f"{prefix} requires 3D packed weights and routing weights after GMM2")
89 if self.hidden_size % 32 or self.intermediate_size % 32:
90 raise ValueError(f"{prefix} requires H and I divisible by 32")
92 def _grouped_gemm_expert_forward(self, gate_up_proj, down_proj, permuted,
93 tokens_per_expert, permuted_probs):
94 """Validate expert groups and dispatch local math without applying scores."""
95 del permuted_probs
96 counts = tokens_per_expert.to(device=permuted.device)
97 if counts.ndim != 1 or counts.shape[0] != gate_up_proj.shape[0]:
98 raise ValueError(f"{self.module_fqn}: counts must contain one count per local expert")
99 if counts.dtype not in (torch.int32, torch.int64):
100 raise ValueError(f"{self.module_fqn}: counts must be integers")
101 # Device counts come from routing; avoid copying them to the host per GMM.
102 # CPU callers can validate values before handing counts to device kernels.
103 if tokens_per_expert.device.type == "cpu":
104 if bool((tokens_per_expert < 0).any()) or int(tokens_per_expert.sum()) != permuted.shape[0]:
105 raise ValueError(f"{self.module_fqn}: counts must be non-negative and sum to token count")
106 return self._local_compute(permuted, gate_up_proj, down_proj, counts)
108 def _local_compute(self, permuted, gate_up_proj, down_proj, counts):
109 """Run the non-fused local chain after common protocol validation."""
110 projected = mxfp8_projection(permuted, gate_up_proj, counts, self.quantizer)
111 gate, up = projected.chunk(2, dim=-1)
112 activated = F.silu(gate) * up
113 return mxfp8_projection(activated, down_proj, counts, self.quantizer)
116@module_replacement
117class FusedMXFP8GroupedSwiGLU(MXFP8GroupedSwiGLU):
118 """Use fused activation/quantization while retaining the same family contract."""
120 def __init__(self, *, module: nn.Module, module_fqn: str = "",
121 context: Mapping[str, Any] | None = None) -> None:
122 """Validate both fused kernels during construction, with module diagnostics."""
123 self.clamp_limit = float(module.limit) if isinstance(module, DeepseekV4Experts) else None
124 if self.clamp_limit is not None and (not math.isfinite(self.clamp_limit) or self.clamp_limit <= 0):
125 raise ValueError(f"{module_fqn or '<root>'}: clipped MXFP8 fusion requires a positive finite limit")
126 super().__init__(module=module, module_fqn=module_fqn, context=context)
127 try:
128 validate_npu_swiglu_runtime(clamp_limit=self.clamp_limit)
129 except RuntimeError as exc:
130 raise RuntimeError(f"{module_fqn or '<root>'}: {exc}") from exc
132 @classmethod
133 def _activation(cls, module: nn.Module, config: Any, gated_linear_unit: bool) -> Any:
134 """Accept DeepSeek V4/V4.1 clipped experts while keeping the custom-gate guard for others."""
135 return (None if isinstance(module, DeepseekV4Experts)
136 else super()._activation(module, config, gated_linear_unit))
138 def _local_compute(self, permuted, gate_up_proj, down_proj, counts):
139 """Use one autograd boundary without passing quantized carriers between nodes."""
140 return fused_grouped_swiglu(
141 permuted, gate_up_proj, down_proj, counts, self.quantizer, clamp_limit=self.clamp_limit)