Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / mxfp8_grouped_swiglu.py: 83%

75 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""MXFP8 local computation with optional fusion and the GroupedExperts contract.""" 

16 

17from __future__ import annotations 

18 

19import math 

20from collections.abc import Mapping 

21from typing import Any 

22 

23import torch 

24from torch import nn 

25from torch.nn import functional as F 

26from transformers.activations import ACT2FN 

27from transformers.models.deepseek_v4.modeling_deepseek_v4 import DeepseekV4Experts 

28 

29from hyper_parallel.components.modules.grouped_experts import GroupedExperts 

30from hyper_parallel.components.quantization.functional.mxfp8_fused_grouped_swiglu import ( 

31 fused_grouped_swiglu, 

32) 

33from hyper_parallel.components.quantization.functional.mxfp8_projection import mxfp8_projection 

34from hyper_parallel.components.quantization.ops.npu_mxfp8 import ( 

35 validate_npu_gmm_runtime, validate_npu_swiglu_runtime, 

36) 

37from hyper_parallel.components.quantization.quantizers.mxfp8 import MXFP8Quantizer 

38from hyper_parallel.models.replacement import module_replacement 

39 

40 

41@module_replacement 

42class MXFP8GroupedSwiGLU(GroupedExperts): 

43 """Retain routing and checkpoint conversion; replace only local expert math. 

44 

45 Persistent weights remain [E,H,2I] and [E,I,H]. Counts name tokens per 

46 local expert, including zero-token experts. Output retains expert-major 

47 order. Scores are accepted by the inherited protocol; outer routing applies 

48 them exactly once after GMM2, with its original dtype policy. 

49 

50 Note: 

51 EP recipes must explicitly set ``use_grouped_gemm: true`` to call this 

52 module's MXFP8 computation. EP binding rejects a missing or false option 

53 because the eager path does not support this module's weight layout. 

54 """ 

55 

56 requires_ep_grouped_gemm = True 

57 

58 def __init__(self, *, module: nn.Module, module_fqn: str = "", 

59 context: Mapping[str, Any] | None = None) -> None: 

60 """Validate the supported structure and initialize a non-fused variant.""" 

61 context = context or {} 

62 self.module_fqn = module_fqn 

63 prefix = f"{module_fqn or '<root>'}: MXFP8GroupedSwiGLU" 

64 if any(context.get(axis) for axis in ("tp", "cp", "pp")): 

65 raise NotImplementedError(f"{prefix} requires TP=CP=PP=1") 

66 try: 

67 super().__init__(module=module, module_fqn=module_fqn, context=context) 

68 except (ValueError, TypeError, KeyError) as exc: 

69 raise ValueError(f"{prefix}: {exc}") from exc 

70 self._validate_source_contract(module, prefix) 

71 try: 

72 validate_npu_gmm_runtime() 

73 except RuntimeError as exc: 

74 raise RuntimeError(f"{prefix}: {exc}") from exc 

75 self.quantizer = MXFP8Quantizer() 

76 

77 def _validate_source_contract(self, module: nn.Module, prefix: str) -> None: 

78 """Check the initial layout and activation before any device computation.""" 

79 hidden_act = (getattr(self.config, "hidden_act", None) 

80 or getattr(self.config, "hidden_activation", None) 

81 or getattr(self.config, "mlp_hidden_act", "silu")) 

82 source_activation = getattr(module, "act_fn", None) 

83 if (not self.has_gate or self.add_bias or hidden_act not in ("silu", "swiglu") 

84 or (source_activation is not None 

85 and not isinstance(source_activation, (nn.SiLU, type(ACT2FN["silu"]))))): 

86 raise ValueError(f"{prefix} requires standard bias-free SwiGLU") 

87 if self.use_2d_experts or not self.router_gating_in_fp32: 

88 raise ValueError(f"{prefix} requires 3D packed weights and routing weights after GMM2") 

89 if self.hidden_size % 32 or self.intermediate_size % 32: 

90 raise ValueError(f"{prefix} requires H and I divisible by 32") 

91 

92 def _grouped_gemm_expert_forward(self, gate_up_proj, down_proj, permuted, 

93 tokens_per_expert, permuted_probs): 

94 """Validate expert groups and dispatch local math without applying scores.""" 

95 del permuted_probs 

96 counts = tokens_per_expert.to(device=permuted.device) 

97 if counts.ndim != 1 or counts.shape[0] != gate_up_proj.shape[0]: 

98 raise ValueError(f"{self.module_fqn}: counts must contain one count per local expert") 

99 if counts.dtype not in (torch.int32, torch.int64): 

100 raise ValueError(f"{self.module_fqn}: counts must be integers") 

101 # Device counts come from routing; avoid copying them to the host per GMM. 

102 # CPU callers can validate values before handing counts to device kernels. 

103 if tokens_per_expert.device.type == "cpu": 

104 if bool((tokens_per_expert < 0).any()) or int(tokens_per_expert.sum()) != permuted.shape[0]: 

105 raise ValueError(f"{self.module_fqn}: counts must be non-negative and sum to token count") 

106 return self._local_compute(permuted, gate_up_proj, down_proj, counts) 

107 

108 def _local_compute(self, permuted, gate_up_proj, down_proj, counts): 

109 """Run the non-fused local chain after common protocol validation.""" 

110 projected = mxfp8_projection(permuted, gate_up_proj, counts, self.quantizer) 

111 gate, up = projected.chunk(2, dim=-1) 

112 activated = F.silu(gate) * up 

113 return mxfp8_projection(activated, down_proj, counts, self.quantizer) 

114 

115 

116@module_replacement 

117class FusedMXFP8GroupedSwiGLU(MXFP8GroupedSwiGLU): 

118 """Use fused activation/quantization while retaining the same family contract.""" 

119 

120 def __init__(self, *, module: nn.Module, module_fqn: str = "", 

121 context: Mapping[str, Any] | None = None) -> None: 

122 """Validate both fused kernels during construction, with module diagnostics.""" 

123 self.clamp_limit = float(module.limit) if isinstance(module, DeepseekV4Experts) else None 

124 if self.clamp_limit is not None and (not math.isfinite(self.clamp_limit) or self.clamp_limit <= 0): 

125 raise ValueError(f"{module_fqn or '<root>'}: clipped MXFP8 fusion requires a positive finite limit") 

126 super().__init__(module=module, module_fqn=module_fqn, context=context) 

127 try: 

128 validate_npu_swiglu_runtime(clamp_limit=self.clamp_limit) 

129 except RuntimeError as exc: 

130 raise RuntimeError(f"{module_fqn or '<root>'}: {exc}") from exc 

131 

132 @classmethod 

133 def _activation(cls, module: nn.Module, config: Any, gated_linear_unit: bool) -> Any: 

134 """Accept DeepSeek V4/V4.1 clipped experts while keeping the custom-gate guard for others.""" 

135 return (None if isinstance(module, DeepseekV4Experts) 

136 else super()._activation(module, config, gated_linear_unit)) 

137 

138 def _local_compute(self, permuted, gate_up_proj, down_proj, counts): 

139 """Use one autograd boundary without passing quantized carriers between nodes.""" 

140 return fused_grouped_swiglu( 

141 permuted, gate_up_proj, down_proj, counts, self.quantizer, clamp_limit=self.clamp_limit)