Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / quantization / functional / mxfp8_fused_grouped_swiglu.py: 21%

75 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Fused SwiGLU/quantization within one grouped-expert autograd boundary.""" 

16 

17from typing import TYPE_CHECKING, Optional 

18 

19import torch 

20from torch.autograd.function import once_differentiable 

21 

22from hyper_parallel.components.quantization.ops.npu_mxfp8 import mxfp8_grouped_matmul 

23from hyper_parallel.components.quantization.tensor import MXFP8Tensor 

24 

25if TYPE_CHECKING: 

26 from hyper_parallel.components.quantization.quantizers.mxfp8 import MXFP8Quantizer 

27 

28 

29def _save_operands(ctx, counts, projected, operands): 

30 """Expose physical tensors to saved-tensor hooks, never carrier wrappers.""" 

31 ctx.operand_metadata = tuple((tuple(q.shape), q.dtype, q.quantizer) for q in operands) 

32 buffers = tuple(buffer for q in operands for buffer in (q.row_data, q.row_scale, q.col_data, q.col_scale)) 

33 ctx.save_for_backward(counts, projected, *buffers) 

34 

35 

36def _restore_operands(ctx): 

37 """Use the same physical-buffer reconstruction as the upstream Dense/GMM path.""" 

38 counts, projected, *buffers = ctx.saved_tensors 

39 operands = [] 

40 for index, (shape, dtype, quantizer) in enumerate(ctx.operand_metadata): 

41 row, row_scale, col, col_scale = buffers[4 * index:4 * index + 4] 

42 operands.append(MXFP8Tensor(shape, dtype, quantizer=quantizer, row_data=row, 

43 row_scale=row_scale, col_data=col, col_scale=col_scale)) 

44 return counts, projected, operands 

45 

46 

47def _fused_quant(projected, quantizer, counts, gradient=None, clamp_limit=None): 

48 """Keep prequantized operands internal to this Function's GMM calls.""" 

49 options = { 

50 "quant_dtype": quantizer.quant_dtype, 

51 "group_index": counts.cumsum(0).to(torch.int64), 

52 "clamp_limit": clamp_limit, 

53 } 

54 if gradient is None: 

55 buffers = quantizer.npu_ops.swiglu_mx_quant_dual_axis(projected, **options) 

56 shape = (projected.shape[0], projected.shape[1] // 2) 

57 else: 

58 buffers = quantizer.npu_ops.swiglu_backward_mx_quant_dual_axis( 

59 projected, gradient.contiguous(), **options) 

60 shape = projected.shape 

61 row, row_scale, col, col_scale = buffers 

62 return MXFP8Tensor(shape, projected.dtype, quantizer=quantizer, 

63 row_data=row, row_scale=row_scale, col_data=col, col_scale=col_scale) 

64 

65 

66def _gmm(left, right, counts, dtype, layout="NN"): 

67 """Use the existing MXFP8 adapter and count-based expert partitioning.""" 

68 return mxfp8_grouped_matmul(left, right, layout=layout, group_list=counts, 

69 group_type=2 if layout == "TN" else 0, 

70 group_list_type=1, output_dtype=dtype) 

71 

72 

73class _FusedGroupedSwiGLU(torch.autograd.Function): 

74 """Own both GMMs and fusion so external autograd edges remain ordinary tensors.""" 

75 

76 @staticmethod 

77 def forward( 

78 ctx: torch.autograd.function.FunctionCtx, inputs: torch.Tensor, 

79 gate_up: torch.Tensor, down: torch.Tensor, counts: torch.Tensor, quantizer: "MXFP8Quantizer", 

80 clamp_limit: Optional[float], 

81 ) -> torch.Tensor: 

82 """Compute two MXFP8 GMMs with a fused activation/quantization boundary. 

83 

84 Args: 

85 ctx: Autograd save context. 

86 inputs: Expert-major token matrix. 

87 gate_up: Canonical gate/up weights. 

88 down: Canonical down weights. 

89 counts: Per-expert token counts. 

90 quantizer: MXFP8 quantizer. 

91 clamp_limit: Optional positive clipping limit for DeepSeek experts. 

92 """ 

93 ctx.shapes = (inputs.shape, gate_up.shape, down.shape) 

94 ctx.dtype, ctx.device, ctx.quantizer = inputs.dtype, inputs.device, quantizer 

95 ctx.clamp_limit = clamp_limit 

96 if inputs.shape[0] == 0: 

97 ctx.save_for_backward() 

98 return inputs.new_empty((0, down.shape[-1])) 

99 input_q = quantizer.quantize(inputs, group_list=counts, group_list_type=1, rowwise=True, colwise=True) 

100 gate_up_q = quantizer.quantize(gate_up.contiguous(), rowwise=True, colwise=True) 

101 down_q = quantizer.quantize(down.contiguous(), rowwise=True, colwise=True) 

102 projected = _gmm(input_q, gate_up_q, counts, inputs.dtype) 

103 activated_q = _fused_quant(projected, quantizer, counts, clamp_limit=clamp_limit) 

104 output = _gmm(activated_q, down_q, counts, inputs.dtype) 

105 _save_operands(ctx, counts, projected, (input_q, gate_up_q, activated_q, down_q)) 

106 return output 

107 

108 @staticmethod 

109 @once_differentiable 

110 def backward( 

111 ctx: torch.autograd.function.FunctionCtx, gradient: torch.Tensor, 

112 ) -> tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor], None, None, None]: 

113 """Fuse SwiGLU derivative/quantization and compute requested first gradients. 

114 

115 Args: 

116 ctx: Saved forward context. 

117 gradient: Output gradient. 

118 """ 

119 needs_input, needs_gate, needs_down = ctx.needs_input_grad[:3] 

120 if ctx.shapes[0][0] == 0: 

121 grads = [torch.zeros(shape, device=ctx.device, dtype=ctx.dtype) if needed else None 

122 for shape, needed in zip(ctx.shapes, (needs_input, needs_gate, needs_down))] 

123 return *grads, None, None, None 

124 counts, projected, (input_q, gate_up_q, activated_q, down_q) = _restore_operands(ctx) 

125 grad_q = ctx.quantizer.quantize( 

126 gradient, group_list=counts if needs_down else None, group_list_type=1, 

127 rowwise=needs_input or needs_gate, colwise=needs_down) 

128 grad_down = _gmm(activated_q, grad_q, counts, ctx.dtype, "TN") if needs_down else None 

129 grad_input = grad_gate = None 

130 if needs_input or needs_gate: 

131 grad_activated = _gmm(grad_q, down_q, counts, ctx.dtype, "NT") 

132 grad_projected_q = _fused_quant(projected, ctx.quantizer, counts, grad_activated, ctx.clamp_limit) 

133 if needs_input: 

134 grad_input = _gmm(grad_projected_q, gate_up_q, counts, ctx.dtype, "NT") 

135 if needs_gate: 

136 grad_gate = _gmm(input_q, grad_projected_q, counts, ctx.dtype, "TN") 

137 return grad_input, grad_gate, grad_down, None, None, None 

138 

139 

140def fused_grouped_swiglu( 

141 inputs: torch.Tensor, gate_up: torch.Tensor, down: torch.Tensor, 

142 counts: torch.Tensor, quantizer: "MXFP8Quantizer", 

143 *, clamp_limit: Optional[float] = None, 

144) -> torch.Tensor: 

145 """Compute expert-major outputs with canonical [E,H,2I]/[E,I,H] weights. 

146 

147 Args: 

148 inputs: BF16 expert-major token matrix. 

149 gate_up: Canonical packed gate/up weights. 

150 down: Canonical down weights. 

151 counts: Per-local-expert token counts, including zero-token experts. 

152 quantizer: MXFP8 recipe with both fused NPU kernels available. 

153 clamp_limit: Optional positive clipping limit for gate and up projections. 

154 

155 Returns: 

156 Ordinary BF16 outputs before outer routing weights are applied. 

157 """ 

158 if inputs.dtype not in (torch.bfloat16, torch.float16): 

159 raise ValueError("Fused MXFP8 SwiGLU requires BF16 or FP16 inputs") 

160 if gate_up.dtype != inputs.dtype or down.dtype != inputs.dtype: 

161 raise ValueError("Fused MXFP8 SwiGLU requires matching input and weight dtypes") 

162 if inputs.ndim != 2 or gate_up.ndim != 3 or down.ndim != 3: 

163 raise ValueError("Fused MXFP8 SwiGLU requires 2D inputs and 3D expert weights") 

164 if (gate_up.shape[0] != down.shape[0] or gate_up.shape[1] != inputs.shape[1] 

165 or gate_up.shape[2] != 2 * down.shape[1] or down.shape[2] != inputs.shape[1]): 

166 raise ValueError("Fused MXFP8 SwiGLU weight dimensions do not match inputs") 

167 if counts.ndim != 1 or counts.numel() != gate_up.shape[0] or counts.dtype not in (torch.int32, torch.int64): 

168 raise ValueError("Fused MXFP8 SwiGLU requires one integer count per expert") 

169 return _FusedGroupedSwiGLU.apply(inputs, gate_up, down, counts, quantizer, clamp_limit)