1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Fused SwiGLU/quantization within one grouped-expert autograd boundary."""
16
17from typing import TYPE_CHECKING, Optional
18
19import torch
20from torch.autograd.function import once_differentiable
21
22from hyper_parallel.components.quantization.ops.npu_mxfp8 import mxfp8_grouped_matmul
23from hyper_parallel.components.quantization.tensor import MXFP8Tensor
24
25if TYPE_CHECKING:
26 from hyper_parallel.components.quantization.quantizers.mxfp8 import MXFP8Quantizer
27
28
29def _save_operands(ctx, counts, projected, operands):
30 """Expose physical tensors to saved-tensor hooks, never carrier wrappers."""
31 ctx.operand_metadata = tuple((tuple(q.shape), q.dtype, q.quantizer) for q in operands)
32 buffers = tuple(buffer for q in operands for buffer in (q.row_data, q.row_scale, q.col_data, q.col_scale))
33 ctx.save_for_backward(counts, projected, *buffers)
34
35
36def _restore_operands(ctx):
37 """Use the same physical-buffer reconstruction as the upstream Dense/GMM path."""
38 counts, projected, *buffers = ctx.saved_tensors
39 operands = []
40 for index, (shape, dtype, quantizer) in enumerate(ctx.operand_metadata):
41 row, row_scale, col, col_scale = buffers[4 * index:4 * index + 4]
42 operands.append(MXFP8Tensor(shape, dtype, quantizer=quantizer, row_data=row,
43 row_scale=row_scale, col_data=col, col_scale=col_scale))
44 return counts, projected, operands
45
46
47def _fused_quant(projected, quantizer, counts, gradient=None, clamp_limit=None):
48 """Keep prequantized operands internal to this Function's GMM calls."""
49 options = {
50 "quant_dtype": quantizer.quant_dtype,
51 "group_index": counts.cumsum(0).to(torch.int64),
52 "clamp_limit": clamp_limit,
53 }
54 if gradient is None:
55 buffers = quantizer.npu_ops.swiglu_mx_quant_dual_axis(projected, **options)
56 shape = (projected.shape[0], projected.shape[1] // 2)
57 else:
58 buffers = quantizer.npu_ops.swiglu_backward_mx_quant_dual_axis(
59 projected, gradient.contiguous(), **options)
60 shape = projected.shape
61 row, row_scale, col, col_scale = buffers
62 return MXFP8Tensor(shape, projected.dtype, quantizer=quantizer,
63 row_data=row, row_scale=row_scale, col_data=col, col_scale=col_scale)
64
65
66def _gmm(left, right, counts, dtype, layout="NN"):
67 """Use the existing MXFP8 adapter and count-based expert partitioning."""
68 return mxfp8_grouped_matmul(left, right, layout=layout, group_list=counts,
69 group_type=2 if layout == "TN" else 0,
70 group_list_type=1, output_dtype=dtype)
71
72
73class _FusedGroupedSwiGLU(torch.autograd.Function):
74 """Own both GMMs and fusion so external autograd edges remain ordinary tensors."""
75
76 @staticmethod
77 def forward(
78 ctx: torch.autograd.function.FunctionCtx, inputs: torch.Tensor,
79 gate_up: torch.Tensor, down: torch.Tensor, counts: torch.Tensor, quantizer: "MXFP8Quantizer",
80 clamp_limit: Optional[float],
81 ) -> torch.Tensor:
82 """Compute two MXFP8 GMMs with a fused activation/quantization boundary.
83
84 Args:
85 ctx: Autograd save context.
86 inputs: Expert-major token matrix.
87 gate_up: Canonical gate/up weights.
88 down: Canonical down weights.
89 counts: Per-expert token counts.
90 quantizer: MXFP8 quantizer.
91 clamp_limit: Optional positive clipping limit for DeepSeek experts.
92 """
93 ctx.shapes = (inputs.shape, gate_up.shape, down.shape)
94 ctx.dtype, ctx.device, ctx.quantizer = inputs.dtype, inputs.device, quantizer
95 ctx.clamp_limit = clamp_limit
96 if inputs.shape[0] == 0:
97 ctx.save_for_backward()
98 return inputs.new_empty((0, down.shape[-1]))
99 input_q = quantizer.quantize(inputs, group_list=counts, group_list_type=1, rowwise=True, colwise=True)
100 gate_up_q = quantizer.quantize(gate_up.contiguous(), rowwise=True, colwise=True)
101 down_q = quantizer.quantize(down.contiguous(), rowwise=True, colwise=True)
102 projected = _gmm(input_q, gate_up_q, counts, inputs.dtype)
103 activated_q = _fused_quant(projected, quantizer, counts, clamp_limit=clamp_limit)
104 output = _gmm(activated_q, down_q, counts, inputs.dtype)
105 _save_operands(ctx, counts, projected, (input_q, gate_up_q, activated_q, down_q))
106 return output
107
108 @staticmethod
109 @once_differentiable
110 def backward(
111 ctx: torch.autograd.function.FunctionCtx, gradient: torch.Tensor,
112 ) -> tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor], None, None, None]:
113 """Fuse SwiGLU derivative/quantization and compute requested first gradients.
114
115 Args:
116 ctx: Saved forward context.
117 gradient: Output gradient.
118 """
119 needs_input, needs_gate, needs_down = ctx.needs_input_grad[:3]
120 if ctx.shapes[0][0] == 0:
121 grads = [torch.zeros(shape, device=ctx.device, dtype=ctx.dtype) if needed else None
122 for shape, needed in zip(ctx.shapes, (needs_input, needs_gate, needs_down))]
123 return *grads, None, None, None
124 counts, projected, (input_q, gate_up_q, activated_q, down_q) = _restore_operands(ctx)
125 grad_q = ctx.quantizer.quantize(
126 gradient, group_list=counts if needs_down else None, group_list_type=1,
127 rowwise=needs_input or needs_gate, colwise=needs_down)
128 grad_down = _gmm(activated_q, grad_q, counts, ctx.dtype, "TN") if needs_down else None
129 grad_input = grad_gate = None
130 if needs_input or needs_gate:
131 grad_activated = _gmm(grad_q, down_q, counts, ctx.dtype, "NT")
132 grad_projected_q = _fused_quant(projected, ctx.quantizer, counts, grad_activated, ctx.clamp_limit)
133 if needs_input:
134 grad_input = _gmm(grad_projected_q, gate_up_q, counts, ctx.dtype, "NT")
135 if needs_gate:
136 grad_gate = _gmm(input_q, grad_projected_q, counts, ctx.dtype, "TN")
137 return grad_input, grad_gate, grad_down, None, None, None
138
139
140def fused_grouped_swiglu(
141 inputs: torch.Tensor, gate_up: torch.Tensor, down: torch.Tensor,
142 counts: torch.Tensor, quantizer: "MXFP8Quantizer",
143 *, clamp_limit: Optional[float] = None,
144) -> torch.Tensor:
145 """Compute expert-major outputs with canonical [E,H,2I]/[E,I,H] weights.
146
147 Args:
148 inputs: BF16 expert-major token matrix.
149 gate_up: Canonical packed gate/up weights.
150 down: Canonical down weights.
151 counts: Per-local-expert token counts, including zero-token experts.
152 quantizer: MXFP8 recipe with both fused NPU kernels available.
153 clamp_limit: Optional positive clipping limit for gate and up projections.
154
155 Returns:
156 Ordinary BF16 outputs before outer routing weights are applied.
157 """
158 if inputs.dtype not in (torch.bfloat16, torch.float16):
159 raise ValueError("Fused MXFP8 SwiGLU requires BF16 or FP16 inputs")
160 if gate_up.dtype != inputs.dtype or down.dtype != inputs.dtype:
161 raise ValueError("Fused MXFP8 SwiGLU requires matching input and weight dtypes")
162 if inputs.ndim != 2 or gate_up.ndim != 3 or down.ndim != 3:
163 raise ValueError("Fused MXFP8 SwiGLU requires 2D inputs and 3D expert weights")
164 if (gate_up.shape[0] != down.shape[0] or gate_up.shape[1] != inputs.shape[1]
165 or gate_up.shape[2] != 2 * down.shape[1] or down.shape[2] != inputs.shape[1]):
166 raise ValueError("Fused MXFP8 SwiGLU weight dimensions do not match inputs")
167 if counts.ndim != 1 or counts.numel() != gate_up.shape[0] or counts.dtype not in (torch.int32, torch.int64):
168 raise ValueError("Fused MXFP8 SwiGLU requires one integer count per expert")
169 return _FusedGroupedSwiGLU.apply(inputs, gate_up, down, counts, quantizer, clamp_limit)