Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / quantization / ops / npu_mxfp8.py: 57%
136 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""A5 MXFP8 quantization and matrix-multiplication operator adapters."""
17from functools import lru_cache
18from typing import Any, Optional
20import torch
22from hyper_parallel.components.quantization.config import (
23 GMM_GROUP_K,
24 GMM_GROUP_M,
25 GMM_SPLIT_MXFP8,
26 GROUP_LIST_COUNTS,
27 GROUP_LIST_OFFSETS,
28 MATRIX_NDIM,
29 MX_BLOCK_SIZE_32,
30 MX_SCALE_ALG_CUBALS,
31)
32from hyper_parallel.components.quantization.tensor import MXFP8Tensor
34_A5_DEVICE_MARKERS = ("Ascend950", "Ascend910_95")
35_SUPPORTED_LAYOUTS = ("NN", "NT", "TN")
36# CANN clipped SwiGLU expects (256 + aclDataType) in forward and raw aclDataType in backward.
37_CLAMPED_SWIGLU_DST_TYPES = {
38 torch.float8_e4m3fn: (292, 36),
39 torch.float8_e5m2: (291, 35),
40}
43class LowPrecisionCapabilityError(RuntimeError):
44 """Report that the active torch_npu stack lacks the A5 MXFP8 contract."""
47@lru_cache(maxsize=1)
48def _get_clamped_swiglu_ops() -> Any:
49 """Load optional CANN registration only when clipped fusion is requested."""
50 try:
51 import cann_ops_nn # pylint: disable=C0415,unused-import
52 except ImportError as exc:
53 raise LowPrecisionCapabilityError(
54 "Clipped MXFP8 SwiGLU requires cann_ops_nn (CANN 9.2.1 on Ascend 950PR/950DT)."
55 ) from exc
56 operators = torch.ops.cann_ops_nn
57 names = ("swiglu_group_quant_with_dual_axis", "swiglu_backward_group_quant_with_dual_axis")
58 missing = [name for name in names if not hasattr(operators, name)]
59 if missing:
60 raise LowPrecisionCapabilityError(f"Clipped MXFP8 SwiGLU requires cann_ops_nn operators {missing}")
61 return operators
64class MXFP8NpuOps:
65 """Adapt the A5 MXFP8 torch_npu operator contract."""
67 def __init__(self, torch_npu_module: Any) -> None:
68 """Validate and retain an imported torch_npu module."""
70 required = (
71 "npu_dynamic_mx_quant",
72 "npu_dynamic_mx_quant_with_dual_axis",
73 "npu_quant_matmul",
74 "float8_e8m0fnu",
75 )
76 missing = [
77 name for name in required if not hasattr(torch_npu_module, name)
78 ]
79 if missing:
80 version = getattr(torch_npu_module, "__version__", "unknown")
81 raise LowPrecisionCapabilityError(
82 "The active torch_npu stack does not provide the A5 MXFP8 "
83 f"operator contract; missing {missing}, torch_npu={version}."
84 )
85 npu_handle = getattr(torch_npu_module, "npu", None)
86 device_name = (
87 npu_handle.get_device_name()
88 if npu_handle is not None
89 and hasattr(npu_handle, "get_device_name")
90 else ""
91 )
92 if not any(marker in device_name for marker in _A5_DEVICE_MARKERS):
93 raise LowPrecisionCapabilityError(
94 "MXFP8 is supported only on Ascend 950PR/950DT (A5), "
95 f"but the active device is {device_name or 'unknown'}."
96 )
97 self._torch_npu = torch_npu_module
99 def validate_grouped_matmul(self) -> None:
100 """Fail when the runtime lacks either MXFP8 GMM-specific operator."""
102 required = (
103 "npu_grouped_dynamic_mx_quant",
104 "npu_grouped_matmul",
105 )
106 missing = [
107 name for name in required if not hasattr(self._torch_npu, name)
108 ]
109 if missing:
110 version = getattr(self._torch_npu, "__version__", "unknown")
111 raise LowPrecisionCapabilityError(
112 "The active torch_npu stack does not provide the A5 MXFP8 "
113 f"GMM contract; missing {missing}, torch_npu={version}."
114 )
116 def dynamic_mx_quant(
117 self,
118 tensor: torch.Tensor,
119 *,
120 axis: int,
121 quant_dtype: Optional[torch.dtype] = None,
122 ) -> tuple[torch.Tensor, torch.Tensor]:
123 """Quantize one tensor along an MX block axis."""
125 return self._torch_npu.npu_dynamic_mx_quant(
126 tensor,
127 axis=axis,
128 dst_type=quant_dtype or torch.float8_e4m3fn,
129 scale_alg=MX_SCALE_ALG_CUBALS,
130 )
132 def dynamic_mx_quant_dual_axis(
133 self,
134 tensor: torch.Tensor,
135 *,
136 quant_dtype: Optional[torch.dtype] = None,
137 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
138 """Quantize one tensor into row-wise and column-wise MX storage."""
140 return self._torch_npu.npu_dynamic_mx_quant_with_dual_axis(
141 tensor,
142 dst_type=quant_dtype or torch.float8_e4m3fn,
143 scale_alg=MX_SCALE_ALG_CUBALS,
144 )
146 def validate_swiglu_fusion(self, *, clamp_limit: Optional[float] = None) -> None:
147 """Require both fused kernels; never silently fall back to plain SwiGLU."""
148 if clamp_limit is not None:
149 _get_clamped_swiglu_ops()
150 return
151 names = ("npu_swiglu_mx_quant_with_dual_axis", "_npu_swiglu_backward_mx_quant_with_dual_axis")
152 missing = [name for name in names if not hasattr(self._torch_npu, name)]
153 if missing:
154 raise LowPrecisionCapabilityError(f"MXFP8 SwiGLU fusion requires {missing}")
156 def swiglu_mx_quant_dual_axis(
157 self,
158 tensor: torch.Tensor,
159 *,
160 quant_dtype: torch.dtype = torch.float8_e4m3fn,
161 group_index: Optional[torch.Tensor] = None,
162 clamp_limit: Optional[float] = None,
163 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
164 """Apply SiLU(gate) * up and quantize both MX directions.
166 Args:
167 tensor: BF16/FP16 matrix containing [gate | up] along its last axis.
168 quant_dtype: Payload dtype, E4M3 for the current MXFP8 recipe.
169 group_index: Cumulative expert token boundaries, or None for Dense.
170 clamp_limit: Positive DeepSeek clipping limit, or None for standard SwiGLU.
172 Returns:
173 Row data, row scale, column data, and column scale.
174 """
175 if clamp_limit is not None:
176 outputs = _get_clamped_swiglu_ops().swiglu_group_quant_with_dual_axis(
177 tensor.detach(), group_index=group_index,
178 dst_type=_CLAMPED_SWIGLU_DST_TYPES[quant_dtype][0], quant_mode=1, clamp_limit=clamp_limit,
179 )
180 return tuple(outputs[:4])
181 return self._torch_npu.npu_swiglu_mx_quant_with_dual_axis(
182 tensor, group_index=group_index, activate_left=True,
183 dst_type=quant_dtype, scale_alg=1,
184 )
186 def swiglu_backward_mx_quant_dual_axis(
187 self,
188 tensor: torch.Tensor,
189 grad_output: torch.Tensor,
190 *,
191 quant_dtype: torch.dtype = torch.float8_e4m3fn,
192 group_index: Optional[torch.Tensor] = None,
193 clamp_limit: Optional[float] = None,
194 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
195 """Compute the SwiGLU derivative and quantize both MX directions.
197 Args:
198 tensor: Saved BF16/FP16 packed Gate/Up matrix.
199 grad_output: BF16/FP16 gradient of the SwiGLU output.
200 quant_dtype: Payload dtype, E4M3 for the current MXFP8 recipe.
201 group_index: The same cumulative boundaries used by forward.
202 clamp_limit: The clipping limit used by forward, or None for standard SwiGLU.
204 Returns:
205 Row data, row scale, column data, and column scale of the gradient.
206 """
207 if clamp_limit is not None:
208 outputs = _get_clamped_swiglu_ops().swiglu_backward_group_quant_with_dual_axis(
209 grad_output, tensor, group_index=group_index,
210 dst_type=_CLAMPED_SWIGLU_DST_TYPES[quant_dtype][1], quant_mode=1, clamp_limit=clamp_limit,
211 )
212 return tuple(outputs[:4])
213 return self._torch_npu._npu_swiglu_backward_mx_quant_with_dual_axis( # pylint: disable=protected-access
214 tensor, grad_output, group_index=group_index, activate_left=True,
215 dst_type=quant_dtype, scale_alg=1,
216 )
218 def quant_matmul(
219 self,
220 x1: torch.Tensor,
221 x2: torch.Tensor,
222 scale: torch.Tensor,
223 *,
224 pertoken_scale: torch.Tensor,
225 output_dtype: torch.dtype,
226 ) -> torch.Tensor:
227 """Execute an A5 MXFP8 matrix multiplication."""
229 return self._torch_npu.npu_quant_matmul(
230 x1,
231 x2,
232 scale,
233 pertoken_scale=pertoken_scale,
234 output_dtype=output_dtype,
235 scale_dtype=self._torch_npu.float8_e8m0fnu,
236 pertoken_scale_dtype=self._torch_npu.float8_e8m0fnu,
237 group_sizes=[1, 1, MX_BLOCK_SIZE_32],
238 )
240 def grouped_dynamic_mx_quant(
241 self,
242 tensor: torch.Tensor,
243 group_list: torch.Tensor,
244 *,
245 quant_dtype: Optional[torch.dtype] = None,
246 ) -> tuple[torch.Tensor, torch.Tensor]:
247 """Quantize the column direction independently for each expert."""
249 grouped_quant = getattr(
250 self._torch_npu,
251 "npu_grouped_dynamic_mx_quant",
252 None,
253 )
254 if grouped_quant is None:
255 version = getattr(self._torch_npu, "__version__", "unknown")
256 raise LowPrecisionCapabilityError(
257 "The active torch_npu stack does not provide "
258 "npu_grouped_dynamic_mx_quant, which MXFP8 GMM requires; "
259 f"torch_npu={version}."
260 )
261 return grouped_quant(
262 tensor,
263 # A5 GroupedDynamicMxQuant's groupIndex contract is int32.
264 group_list.to(torch.int32),
265 round_mode="rint",
266 dst_type=quant_dtype or torch.float8_e4m3fn,
267 blocksize=MX_BLOCK_SIZE_32,
268 )
270 def quant_grouped_matmul(
271 self,
272 x1: torch.Tensor,
273 x2: torch.Tensor,
274 x2_scale: torch.Tensor,
275 *,
276 x1_scale: torch.Tensor,
277 group_list: torch.Tensor,
278 group_type: int,
279 group_list_type: int,
280 output_dtype: torch.dtype,
281 bias: Optional[list[torch.Tensor]] = None,
282 ) -> torch.Tensor:
283 """Execute one A5 MXFP8 grouped matrix multiplication."""
285 grouped_matmul = getattr(self._torch_npu, "npu_grouped_matmul", None)
286 if grouped_matmul is None:
287 version = getattr(self._torch_npu, "__version__", "unknown")
288 raise LowPrecisionCapabilityError(
289 "The active torch_npu stack does not provide "
290 f"npu_grouped_matmul; torch_npu={version}."
291 )
292 # A5 GroupedMatmul's groupList contract is int64.
293 group_list = group_list.to(torch.int64)
294 return grouped_matmul(
295 [x1],
296 [x2],
297 bias=[] if bias is None else bias,
298 scale=[x2_scale],
299 per_token_scale=[x1_scale],
300 group_list=group_list,
301 group_type=group_type,
302 output_dtype=output_dtype,
303 group_list_type=group_list_type,
304 scale_dtype=self._torch_npu.float8_e8m0fnu,
305 per_token_scale_dtype=self._torch_npu.float8_e8m0fnu,
306 split_item=GMM_SPLIT_MXFP8,
307 )[0]
309 def is_e8m0_dtype(self, dtype: torch.dtype) -> bool:
310 """Return whether a scale dtype is this runtime's E8M0 representation."""
311 return dtype == self._torch_npu.float8_e8m0fnu
314@lru_cache(maxsize=1)
315def _get_npu_ops() -> MXFP8NpuOps:
316 """Return the process-local A5 MXFP8 operator adapter."""
318 try:
319 import torch_npu # pylint: disable=C0415
320 except ImportError as exc:
321 raise LowPrecisionCapabilityError(
322 "A5 MXFP8 requires torch_npu, but it is not importable."
323 ) from exc
324 return MXFP8NpuOps(torch_npu)
327def validate_npu_runtime() -> None:
328 """Fail during model setup when the MXFP8 runtime is unavailable."""
330 _get_npu_ops()
333def validate_npu_swiglu_runtime(*, clamp_limit: Optional[float] = None) -> None:
334 """Validate fusion without storing runtime modules on a model's quantizer."""
335 _get_npu_ops().validate_swiglu_fusion(clamp_limit=clamp_limit)
338def validate_npu_gmm_runtime() -> None:
339 """Fail during expert setup when the MXFP8 GMM runtime is unavailable."""
341 _get_npu_ops().validate_grouped_matmul()
344def _transpose_scale(scale: torch.Tensor) -> torch.Tensor:
345 """Transpose the two matrix dimensions of an MX scale tensor."""
347 if scale.ndim < MATRIX_NDIM:
348 raise ValueError(
349 "MXFP8 scale transpose requires at least two dimensions, got "
350 f"shape {tuple(scale.shape)}."
351 )
352 if scale.ndim == MATRIX_NDIM:
353 return scale.transpose(0, 1)
354 return scale.transpose(-3, -2)
357def _left_operand(
358 tensor: MXFP8Tensor,
359 *,
360 transpose: bool,
361) -> tuple[torch.Tensor, torch.Tensor]:
362 """Select MX data and scales for the requested left-operand layout."""
363 if transpose:
364 if not tensor.is_colwise():
365 raise ValueError(
366 "Transposed MXFP8 left operand requires column-wise data."
367 )
368 return (
369 tensor.col_data.transpose(-1, -2),
370 _transpose_scale(tensor.col_scale),
371 )
372 if not tensor.is_rowwise():
373 raise ValueError(
374 "Non-transposed MXFP8 left operand requires row-wise data."
375 )
376 return tensor.row_data, tensor.row_scale
379def _right_operand(
380 tensor: MXFP8Tensor,
381 *,
382 transpose: bool,
383) -> tuple[torch.Tensor, torch.Tensor]:
384 """Select MX data and scales for the requested right-operand layout."""
385 if transpose:
386 if not tensor.is_rowwise():
387 raise ValueError(
388 "Transposed MXFP8 right operand requires row-wise data."
389 )
390 return (
391 tensor.row_data.transpose(-1, -2),
392 _transpose_scale(tensor.row_scale),
393 )
394 if not tensor.is_colwise():
395 raise ValueError(
396 "Non-transposed MXFP8 right operand requires column-wise data."
397 )
398 return tensor.col_data, tensor.col_scale
401def mxfp8_matmul(
402 left: MXFP8Tensor,
403 right: MXFP8Tensor,
404 *,
405 layout: str,
406 output_dtype: Optional[torch.dtype] = None,
407) -> torch.Tensor:
408 """Multiply typed MXFP8 operands and return a high-precision Tensor."""
410 if not isinstance(left, MXFP8Tensor) or not isinstance(
411 right,
412 MXFP8Tensor,
413 ):
414 raise TypeError("mxfp8_matmul requires two MXFP8Tensor operands.")
415 if layout not in _SUPPORTED_LAYOUTS:
416 raise ValueError(
417 f"Unsupported MXFP8 layout {layout!r}; "
418 f"expected one of {_SUPPORTED_LAYOUTS}."
419 )
421 left_data, left_scale = _left_operand(
422 left,
423 transpose=layout[0] == "T",
424 )
425 right_data, right_scale = _right_operand(
426 right,
427 transpose=layout[1] == "T",
428 )
429 return left.quantizer.npu_ops.quant_matmul(
430 left_data,
431 right_data,
432 right_scale,
433 pertoken_scale=left_scale,
434 output_dtype=output_dtype or left.dtype,
435 )
438def mxfp8_grouped_matmul(
439 left: MXFP8Tensor,
440 right: MXFP8Tensor,
441 *,
442 layout: str,
443 group_list: torch.Tensor,
444 group_type: int = GMM_GROUP_M,
445 group_list_type: int = GROUP_LIST_OFFSETS,
446 output_dtype: Optional[torch.dtype] = None,
447 bias: Optional[torch.Tensor] = None,
448) -> torch.Tensor:
449 """Multiply expert-grouped MXFP8 operands and return a high-precision Tensor.
451 The layout follows ``Y = op(left) @ op(right)``. Forward and input-gradient
452 GMM split the M axis with ``group_type=0``; weight-gradient GMM splits the
453 contracting axis with ``group_type=2``.
455 Args:
456 left: Quantized left operand.
457 right: Quantized right operand.
458 layout: One of ``"NN"``, ``"NT"``, or ``"TN"``.
459 group_list: Expert boundaries or per-expert token counts.
460 group_type: Axis grouped by the NPU kernel.
461 group_list_type: ``0`` for cumulative boundaries or ``1`` for counts.
462 output_dtype: Optional high-precision output dtype.
463 bias: Optional grouped bias tensor.
465 Returns:
466 The grouped matrix-multiplication result.
467 """
469 if not isinstance(left, MXFP8Tensor) or not isinstance(right, MXFP8Tensor):
470 raise TypeError("mxfp8_grouped_matmul requires two MXFP8Tensor operands.")
471 if layout not in _SUPPORTED_LAYOUTS:
472 raise ValueError(
473 f"Unsupported MXFP8 GMM layout {layout!r}; "
474 f"expected one of {_SUPPORTED_LAYOUTS}."
475 )
476 if not isinstance(group_list, torch.Tensor) or group_list.ndim != 1:
477 raise ValueError("MXFP8 GMM group_list must be a one-dimensional Tensor.")
478 if group_type not in (GMM_GROUP_M, GMM_GROUP_K):
479 raise ValueError(
480 f"MXFP8 GMM group_type must be {GMM_GROUP_M} or {GMM_GROUP_K}, "
481 f"but got {group_type}."
482 )
483 if group_list_type not in (GROUP_LIST_OFFSETS, GROUP_LIST_COUNTS):
484 raise ValueError(
485 "MXFP8 GMM group_list_type must be "
486 f"{GROUP_LIST_OFFSETS} or {GROUP_LIST_COUNTS}, "
487 f"but got {group_list_type}."
488 )
490 left_data, left_scale = _left_operand(
491 left,
492 transpose=layout[0] == "T",
493 )
494 right_data, right_scale = _right_operand(
495 right,
496 transpose=layout[1] == "T",
497 )
498 bias_list = None if bias is None else [bias]
499 return left.quantizer.npu_ops.quant_grouped_matmul(
500 left_data,
501 right_data,
502 right_scale,
503 x1_scale=left_scale,
504 group_list=group_list,
505 group_type=group_type,
506 group_list_type=group_list_type,
507 output_dtype=output_dtype or left.dtype,
508 bias=bias_list,
509 )