Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / quantization / ops / npu_mxfp8.py: 57%

136 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""A5 MXFP8 quantization and matrix-multiplication operator adapters.""" 

16 

17from functools import lru_cache 

18from typing import Any, Optional 

19 

20import torch 

21 

22from hyper_parallel.components.quantization.config import ( 

23 GMM_GROUP_K, 

24 GMM_GROUP_M, 

25 GMM_SPLIT_MXFP8, 

26 GROUP_LIST_COUNTS, 

27 GROUP_LIST_OFFSETS, 

28 MATRIX_NDIM, 

29 MX_BLOCK_SIZE_32, 

30 MX_SCALE_ALG_CUBALS, 

31) 

32from hyper_parallel.components.quantization.tensor import MXFP8Tensor 

33 

34_A5_DEVICE_MARKERS = ("Ascend950", "Ascend910_95") 

35_SUPPORTED_LAYOUTS = ("NN", "NT", "TN") 

36# CANN clipped SwiGLU expects (256 + aclDataType) in forward and raw aclDataType in backward. 

37_CLAMPED_SWIGLU_DST_TYPES = { 

38 torch.float8_e4m3fn: (292, 36), 

39 torch.float8_e5m2: (291, 35), 

40} 

41 

42 

43class LowPrecisionCapabilityError(RuntimeError): 

44 """Report that the active torch_npu stack lacks the A5 MXFP8 contract.""" 

45 

46 

47@lru_cache(maxsize=1) 

48def _get_clamped_swiglu_ops() -> Any: 

49 """Load optional CANN registration only when clipped fusion is requested.""" 

50 try: 

51 import cann_ops_nn # pylint: disable=C0415,unused-import 

52 except ImportError as exc: 

53 raise LowPrecisionCapabilityError( 

54 "Clipped MXFP8 SwiGLU requires cann_ops_nn (CANN 9.2.1 on Ascend 950PR/950DT)." 

55 ) from exc 

56 operators = torch.ops.cann_ops_nn 

57 names = ("swiglu_group_quant_with_dual_axis", "swiglu_backward_group_quant_with_dual_axis") 

58 missing = [name for name in names if not hasattr(operators, name)] 

59 if missing: 

60 raise LowPrecisionCapabilityError(f"Clipped MXFP8 SwiGLU requires cann_ops_nn operators {missing}") 

61 return operators 

62 

63 

64class MXFP8NpuOps: 

65 """Adapt the A5 MXFP8 torch_npu operator contract.""" 

66 

67 def __init__(self, torch_npu_module: Any) -> None: 

68 """Validate and retain an imported torch_npu module.""" 

69 

70 required = ( 

71 "npu_dynamic_mx_quant", 

72 "npu_dynamic_mx_quant_with_dual_axis", 

73 "npu_quant_matmul", 

74 "float8_e8m0fnu", 

75 ) 

76 missing = [ 

77 name for name in required if not hasattr(torch_npu_module, name) 

78 ] 

79 if missing: 

80 version = getattr(torch_npu_module, "__version__", "unknown") 

81 raise LowPrecisionCapabilityError( 

82 "The active torch_npu stack does not provide the A5 MXFP8 " 

83 f"operator contract; missing {missing}, torch_npu={version}." 

84 ) 

85 npu_handle = getattr(torch_npu_module, "npu", None) 

86 device_name = ( 

87 npu_handle.get_device_name() 

88 if npu_handle is not None 

89 and hasattr(npu_handle, "get_device_name") 

90 else "" 

91 ) 

92 if not any(marker in device_name for marker in _A5_DEVICE_MARKERS): 

93 raise LowPrecisionCapabilityError( 

94 "MXFP8 is supported only on Ascend 950PR/950DT (A5), " 

95 f"but the active device is {device_name or 'unknown'}." 

96 ) 

97 self._torch_npu = torch_npu_module 

98 

99 def validate_grouped_matmul(self) -> None: 

100 """Fail when the runtime lacks either MXFP8 GMM-specific operator.""" 

101 

102 required = ( 

103 "npu_grouped_dynamic_mx_quant", 

104 "npu_grouped_matmul", 

105 ) 

106 missing = [ 

107 name for name in required if not hasattr(self._torch_npu, name) 

108 ] 

109 if missing: 

110 version = getattr(self._torch_npu, "__version__", "unknown") 

111 raise LowPrecisionCapabilityError( 

112 "The active torch_npu stack does not provide the A5 MXFP8 " 

113 f"GMM contract; missing {missing}, torch_npu={version}." 

114 ) 

115 

116 def dynamic_mx_quant( 

117 self, 

118 tensor: torch.Tensor, 

119 *, 

120 axis: int, 

121 quant_dtype: Optional[torch.dtype] = None, 

122 ) -> tuple[torch.Tensor, torch.Tensor]: 

123 """Quantize one tensor along an MX block axis.""" 

124 

125 return self._torch_npu.npu_dynamic_mx_quant( 

126 tensor, 

127 axis=axis, 

128 dst_type=quant_dtype or torch.float8_e4m3fn, 

129 scale_alg=MX_SCALE_ALG_CUBALS, 

130 ) 

131 

132 def dynamic_mx_quant_dual_axis( 

133 self, 

134 tensor: torch.Tensor, 

135 *, 

136 quant_dtype: Optional[torch.dtype] = None, 

137 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 

138 """Quantize one tensor into row-wise and column-wise MX storage.""" 

139 

140 return self._torch_npu.npu_dynamic_mx_quant_with_dual_axis( 

141 tensor, 

142 dst_type=quant_dtype or torch.float8_e4m3fn, 

143 scale_alg=MX_SCALE_ALG_CUBALS, 

144 ) 

145 

146 def validate_swiglu_fusion(self, *, clamp_limit: Optional[float] = None) -> None: 

147 """Require both fused kernels; never silently fall back to plain SwiGLU.""" 

148 if clamp_limit is not None: 

149 _get_clamped_swiglu_ops() 

150 return 

151 names = ("npu_swiglu_mx_quant_with_dual_axis", "_npu_swiglu_backward_mx_quant_with_dual_axis") 

152 missing = [name for name in names if not hasattr(self._torch_npu, name)] 

153 if missing: 

154 raise LowPrecisionCapabilityError(f"MXFP8 SwiGLU fusion requires {missing}") 

155 

156 def swiglu_mx_quant_dual_axis( 

157 self, 

158 tensor: torch.Tensor, 

159 *, 

160 quant_dtype: torch.dtype = torch.float8_e4m3fn, 

161 group_index: Optional[torch.Tensor] = None, 

162 clamp_limit: Optional[float] = None, 

163 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 

164 """Apply SiLU(gate) * up and quantize both MX directions. 

165 

166 Args: 

167 tensor: BF16/FP16 matrix containing [gate | up] along its last axis. 

168 quant_dtype: Payload dtype, E4M3 for the current MXFP8 recipe. 

169 group_index: Cumulative expert token boundaries, or None for Dense. 

170 clamp_limit: Positive DeepSeek clipping limit, or None for standard SwiGLU. 

171 

172 Returns: 

173 Row data, row scale, column data, and column scale. 

174 """ 

175 if clamp_limit is not None: 

176 outputs = _get_clamped_swiglu_ops().swiglu_group_quant_with_dual_axis( 

177 tensor.detach(), group_index=group_index, 

178 dst_type=_CLAMPED_SWIGLU_DST_TYPES[quant_dtype][0], quant_mode=1, clamp_limit=clamp_limit, 

179 ) 

180 return tuple(outputs[:4]) 

181 return self._torch_npu.npu_swiglu_mx_quant_with_dual_axis( 

182 tensor, group_index=group_index, activate_left=True, 

183 dst_type=quant_dtype, scale_alg=1, 

184 ) 

185 

186 def swiglu_backward_mx_quant_dual_axis( 

187 self, 

188 tensor: torch.Tensor, 

189 grad_output: torch.Tensor, 

190 *, 

191 quant_dtype: torch.dtype = torch.float8_e4m3fn, 

192 group_index: Optional[torch.Tensor] = None, 

193 clamp_limit: Optional[float] = None, 

194 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 

195 """Compute the SwiGLU derivative and quantize both MX directions. 

196 

197 Args: 

198 tensor: Saved BF16/FP16 packed Gate/Up matrix. 

199 grad_output: BF16/FP16 gradient of the SwiGLU output. 

200 quant_dtype: Payload dtype, E4M3 for the current MXFP8 recipe. 

201 group_index: The same cumulative boundaries used by forward. 

202 clamp_limit: The clipping limit used by forward, or None for standard SwiGLU. 

203 

204 Returns: 

205 Row data, row scale, column data, and column scale of the gradient. 

206 """ 

207 if clamp_limit is not None: 

208 outputs = _get_clamped_swiglu_ops().swiglu_backward_group_quant_with_dual_axis( 

209 grad_output, tensor, group_index=group_index, 

210 dst_type=_CLAMPED_SWIGLU_DST_TYPES[quant_dtype][1], quant_mode=1, clamp_limit=clamp_limit, 

211 ) 

212 return tuple(outputs[:4]) 

213 return self._torch_npu._npu_swiglu_backward_mx_quant_with_dual_axis( # pylint: disable=protected-access 

214 tensor, grad_output, group_index=group_index, activate_left=True, 

215 dst_type=quant_dtype, scale_alg=1, 

216 ) 

217 

218 def quant_matmul( 

219 self, 

220 x1: torch.Tensor, 

221 x2: torch.Tensor, 

222 scale: torch.Tensor, 

223 *, 

224 pertoken_scale: torch.Tensor, 

225 output_dtype: torch.dtype, 

226 ) -> torch.Tensor: 

227 """Execute an A5 MXFP8 matrix multiplication.""" 

228 

229 return self._torch_npu.npu_quant_matmul( 

230 x1, 

231 x2, 

232 scale, 

233 pertoken_scale=pertoken_scale, 

234 output_dtype=output_dtype, 

235 scale_dtype=self._torch_npu.float8_e8m0fnu, 

236 pertoken_scale_dtype=self._torch_npu.float8_e8m0fnu, 

237 group_sizes=[1, 1, MX_BLOCK_SIZE_32], 

238 ) 

239 

240 def grouped_dynamic_mx_quant( 

241 self, 

242 tensor: torch.Tensor, 

243 group_list: torch.Tensor, 

244 *, 

245 quant_dtype: Optional[torch.dtype] = None, 

246 ) -> tuple[torch.Tensor, torch.Tensor]: 

247 """Quantize the column direction independently for each expert.""" 

248 

249 grouped_quant = getattr( 

250 self._torch_npu, 

251 "npu_grouped_dynamic_mx_quant", 

252 None, 

253 ) 

254 if grouped_quant is None: 

255 version = getattr(self._torch_npu, "__version__", "unknown") 

256 raise LowPrecisionCapabilityError( 

257 "The active torch_npu stack does not provide " 

258 "npu_grouped_dynamic_mx_quant, which MXFP8 GMM requires; " 

259 f"torch_npu={version}." 

260 ) 

261 return grouped_quant( 

262 tensor, 

263 # A5 GroupedDynamicMxQuant's groupIndex contract is int32. 

264 group_list.to(torch.int32), 

265 round_mode="rint", 

266 dst_type=quant_dtype or torch.float8_e4m3fn, 

267 blocksize=MX_BLOCK_SIZE_32, 

268 ) 

269 

270 def quant_grouped_matmul( 

271 self, 

272 x1: torch.Tensor, 

273 x2: torch.Tensor, 

274 x2_scale: torch.Tensor, 

275 *, 

276 x1_scale: torch.Tensor, 

277 group_list: torch.Tensor, 

278 group_type: int, 

279 group_list_type: int, 

280 output_dtype: torch.dtype, 

281 bias: Optional[list[torch.Tensor]] = None, 

282 ) -> torch.Tensor: 

283 """Execute one A5 MXFP8 grouped matrix multiplication.""" 

284 

285 grouped_matmul = getattr(self._torch_npu, "npu_grouped_matmul", None) 

286 if grouped_matmul is None: 

287 version = getattr(self._torch_npu, "__version__", "unknown") 

288 raise LowPrecisionCapabilityError( 

289 "The active torch_npu stack does not provide " 

290 f"npu_grouped_matmul; torch_npu={version}." 

291 ) 

292 # A5 GroupedMatmul's groupList contract is int64. 

293 group_list = group_list.to(torch.int64) 

294 return grouped_matmul( 

295 [x1], 

296 [x2], 

297 bias=[] if bias is None else bias, 

298 scale=[x2_scale], 

299 per_token_scale=[x1_scale], 

300 group_list=group_list, 

301 group_type=group_type, 

302 output_dtype=output_dtype, 

303 group_list_type=group_list_type, 

304 scale_dtype=self._torch_npu.float8_e8m0fnu, 

305 per_token_scale_dtype=self._torch_npu.float8_e8m0fnu, 

306 split_item=GMM_SPLIT_MXFP8, 

307 )[0] 

308 

309 def is_e8m0_dtype(self, dtype: torch.dtype) -> bool: 

310 """Return whether a scale dtype is this runtime's E8M0 representation.""" 

311 return dtype == self._torch_npu.float8_e8m0fnu 

312 

313 

314@lru_cache(maxsize=1) 

315def _get_npu_ops() -> MXFP8NpuOps: 

316 """Return the process-local A5 MXFP8 operator adapter.""" 

317 

318 try: 

319 import torch_npu # pylint: disable=C0415 

320 except ImportError as exc: 

321 raise LowPrecisionCapabilityError( 

322 "A5 MXFP8 requires torch_npu, but it is not importable." 

323 ) from exc 

324 return MXFP8NpuOps(torch_npu) 

325 

326 

327def validate_npu_runtime() -> None: 

328 """Fail during model setup when the MXFP8 runtime is unavailable.""" 

329 

330 _get_npu_ops() 

331 

332 

333def validate_npu_swiglu_runtime(*, clamp_limit: Optional[float] = None) -> None: 

334 """Validate fusion without storing runtime modules on a model's quantizer.""" 

335 _get_npu_ops().validate_swiglu_fusion(clamp_limit=clamp_limit) 

336 

337 

338def validate_npu_gmm_runtime() -> None: 

339 """Fail during expert setup when the MXFP8 GMM runtime is unavailable.""" 

340 

341 _get_npu_ops().validate_grouped_matmul() 

342 

343 

344def _transpose_scale(scale: torch.Tensor) -> torch.Tensor: 

345 """Transpose the two matrix dimensions of an MX scale tensor.""" 

346 

347 if scale.ndim < MATRIX_NDIM: 

348 raise ValueError( 

349 "MXFP8 scale transpose requires at least two dimensions, got " 

350 f"shape {tuple(scale.shape)}." 

351 ) 

352 if scale.ndim == MATRIX_NDIM: 

353 return scale.transpose(0, 1) 

354 return scale.transpose(-3, -2) 

355 

356 

357def _left_operand( 

358 tensor: MXFP8Tensor, 

359 *, 

360 transpose: bool, 

361) -> tuple[torch.Tensor, torch.Tensor]: 

362 """Select MX data and scales for the requested left-operand layout.""" 

363 if transpose: 

364 if not tensor.is_colwise(): 

365 raise ValueError( 

366 "Transposed MXFP8 left operand requires column-wise data." 

367 ) 

368 return ( 

369 tensor.col_data.transpose(-1, -2), 

370 _transpose_scale(tensor.col_scale), 

371 ) 

372 if not tensor.is_rowwise(): 

373 raise ValueError( 

374 "Non-transposed MXFP8 left operand requires row-wise data." 

375 ) 

376 return tensor.row_data, tensor.row_scale 

377 

378 

379def _right_operand( 

380 tensor: MXFP8Tensor, 

381 *, 

382 transpose: bool, 

383) -> tuple[torch.Tensor, torch.Tensor]: 

384 """Select MX data and scales for the requested right-operand layout.""" 

385 if transpose: 

386 if not tensor.is_rowwise(): 

387 raise ValueError( 

388 "Transposed MXFP8 right operand requires row-wise data." 

389 ) 

390 return ( 

391 tensor.row_data.transpose(-1, -2), 

392 _transpose_scale(tensor.row_scale), 

393 ) 

394 if not tensor.is_colwise(): 

395 raise ValueError( 

396 "Non-transposed MXFP8 right operand requires column-wise data." 

397 ) 

398 return tensor.col_data, tensor.col_scale 

399 

400 

401def mxfp8_matmul( 

402 left: MXFP8Tensor, 

403 right: MXFP8Tensor, 

404 *, 

405 layout: str, 

406 output_dtype: Optional[torch.dtype] = None, 

407) -> torch.Tensor: 

408 """Multiply typed MXFP8 operands and return a high-precision Tensor.""" 

409 

410 if not isinstance(left, MXFP8Tensor) or not isinstance( 

411 right, 

412 MXFP8Tensor, 

413 ): 

414 raise TypeError("mxfp8_matmul requires two MXFP8Tensor operands.") 

415 if layout not in _SUPPORTED_LAYOUTS: 

416 raise ValueError( 

417 f"Unsupported MXFP8 layout {layout!r}; " 

418 f"expected one of {_SUPPORTED_LAYOUTS}." 

419 ) 

420 

421 left_data, left_scale = _left_operand( 

422 left, 

423 transpose=layout[0] == "T", 

424 ) 

425 right_data, right_scale = _right_operand( 

426 right, 

427 transpose=layout[1] == "T", 

428 ) 

429 return left.quantizer.npu_ops.quant_matmul( 

430 left_data, 

431 right_data, 

432 right_scale, 

433 pertoken_scale=left_scale, 

434 output_dtype=output_dtype or left.dtype, 

435 ) 

436 

437 

438def mxfp8_grouped_matmul( 

439 left: MXFP8Tensor, 

440 right: MXFP8Tensor, 

441 *, 

442 layout: str, 

443 group_list: torch.Tensor, 

444 group_type: int = GMM_GROUP_M, 

445 group_list_type: int = GROUP_LIST_OFFSETS, 

446 output_dtype: Optional[torch.dtype] = None, 

447 bias: Optional[torch.Tensor] = None, 

448) -> torch.Tensor: 

449 """Multiply expert-grouped MXFP8 operands and return a high-precision Tensor. 

450 

451 The layout follows ``Y = op(left) @ op(right)``. Forward and input-gradient 

452 GMM split the M axis with ``group_type=0``; weight-gradient GMM splits the 

453 contracting axis with ``group_type=2``. 

454 

455 Args: 

456 left: Quantized left operand. 

457 right: Quantized right operand. 

458 layout: One of ``"NN"``, ``"NT"``, or ``"TN"``. 

459 group_list: Expert boundaries or per-expert token counts. 

460 group_type: Axis grouped by the NPU kernel. 

461 group_list_type: ``0`` for cumulative boundaries or ``1`` for counts. 

462 output_dtype: Optional high-precision output dtype. 

463 bias: Optional grouped bias tensor. 

464 

465 Returns: 

466 The grouped matrix-multiplication result. 

467 """ 

468 

469 if not isinstance(left, MXFP8Tensor) or not isinstance(right, MXFP8Tensor): 

470 raise TypeError("mxfp8_grouped_matmul requires two MXFP8Tensor operands.") 

471 if layout not in _SUPPORTED_LAYOUTS: 

472 raise ValueError( 

473 f"Unsupported MXFP8 GMM layout {layout!r}; " 

474 f"expected one of {_SUPPORTED_LAYOUTS}." 

475 ) 

476 if not isinstance(group_list, torch.Tensor) or group_list.ndim != 1: 

477 raise ValueError("MXFP8 GMM group_list must be a one-dimensional Tensor.") 

478 if group_type not in (GMM_GROUP_M, GMM_GROUP_K): 

479 raise ValueError( 

480 f"MXFP8 GMM group_type must be {GMM_GROUP_M} or {GMM_GROUP_K}, " 

481 f"but got {group_type}." 

482 ) 

483 if group_list_type not in (GROUP_LIST_OFFSETS, GROUP_LIST_COUNTS): 

484 raise ValueError( 

485 "MXFP8 GMM group_list_type must be " 

486 f"{GROUP_LIST_OFFSETS} or {GROUP_LIST_COUNTS}, " 

487 f"but got {group_list_type}." 

488 ) 

489 

490 left_data, left_scale = _left_operand( 

491 left, 

492 transpose=layout[0] == "T", 

493 ) 

494 right_data, right_scale = _right_operand( 

495 right, 

496 transpose=layout[1] == "T", 

497 ) 

498 bias_list = None if bias is None else [bias] 

499 return left.quantizer.npu_ops.quant_grouped_matmul( 

500 left_data, 

501 right_data, 

502 right_scale, 

503 x1_scale=left_scale, 

504 group_list=group_list, 

505 group_type=group_type, 

506 group_list_type=group_list_type, 

507 output_dtype=output_dtype or left.dtype, 

508 bias=bias_list, 

509 )