Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / attention_residuals.py: 0%

41 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Block attention residuals with eager math and the ops-transformer fused API.""" 

16 

17import importlib 

18from collections.abc import Callable 

19 

20# Torch is the native framework; the legacy platform-agnostic lint rule is obsolete here. 

21import torch 

22 

23 

24def _block_attention_residuals_eager(prefix: torch.Tensor, history: torch.Tensor, 

25 projection: torch.Tensor, norm_weight: torch.Tensor, 

26 norm_eps: float) -> torch.Tensor: 

27 values = torch.cat((history, prefix.unsqueeze(1)), dim=1) 

28 values_fp32 = values.float() 

29 normalized = values_fp32 * torch.rsqrt(values_fp32.square().mean(-1, keepdim=True) + norm_eps) 

30 query = projection.float().reshape(-1) * norm_weight.float() 

31 probabilities = (normalized * query).sum(-1).softmax(-1) 

32 return (probabilities.unsqueeze(-1) * values_fp32).sum(1).to(values.dtype) 

33 

34 

35def _load_block_attention_residuals() -> Callable: 

36 """Resolve the differentiable ops-transformer API and its backward dependency.""" 

37 try: 

38 # CANN is optional for CPU and eager-only callers. 

39 ops = importlib.import_module("cann_ops_transformer") 

40 except (ImportError, OSError) as error: 

41 raise ImportError( 

42 "AttnRes backend='ascendc' could not load cann_ops_transformer. Install the standalone " 

43 "ops-transformer Python package with compatible PyTorch, torch-npu and CANN libraries. " 

44 f"See hyper_parallel/components/functional/README.md. Original error: {error}" 

45 ) from error 

46 block_attention_residuals_op = getattr(ops, "block_attention_residuals", None) 

47 block_attention_residuals_backward = getattr(ops, "block_attention_residuals_backward", None) 

48 if not callable(block_attention_residuals_op) or not callable(block_attention_residuals_backward): 

49 raise ImportError( 

50 "AttnRes backend='ascendc' requires cann_ops_transformer.block_attention_residuals and " 

51 "block_attention_residuals_backward. Install a package containing both training APIs " 

52 "and build block_attention_residuals plus block_attention_residuals_grad. " 

53 "See hyper_parallel/components/functional/README.md." 

54 ) 

55 return block_attention_residuals_op 

56 

57 

58def _block_attention_residuals_fused(prefix: torch.Tensor, history: torch.Tensor, 

59 projection: torch.Tensor, norm_weight: torch.Tensor, 

60 norm_eps: float) -> torch.Tensor: 

61 """Prepare differentiable inputs and invoke the fused block attention residuals API.""" 

62 block_attention_residuals_op = _load_block_attention_residuals() 

63 # Native forward accepts [H], but its training backward requires [1,H]. 

64 if projection.ndim == 1: 

65 projection = projection.unsqueeze(0) 

66 inputs = (prefix, history, projection, norm_weight) 

67 common_dtype = prefix.dtype 

68 for tensor in inputs[1:]: 

69 common_dtype = torch.promote_types(common_dtype, tensor.dtype) 

70 # The native API requires one dtype; casts must preserve all four gradient edges. 

71 prepared = tuple(tensor.to(dtype=common_dtype).contiguous() for tensor in inputs) 

72 result = block_attention_residuals_op(*prepared, valid_block_num=-1, norm_eps=norm_eps) 

73 return result.to(torch.promote_types(prefix.dtype, history.dtype)) 

74 

75 

76def block_attention_residuals(prefix: torch.Tensor, history: torch.Tensor, 

77 projection: torch.Tensor, norm_weight: torch.Tensor, *, norm_eps: float, 

78 backend: str = "eager") -> torch.Tensor: 

79 """Mix previous blocks and the current residual through depth attention. 

80 

81 Args: 

82 prefix: Current residual of shape [T,H], with complete hidden H. 

83 history: Previous blocks of shape [T,N,H]. 

84 projection: Learned pseudo-query of shape [H] or [1,H]. 

85 norm_weight: Learned RMS normalization scale of shape [H]. 

86 norm_eps: RMSNorm epsilon from the calling model layer. 

87 backend: ``eager`` uses FP32 intermediates. ``ascendc`` calls the 

88 differentiable ops-transformer API; native errors propagate unchanged. 

89 

90 Returns: 

91 Tensor of shape [T,H] with the promoted prefix/history dtype. Mixed 

92 inputs are differentiably promoted to one dtype for the fused operator. 

93 Empty T or N retains zero gradients without a native launch. 

94 

95 Raises: 

96 ImportError: The optional CANN package cannot load or lacks a training API. 

97 

98 Note: 

99 Models own block boundaries and learned parameters. The fused operator 

100 supports first-order gradients. Its dispatcher and native entry validate 

101 device and shape limits; kernel execution errors propagate unchanged. 

102 """ 

103 if backend == "eager": 

104 return _block_attention_residuals_eager(prefix, history, projection, norm_weight, norm_eps) 

105 if backend != "ascendc": 

106 raise ValueError("AttnRes backend must be 'eager' or 'ascendc'") 

107 if prefix.shape[0] == 0 or history.shape[1] == 0: 

108 # Empty views keep zero gradients without inspecting unused values. 

109 zero = sum(tensor.reshape(-1)[:0].sum() for tensor in (history, projection, norm_weight)) 

110 dtype = torch.promote_types(prefix.dtype, history.dtype) 

111 return prefix.to(dtype) + zero.to(dtype) 

112 return _block_attention_residuals_fused(prefix, history, projection, norm_weight, norm_eps)