Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / activation_situ.py: 0%

56 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""SiTU eager reference and first-order autograd over CANN ops-nn.""" 

16 

17import importlib 

18from collections.abc import Callable 

19from typing import Any, Optional 

20 

21# Torch is the native framework; the legacy platform-agnostic lint rule is obsolete here. 

22import torch 

23from torch.autograd.function import once_differentiable 

24 

25 

26def _situ_eager(gate: torch.Tensor, up: torch.Tensor, beta: float, 

27 linear_beta: Optional[float]) -> torch.Tensor: 

28 gate_float, up_float = gate.float(), up.float() 

29 activated_gate = beta * torch.tanh(gate_float / beta) * torch.sigmoid(gate_float) 

30 if linear_beta is not None and linear_beta > 0: 

31 up_float = linear_beta * torch.tanh(up_float / linear_beta) 

32 return (activated_gate * up_float).to(gate.dtype) 

33 

34 

35def _load_situ_ops() -> tuple[Callable, Callable]: 

36 """Resolve the optional ops-nn training APIs without hiding native errors.""" 

37 try: 

38 # CANN is optional for CPU and eager-only callers. 

39 ops = importlib.import_module("cann_ops_nn") 

40 except (ImportError, OSError) as error: 

41 raise ImportError( 

42 "SiTU backend='ascendc' could not load cann_ops_nn. Install the standalone ops-nn Python package " 

43 "with compatible PyTorch, torch-npu and CANN libraries. " 

44 f"See hyper_parallel/components/functional/README.md. Original error: {error}" 

45 ) from error 

46 situ_glu = getattr(ops, "situ_glu", None) 

47 situ_glu_grad = getattr(ops, "situ_glu_grad", None) 

48 if not callable(situ_glu) or not callable(situ_glu_grad): 

49 raise ImportError( 

50 "SiTU backend='ascendc' requires cann_ops_nn.situ_glu and cann_ops_nn.situ_glu_grad. " 

51 "Install a package containing both training APIs and build both native operators. " 

52 "See hyper_parallel/components/functional/README.md." 

53 ) 

54 return situ_glu, situ_glu_grad 

55 

56 

57class _SituAutograd(torch.autograd.Function): 

58 """Connect ops-nn's separate SituGlu and SituGluGrad entry points.""" 

59 

60 @staticmethod 

61 def forward(ctx: Any, packed: torch.Tensor, beta: float, linear_beta: float) -> torch.Tensor: 

62 """Run dense gate-first rows and retain the native backward inputs. 

63 

64 Args: 

65 ctx: Autograd context holding inputs for backward. 

66 packed: Input tensor with gate channels followed by up channels. 

67 beta: Gate saturation scale. 

68 linear_beta: Up saturation scale; nonpositive values disable saturation. 

69 

70 Returns: 

71 Activated tensor with half the input's last dimension. 

72 """ 

73 situ_glu, situ_glu_grad = _load_situ_ops() 

74 rows = packed.reshape(-1, packed.shape[-1]).contiguous() 

75 ctx.save_for_backward(rows) 

76 ctx.input_shape = packed.shape 

77 ctx.beta = beta 

78 ctx.linear_beta = linear_beta 

79 ctx.situ_glu_grad = situ_glu_grad 

80 output = situ_glu(rows, dim=-1, beta=beta, linear_beta=linear_beta, activate_left=True) 

81 return output.reshape(*packed.shape[:-1], packed.shape[-1] // 2) 

82 

83 @staticmethod 

84 @once_differentiable 

85 def backward(ctx: Any, grad_output: torch.Tensor) -> tuple[torch.Tensor, None, None]: 

86 """Restore the packed gradient with the same saturation parameters. 

87 

88 Args: 

89 ctx: Autograd context populated by forward. 

90 grad_output: Gradient with respect to the activated output. 

91 

92 Returns: 

93 Packed input gradient and no gradients for the scalar attributes. 

94 """ 

95 (rows,) = ctx.saved_tensors 

96 grad_rows = grad_output.reshape(-1, rows.shape[-1] // 2).contiguous() 

97 gradient = ctx.situ_glu_grad( 

98 grad_rows, rows, dim=-1, beta=ctx.beta, linear_beta=ctx.linear_beta, activate_left=True, 

99 ) 

100 return gradient.reshape(ctx.input_shape), None, None 

101 

102 

103def _situ_fused(packed: torch.Tensor, beta: float, linear_beta: float) -> torch.Tensor: 

104 return _SituAutograd.apply(packed, beta, linear_beta) 

105 

106 

107def situ( 

108 gate: torch.Tensor, 

109 up: torch.Tensor, 

110 beta: float = 1.0, 

111 linear_beta: Optional[float] = None, 

112 backend: str = "eager", 

113) -> torch.Tensor: 

114 """Apply SiTU to separate gate and up projections. 

115 

116 Args: 

117 gate: Gate projection with at least one dimension. 

118 up: Up projection with the same shape, dtype, and device as gate. 

119 beta: Positive gate saturation scale. 

120 linear_beta: Up saturation scale; nonpositive values or None disable saturation. 

121 backend: ``eager`` or ``ascendc`` (the fused ops-nn implementation). 

122 

123 Returns: 

124 Activated values with gate's shape and dtype. 

125 """ 

126 # Packing would lose the original split if gate and up had unequal widths. 

127 if gate.shape != up.shape: 

128 raise ValueError("SiTU gate and up must have identical shapes") 

129 if backend == "eager": 

130 return _situ_eager(gate, up, beta, linear_beta) 

131 return situ_packed(torch.cat((gate, up), dim=-1), beta, linear_beta, backend) 

132 

133 

134def situ_packed( 

135 packed: torch.Tensor, 

136 beta: float = 1.0, 

137 linear_beta: Optional[float] = None, 

138 backend: str = "eager", 

139) -> torch.Tensor: 

140 """Apply SiTU to a tensor with gate channels followed by up channels. 

141 

142 Args: 

143 packed: Tensor of shape [..., 2H]; noncontiguous inputs are supported. 

144 beta: Positive gate saturation scale. 

145 linear_beta: Up saturation scale; nonpositive values or None disable saturation. 

146 backend: ``eager`` uses FP32 intermediates. ``ascendc`` calls ops-nn 

147 on NPU with first-order autograd; native errors propagate unchanged. 

148 

149 Raises: 

150 ImportError: The optional CANN package cannot load or lacks a training API. 

151 

152 Returns: 

153 Tensor of shape [..., H] in the input dtype. Empty inputs keep connected 

154 gradients through tensor operations without launching a native kernel. 

155 """ 

156 if backend not in ("eager", "ascendc"): 

157 raise ValueError("SiTU backend must be 'eager' or 'ascendc'") 

158 linear_beta = 0.0 if linear_beta is None else linear_beta 

159 if backend == "eager" or packed.numel() == 0: 

160 gate, up = packed.unflatten(-1, (2, packed.shape[-1] // 2)).unbind(-2) 

161 return _situ_eager(gate, up, beta, linear_beta) 

162 return _situ_fused(packed, beta, linear_beta)