Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / activation_situ.py: 0%
56 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""SiTU eager reference and first-order autograd over CANN ops-nn."""
17import importlib
18from collections.abc import Callable
19from typing import Any, Optional
21# Torch is the native framework; the legacy platform-agnostic lint rule is obsolete here.
22import torch
23from torch.autograd.function import once_differentiable
26def _situ_eager(gate: torch.Tensor, up: torch.Tensor, beta: float,
27 linear_beta: Optional[float]) -> torch.Tensor:
28 gate_float, up_float = gate.float(), up.float()
29 activated_gate = beta * torch.tanh(gate_float / beta) * torch.sigmoid(gate_float)
30 if linear_beta is not None and linear_beta > 0:
31 up_float = linear_beta * torch.tanh(up_float / linear_beta)
32 return (activated_gate * up_float).to(gate.dtype)
35def _load_situ_ops() -> tuple[Callable, Callable]:
36 """Resolve the optional ops-nn training APIs without hiding native errors."""
37 try:
38 # CANN is optional for CPU and eager-only callers.
39 ops = importlib.import_module("cann_ops_nn")
40 except (ImportError, OSError) as error:
41 raise ImportError(
42 "SiTU backend='ascendc' could not load cann_ops_nn. Install the standalone ops-nn Python package "
43 "with compatible PyTorch, torch-npu and CANN libraries. "
44 f"See hyper_parallel/components/functional/README.md. Original error: {error}"
45 ) from error
46 situ_glu = getattr(ops, "situ_glu", None)
47 situ_glu_grad = getattr(ops, "situ_glu_grad", None)
48 if not callable(situ_glu) or not callable(situ_glu_grad):
49 raise ImportError(
50 "SiTU backend='ascendc' requires cann_ops_nn.situ_glu and cann_ops_nn.situ_glu_grad. "
51 "Install a package containing both training APIs and build both native operators. "
52 "See hyper_parallel/components/functional/README.md."
53 )
54 return situ_glu, situ_glu_grad
57class _SituAutograd(torch.autograd.Function):
58 """Connect ops-nn's separate SituGlu and SituGluGrad entry points."""
60 @staticmethod
61 def forward(ctx: Any, packed: torch.Tensor, beta: float, linear_beta: float) -> torch.Tensor:
62 """Run dense gate-first rows and retain the native backward inputs.
64 Args:
65 ctx: Autograd context holding inputs for backward.
66 packed: Input tensor with gate channels followed by up channels.
67 beta: Gate saturation scale.
68 linear_beta: Up saturation scale; nonpositive values disable saturation.
70 Returns:
71 Activated tensor with half the input's last dimension.
72 """
73 situ_glu, situ_glu_grad = _load_situ_ops()
74 rows = packed.reshape(-1, packed.shape[-1]).contiguous()
75 ctx.save_for_backward(rows)
76 ctx.input_shape = packed.shape
77 ctx.beta = beta
78 ctx.linear_beta = linear_beta
79 ctx.situ_glu_grad = situ_glu_grad
80 output = situ_glu(rows, dim=-1, beta=beta, linear_beta=linear_beta, activate_left=True)
81 return output.reshape(*packed.shape[:-1], packed.shape[-1] // 2)
83 @staticmethod
84 @once_differentiable
85 def backward(ctx: Any, grad_output: torch.Tensor) -> tuple[torch.Tensor, None, None]:
86 """Restore the packed gradient with the same saturation parameters.
88 Args:
89 ctx: Autograd context populated by forward.
90 grad_output: Gradient with respect to the activated output.
92 Returns:
93 Packed input gradient and no gradients for the scalar attributes.
94 """
95 (rows,) = ctx.saved_tensors
96 grad_rows = grad_output.reshape(-1, rows.shape[-1] // 2).contiguous()
97 gradient = ctx.situ_glu_grad(
98 grad_rows, rows, dim=-1, beta=ctx.beta, linear_beta=ctx.linear_beta, activate_left=True,
99 )
100 return gradient.reshape(ctx.input_shape), None, None
103def _situ_fused(packed: torch.Tensor, beta: float, linear_beta: float) -> torch.Tensor:
104 return _SituAutograd.apply(packed, beta, linear_beta)
107def situ(
108 gate: torch.Tensor,
109 up: torch.Tensor,
110 beta: float = 1.0,
111 linear_beta: Optional[float] = None,
112 backend: str = "eager",
113) -> torch.Tensor:
114 """Apply SiTU to separate gate and up projections.
116 Args:
117 gate: Gate projection with at least one dimension.
118 up: Up projection with the same shape, dtype, and device as gate.
119 beta: Positive gate saturation scale.
120 linear_beta: Up saturation scale; nonpositive values or None disable saturation.
121 backend: ``eager`` or ``ascendc`` (the fused ops-nn implementation).
123 Returns:
124 Activated values with gate's shape and dtype.
125 """
126 # Packing would lose the original split if gate and up had unequal widths.
127 if gate.shape != up.shape:
128 raise ValueError("SiTU gate and up must have identical shapes")
129 if backend == "eager":
130 return _situ_eager(gate, up, beta, linear_beta)
131 return situ_packed(torch.cat((gate, up), dim=-1), beta, linear_beta, backend)
134def situ_packed(
135 packed: torch.Tensor,
136 beta: float = 1.0,
137 linear_beta: Optional[float] = None,
138 backend: str = "eager",
139) -> torch.Tensor:
140 """Apply SiTU to a tensor with gate channels followed by up channels.
142 Args:
143 packed: Tensor of shape [..., 2H]; noncontiguous inputs are supported.
144 beta: Positive gate saturation scale.
145 linear_beta: Up saturation scale; nonpositive values or None disable saturation.
146 backend: ``eager`` uses FP32 intermediates. ``ascendc`` calls ops-nn
147 on NPU with first-order autograd; native errors propagate unchanged.
149 Raises:
150 ImportError: The optional CANN package cannot load or lacks a training API.
152 Returns:
153 Tensor of shape [..., H] in the input dtype. Empty inputs keep connected
154 gradients through tensor operations without launching a native kernel.
155 """
156 if backend not in ("eager", "ascendc"):
157 raise ValueError("SiTU backend must be 'eager' or 'ascendc'")
158 linear_beta = 0.0 if linear_beta is None else linear_beta
159 if backend == "eager" or packed.numel() == 0:
160 gate, up = packed.unflatten(-1, (2, packed.shape[-1] // 2)).unbind(-2)
161 return _situ_eager(gate, up, beta, linear_beta)
162 return _situ_fused(packed, beta, linear_beta)