1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
16
17# pylint: disable=line-too-long,missing-public-type-hints,missing-public-docstring
18# pylint: disable=unused-argument,invalid-name,missing-module-docstring
19# pylint: disable=missing-function-docstring
20
21from typing import Optional
22
23import torch
24import triton
25import triton.language as tl
26
27from .utils import prepare_chunk_indices
28
29
30@triton.heuristics({
31 'USE_G': lambda args: args['g'] is not None,
32 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
33})
34@triton.jit(do_not_specialize=['T', 'NT', 'TOTAL_TASKS'])
35def chunk_scaled_dot_kkt_fwd_kernel(
36 k,
37 g,
38 beta,
39 A,
40 cu_seqlens,
41 chunk_indices,
42 T,
43 H: tl.constexpr,
44 K: tl.constexpr,
45 BT: tl.constexpr,
46 BK: tl.constexpr,
47 IS_VARLEN: tl.constexpr,
48 USE_G: tl.constexpr,
49 NT,
50 B,
51 TOTAL_TASKS,
52):
53 core_id = tl.program_id(0)
54 num_blocks = tl.num_programs(0)
55 T_max = T
56
57 base_tasks_per_block = TOTAL_TASKS // num_blocks
58 remainder_tasks = TOTAL_TASKS % num_blocks
59
60 if core_id < remainder_tasks:
61 tasks_this_core = base_tasks_per_block + 1
62 start_idx = core_id * tasks_this_core
63 else:
64 tasks_this_core = base_tasks_per_block
65 start_idx = core_id * base_tasks_per_block + remainder_tasks
66
67 for idx in range(start_idx, start_idx + tasks_this_core):
68 i_b = idx // NT
69 local_idx = idx % NT
70
71 if IS_VARLEN:
72 i_n = tl.load(chunk_indices + local_idx * 2).to(tl.int32)
73 i_t = tl.load(chunk_indices + local_idx * 2 + 1).to(tl.int32)
74 bos = tl.load(cu_seqlens + i_n).to(tl.int32)
75 eos = tl.load(cu_seqlens + i_n + 1).to(tl.int32)
76 T_local = eos - bos
77 else:
78 bos, eos = 0, T
79 i_t = local_idx
80 T_local = T
81
82 for i_h in range(H):
83 k_batch_off = i_b * T_max * H * K
84 beta_batch_off = i_b * H * T_max
85 g_batch_off = i_b * H * T_max
86 A_batch_off = i_b * T_max * H * BT
87
88 p_beta = tl.make_block_ptr(beta + beta_batch_off + bos + i_h * T_max, (T_local,), (1,), (i_t * BT,), (BT,), (0,))
89 b_beta = tl.load(p_beta, boundary_check=(0,))
90
91 b_A = tl.zeros([BT, BT], dtype=tl.float32)
92 for i_k in range(tl.cdiv(K, BK)):
93 p_k = tl.make_block_ptr(k + k_batch_off + (bos * H + i_h) * K, (T_local, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
94 b_k = tl.load(p_k, boundary_check=(0, 1))
95 dot_product = tl.dot(b_k, tl.trans(b_k))
96
97 o_t = i_t * BT + tl.arange(0, BT)
98 o_t = o_t.to(tl.float32)
99 T_mask = (o_t < T_local).to(tl.float32)
100
101 row_indices = tl.arange(0, BT)[:, None]
102 col_indices = tl.arange(0, BT)[None, :]
103 tril_mask = (row_indices > col_indices).to(tl.float32)
104 tril_mask = tril_mask * T_mask[:, None]
105 masked_dot = dot_product * tril_mask
106 b_A += masked_dot
107
108 if USE_G:
109 p_g = tl.make_block_ptr(g + g_batch_off + bos + i_h * T_max, (T_local,), (1,), (i_t * BT,), (BT,), (0,))
110 b_g = tl.load(p_g, boundary_check=(0,))
111 b_g_diff = b_g[:, None] - b_g[None, :]
112 b_g_diff = tl.minimum(tl.maximum(b_g_diff, -50.0), 50.0)
113 b_A *= tl.exp(b_g_diff)
114 b_A *= b_beta[:, None]
115
116 p_A = tl.make_block_ptr(A + A_batch_off + (bos * H + i_h) * BT, (T_local, BT), (BT * H, 1), (i_t * BT, 0), (BT, BT), (1, 0))
117 tl.store(p_A, b_A.to(p_A.dtype.element_ty), boundary_check=(0, 1))
118
119
120@triton.heuristics({
121 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None
122})
123@triton.autotune(
124 configs=[
125 triton.Config({'BK': BK})
126 for BK in [32, 64]
127 ],
128 key=["BC"]
129)
130@triton.jit(do_not_specialize=['T'])
131def chunk_scaled_dot_kkt_fwd_kernel_intra_sub_inter(
132 k,
133 g,
134 beta,
135 A,
136 cu_seqlens,
137 chunk_indices,
138 T,
139 H: tl.constexpr,
140 K: tl.constexpr,
141 BT: tl.constexpr,
142 BC: tl.constexpr,
143 BK: tl.constexpr,
144 NC: tl.constexpr,
145 IS_VARLEN: tl.constexpr,
146):
147 i_t, i_c, i_b = tl.program_id(0), tl.program_id(1), tl.program_id(2)
148 i_i, i_j = i_c // NC, i_c % NC
149
150 for i_h in range(H):
151 if IS_VARLEN:
152 i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
153 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
154 T_val = eos - bos
155 else:
156 bos, eos = i_b * T, i_b * T + T
157 T_val = T
158
159 should_compute = (i_t * BT + i_i * BC < T_val) and (i_i > i_j)
160
161 if should_compute:
162 k_ptr = k + (bos * H + i_h) * K
163 g_ptr = g + (bos * H + i_h) * K
164 A_ptr = A + (bos * H + i_h) * BT
165
166 p_beta = tl.make_block_ptr(beta + bos * H + i_h, (T_val,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,))
167 b_beta = tl.load(p_beta, boundary_check=(0,))
168
169 b_A = tl.zeros([BC, BC], dtype=tl.float32)
170 for i_k in range(tl.cdiv(K, BK)):
171 p_k = tl.make_block_ptr(k_ptr, (T_val, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK),
172 (1, 0))
173 p_g = tl.make_block_ptr(g_ptr, (T_val, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK),
174 (1, 0))
175 b_kt = tl.make_block_ptr(k_ptr, (K, T_val), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC),
176 (0, 1))
177 p_gk = tl.make_block_ptr(g_ptr, (K, T_val), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC),
178 (0, 1))
179
180 o_k = i_k * BK + tl.arange(0, BK)
181 m_k = o_k < K
182 b_gn = tl.load(g_ptr + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0)
183 b_g = tl.load(p_g, boundary_check=(0, 1))
184 b_k = tl.load(p_k, boundary_check=(0, 1)) * tl.exp(b_g - b_gn[None, :])
185 b_gk = tl.load(p_gk, boundary_check=(0, 1))
186 b_kt = tl.load(b_kt, boundary_check=(0, 1)) * tl.exp(b_gn[:, None] - b_gk)
187 b_A += tl.dot(b_k, b_kt)
188 b_A *= b_beta[:, None]
189
190 p_A = tl.make_block_ptr(A_ptr, (T_val, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0))
191 tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1))
192
193
194@triton.heuristics({
195 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None
196})
197@triton.jit(do_not_specialize=['T'])
198def chunk_scaled_dot_kkt_fwd_kernel_intra_sub_intra(
199 k,
200 g,
201 beta,
202 A,
203 cu_seqlens,
204 chunk_indices,
205 T,
206 H: tl.constexpr,
207 K: tl.constexpr,
208 BT: tl.constexpr,
209 BC: tl.constexpr,
210 BK: tl.constexpr,
211 IS_VARLEN: tl.constexpr,
212):
213 i_t, i_i, i_b = tl.program_id(0), tl.program_id(1), tl.program_id(2)
214
215 for i_h in range(H):
216 if IS_VARLEN:
217 i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
218 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
219 T_val = eos - bos
220 else:
221 bos, eos = i_b * T, i_b * T + T
222 T_val = T
223
224 should_compute = i_t * BT + i_i * BC < T_val
225
226 if should_compute:
227 o_i = tl.arange(0, BC)
228 o_k = tl.arange(0, BK)
229 m_k = o_k < K
230 m_A = (i_t * BT + i_i * BC + o_i) < T_val
231 o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC
232
233 p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T_val, K), (H * K, 1), (i_t * BT + i_i * BC, 0), (BC, BK),
234 (1, 0))
235 p_g = tl.make_block_ptr(g + (bos * H + i_h) * K, (T_val, K), (H * K, 1), (i_t * BT + i_i * BC, 0), (BC, BK),
236 (1, 0))
237 p_beta = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h
238
239 b_k = tl.load(p_k, boundary_check=(0, 1)) * tl.load(p_beta, mask=m_A, other=0)[:, None]
240 b_g = tl.load(p_g, boundary_check=(0, 1))
241
242 p_kt = k + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k
243 p_gk = g + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k
244
245 for j in range(0, min(BC, T_val - i_t * BT - i_i * BC)):
246 b_kt = tl.load(p_kt, mask=m_k, other=0).to(tl.float32)
247 b_gk = tl.load(p_gk, mask=m_k, other=0).to(tl.float32)
248 b_A = tl.sum(b_k * b_kt[None, :] * tl.exp(b_g - b_gk[None, :]), 1)
249 # 转化成f32
250 o_i_tmp = o_i.to(tl.float32)
251 b_A = tl.where(o_i_tmp > j, b_A, 0.)
252
253 tl.store(A + o_A + j, b_A, mask=m_A)
254 p_kt += H * K
255 p_gk += H * K
256
257
258def chunk_scaled_dot_kkt_fwd(
259 k: torch.Tensor,
260 g: Optional[torch.Tensor] = None,
261 gk: Optional[torch.Tensor] = None,
262 beta: Optional[torch.Tensor] = None,
263 cu_seqlens: Optional[torch.LongTensor] = None,
264 chunk_size: int = 64,
265 output_dtype: torch.dtype = torch.float32
266) -> torch.Tensor:
267 r"""
268 Compute beta * K * K^T.
269
270 Args:
271 k (torch.Tensor):
272 The key tensor of shape `[B, T, H, K]`.
273 beta (torch.Tensor):
274 The beta tensor of shape `[B, T, H]`.
275 g (torch.Tensor):
276 The cumulative sum of the gate tensor of shape `[B, T, H]`. Default: `None`.
277 gk (torch.Tensor):
278 The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`.
279 cu_seqlens (torch.LongTensor):
280 The cumulative sequence lengths of the input tensor.
281 Default: None
282 chunk_size (int):
283 The chunk size. Default: 64.
284 output_dtype (torch.dtype):
285 The dtype of the output tensor. Default: `torch.float32`
286
287 Returns:
288 beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size.
289 """
290 B, T, H, K = k.shape
291 BT = chunk_size
292 chunk_indices = prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None
293 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
294 beta = beta.transpose(1, 2).contiguous()
295 g = g.transpose(1, 2).contiguous()
296 BK = 128
297 kernel_num = 24
298
299 if gk is None:
300 A = torch.empty(B, T, H, BT, device=k.device, dtype=output_dtype)
301 chunk_scaled_dot_kkt_fwd_kernel[(kernel_num,)](
302 k=k,
303 g=g,
304 beta=beta,
305 A=A,
306 cu_seqlens=cu_seqlens,
307 chunk_indices=chunk_indices,
308 T=T,
309 H=H,
310 K=K,
311 BT=BT,
312 BK=BK,
313 NT=NT,
314 B=B,
315 TOTAL_TASKS=B * NT,
316 )
317 return A
318
319 BC = min(16, BT)
320 NC = triton.cdiv(BT, BC)
321 BK = max(triton.next_power_of_2(K), 16)
322 A = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype)
323 grid = (NT, NC * NC, B)
324 chunk_scaled_dot_kkt_fwd_kernel_intra_sub_inter[grid](
325 k=k,
326 g=gk,
327 beta=beta,
328 A=A,
329 cu_seqlens=cu_seqlens,
330 chunk_indices=chunk_indices,
331 T=T,
332 H=H,
333 K=K,
334 BT=BT,
335 BC=BC,
336 NC=NC,
337 )
338
339 grid = (NT, NC, B)
340 chunk_scaled_dot_kkt_fwd_kernel_intra_sub_intra[grid](
341 k=k,
342 g=gk,
343 beta=beta,
344 A=A,
345 cu_seqlens=cu_seqlens,
346 chunk_indices=chunk_indices,
347 T=T,
348 H=H,
349 K=K,
350 BT=BT,
351 BC=BC,
352 BK=BK,
353 )
354 return A