1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
16
17# pylint: disable=missing-public-type-hints,missing-public-docstring,disallowed-name
18# pylint: disable=invalid-name,missing-module-docstring,missing-function-docstring
19# pylint: disable=unused-variable,too-many-nested-blocks
20
21from typing import Optional, Tuple
22
23import torch
24import triton
25import triton.language as tl
26
27from .utils import prepare_chunk_indices, exp, prepare_chunk_offsets
28
29
30@triton.heuristics({
31 'USE_G': lambda args: args['g'] is not None,
32 'USE_G_GAMMA': lambda args: args['g_gamma'] is not None,
33 'USE_DW': lambda args: args['dw'] is not None,
34 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
35})
36@triton.jit(do_not_specialize=['T'])
37def chunk_bwd_kernel_dqkwg(
38 q,
39 k,
40 v,
41 h,
42 g,
43 g_gamma,
44 do,
45 dh,
46 dq,
47 dk,
48 dg,
49 w,
50 dv,
51 dw,
52 cu_seqlens,
53 chunk_indices,
54 scale,
55 B: tl.constexpr,
56 T,
57 H: tl.constexpr,
58 K: tl.constexpr,
59 V: tl.constexpr,
60 BT: tl.constexpr,
61 BK: tl.constexpr,
62 BV: tl.constexpr,
63 USE_G: tl.constexpr,
64 USE_G_GAMMA: tl.constexpr,
65 USE_DW: tl.constexpr,
66 IS_VARLEN: tl.constexpr,
67 gdiff,
68):
69 i_t, i_b = tl.program_id(0), tl.program_id(1)
70 T_max = T
71 if IS_VARLEN:
72 i_tg = i_t
73 i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
74 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
75 total = B * T_max
76 T = eos - bos
77 else:
78 NT = tl.cdiv(T, BT)
79 i_tg = i_b * NT + i_t
80 bos, eos = i_b * T, i_b * T + T
81 total = B * T_max
82
83 NK = tl.cdiv(K, BK)
84 for i_k in range(NK):
85 if USE_G:
86 dg_k = dg + i_k * total * H
87
88 for i_h in range(H):
89 v_h = v + (bos * H + i_h) * V
90 do_h = do + (bos * H + i_h) * V
91 h_h = h + (i_tg * H + i_h).to(tl.int64) * K * V
92 dh_h = dh + (i_tg * H + i_h).to(tl.int64) * K * V
93 q_h = q + (bos * H + i_h) * K
94 k_h = k + (bos * H + i_h) * K
95 dq_h = dq + (bos * H + i_h) * K
96 dk_h = dk + (bos * H + i_h) * K
97
98 if USE_DW:
99 w_h = w + (bos * H + i_h) * K
100 dw_h = dw + (bos * H + i_h) * K
101 dv_h = dv + (bos * H + i_h) * V
102
103 if USE_G:
104 if IS_VARLEN:
105 dg_h = dg_k + i_h * T_max + bos
106 g_h = g + i_h * T_max + bos
107 else:
108 dg_h = dg_k + (i_b * H + i_h) * T_max
109 g_h = g + (i_b * H + i_h) * T_max
110 b_dg_last = tl.zeros([1, ], dtype=tl.float32)
111
112 if USE_G_GAMMA:
113 b_gamma = tl.load(g_gamma + i_h)
114 b_g = b_gamma * (tl.arange(0, BT) + 1)
115 b_g_last = b_gamma * min(BT, T - i_t * BT)
116
117 b_dq = tl.zeros([BT, BK], dtype=tl.float32)
118 b_dk = tl.zeros([BT, BK], dtype=tl.float32)
119 b_ds = tl.zeros([BT, BT], dtype=tl.float32)
120 b_dw = tl.zeros([BT, BK], dtype=tl.float32) if USE_DW else None
121
122 for i_v in range(tl.cdiv(V, BV)):
123 p_v = tl.make_block_ptr(v_h, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
124 p_do = tl.make_block_ptr(do_h, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
125 p_h = tl.make_block_ptr(h_h, (V, K), (1, V), (i_v * BV, i_k * BK), (BV, BK), (0, 1))
126 p_dh = tl.make_block_ptr(dh_h, (V, K), (1, V), (i_v * BV, i_k * BK), (BV, BK), (0, 1))
127
128 b_v = tl.load(p_v, boundary_check=(0, 1))
129 b_do = tl.load(p_do, boundary_check=(0, 1))
130 b_h = tl.load(p_h, boundary_check=(0, 1))
131 b_dh = tl.load(p_dh, boundary_check=(0, 1))
132
133 if USE_G:
134 b_dg_last += (tl.sum(b_h * b_dh))
135
136 b_ds += tl.dot(b_do, tl.trans(b_v))
137 b_dq += tl.dot(b_do, b_h.to(b_do.dtype))
138 b_dk += tl.dot(b_v, b_dh.to(b_v.dtype))
139
140 if USE_DW:
141 p_dv = tl.make_block_ptr(dv_h, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
142 b_dv = tl.load(p_dv, boundary_check=(0, 1))
143 b_dw += tl.dot(b_dv.to(b_v.dtype), b_h.to(b_v.dtype))
144
145 if USE_DW:
146 p_dw = tl.make_block_ptr(dw_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
147 tl.store(p_dw, -b_dw.to(p_dw.dtype.element_ty), boundary_check=(0, 1))
148
149 tl.debug_barrier()
150
151 p_q = tl.make_block_ptr(q_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
152 p_k = tl.make_block_ptr(k_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
153 b_q = tl.load(p_q, boundary_check=(0, 1))
154 b_k = tl.load(p_k, boundary_check=(0, 1))
155
156 p_dq = tl.make_block_ptr(dq_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
157 p_dk = tl.make_block_ptr(dk_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
158
159 o_t = i_t * BT + tl.arange(0, BT)
160 m_t = o_t < T
161 m_A = (o_t[:, None] >= o_t[None, :]) & (m_t[:, None] & m_t)
162
163 if USE_G:
164 b_dg = tl.zeros([BT, ], dtype=tl.float32)
165 p_g = tl.make_block_ptr(g_h, (T,), (1,), (i_t * BT,), (BT,), (0,))
166 b_g = tl.load(p_g, boundary_check=(0,))
167 b_g_last = tl.load(g_h + (min(i_t * BT + BT, T) - 1) * 1)
168 b_dg_last *= tl.exp(b_g_last)
169
170 b_dq = b_dq * tl.exp(b_g)[:, None] * scale
171 b_dg += tl.sum(b_dq * b_q, axis=1)
172
173 b_dk = b_dk * tl.where(m_t, tl.exp(-b_g + b_g_last), 0)[:, None]
174 b_dg -= tl.sum(b_k * b_dk, axis=1)
175 b_dg_last += tl.sum(b_dk * b_k)
176
177 if IS_VARLEN:
178 b_ds = tl.where(m_A, b_ds * exp(b_g[:, None] - b_g[None, :]), 0) * scale
179 else:
180 p_gdiff = tl.make_block_ptr(gdiff + i_b * H * NT * BT * BT + i_h * NT * BT * BT + i_t * BT * BT,
181 (BT, BT), (BT, 1), (0, 0), (BT, BT), (1, 0))
182 gdiff_ = tl.load(p_gdiff)
183 b_ds = b_ds * gdiff_ * scale
184
185 b_ds2 = b_ds * tl.dot(b_q, tl.trans(b_k))
186 b_dg += tl.sum(b_ds2, axis=1)
187 b_dg -= tl.sum(b_ds2, axis=0)
188
189 b_ds = b_ds.to(b_k.dtype)
190 b_dq += tl.dot(b_ds, b_k)
191 b_dk += tl.dot(tl.trans(b_ds), b_q)
192 p_dg = tl.make_block_ptr(dg_h, (T,), (1,), (i_t * BT,), (BT,), (0,))
193
194 last_index_local = min(BT, T - i_t * BT) - 1
195 if last_index_local >= 0:
196 is_last_mask = tl.arange(0, BT) == last_index_local
197 b_dg = tl.where(is_last_mask, b_dg + b_dg_last, b_dg)
198 else:
199 pass
200
201 tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1))
202 tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1))
203 tl.store(p_dg, b_dg.to(p_dg.dtype.element_ty), boundary_check=(0,))
204
205 elif USE_G_GAMMA:
206 b_dq = b_dq * exp(b_g)[:, None] * scale
207 b_dk = b_dk * tl.where(m_t, exp(-b_g + b_g_last), 0)[:, None]
208 b_ds = tl.where(m_A, b_ds * exp(b_g[:, None] - b_g[None, :]), 0) * scale
209 b_ds = b_ds.to(b_k.dtype)
210 b_dq += tl.dot(b_ds, b_k)
211 b_dk += tl.dot(tl.trans(b_ds), b_q)
212 tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1))
213 tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1))
214
215 else:
216 b_ds = tl.where(m_A, b_ds, 0)
217 b_ds = b_ds.to(b_k.dtype)
218 b_dq += tl.dot(b_ds, b_k)
219 b_dk += tl.dot(tl.trans(b_ds), b_q) * scale
220 b_dq *= scale
221 tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1))
222 tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1))
223
224
225@triton.heuristics({
226 'USE_G': lambda args: args['g'] is not None,
227 'USE_G_GAMMA': lambda args: args['g_gamma'] is not None,
228 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
229})
230@triton.jit(do_not_specialize=['T'])
231def chunk_bwd_kernel_dv_local(
232 q,
233 k,
234 g,
235 g_gamma,
236 do,
237 dv,
238 cu_seqlens,
239 chunk_indices,
240 scale,
241 T,
242 H: tl.constexpr,
243 K: tl.constexpr,
244 V: tl.constexpr,
245 BT: tl.constexpr,
246 BK: tl.constexpr,
247 BV: tl.constexpr,
248 USE_G: tl.constexpr,
249 USE_G_GAMMA: tl.constexpr,
250 IS_VARLEN: tl.constexpr,
251):
252 i_t, i_b = tl.program_id(0), tl.program_id(1)
253 T_max = T
254
255 if IS_VARLEN:
256 i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32)
257 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
258 T = eos - bos
259 else:
260 bos, eos = i_b * T, i_b * T + T
261
262 for i_h in range(H):
263 offset_kh = (bos * H + i_h) * K
264 offset_vh = (bos * H + i_h) * V
265
266 b_A = tl.zeros([BT, BT], dtype=tl.float32)
267 for i_k in range(tl.cdiv(K, BK)):
268 p_k = tl.make_block_ptr(k + offset_kh, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0))
269 p_q = tl.make_block_ptr(q + offset_kh, (K, T), (1, H * K), (i_k * BK, i_t * BT), (BK, BT), (0, 1))
270 b_q = tl.load(p_q, boundary_check=(0, 1))
271 b_k = tl.load(p_k, boundary_check=(0, 1))
272 b_A += tl.dot(b_k, b_q)
273
274 if USE_G:
275 if IS_VARLEN:
276 offset_g = i_h * T_max + bos
277 else:
278 offset_g = i_b * H * T_max + i_h * T_max
279
280 p_g = tl.make_block_ptr(g + offset_g, (T,), (1,), (i_t * BT,), (BT,), (0,))
281 b_g = tl.load(p_g, boundary_check=(0,))
282
283 if USE_G_GAMMA:
284 b_gamma = tl.load(g_gamma + i_h)
285 b_g = b_gamma * (tl.arange(0, BT) + 1)
286
287 o_t = i_t * BT + tl.arange(0, BT)
288 m_t = o_t < T
289 m_A = (o_t[:, None] <= o_t[None, :]) & (m_t[:, None] & m_t)
290
291 if USE_G:
292 b_A = tl.where(m_A, b_A * tl.exp(b_g[None, :] - b_g[:, None]) * scale, 0).to(do.dtype.element_ty)
293 else:
294 b_A = tl.where(m_A, b_A * scale, 0).to(do.dtype.element_ty)
295
296 for i_v in range(tl.cdiv(V, BV)):
297 p_do = tl.make_block_ptr(do + offset_vh, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
298 p_dv = tl.make_block_ptr(dv + offset_vh, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
299 b_do = tl.load(p_do, boundary_check=(0, 1))
300 b_dv = tl.dot(b_A.to(b_do.dtype), b_do)
301 tl.store(p_dv, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1))
302
303
304@triton.heuristics({
305 'USE_G': lambda args: args['g'] is not None,
306 'USE_G_GAMMA': lambda args: args['g_gamma'] is not None,
307 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None
308})
309@triton.jit(do_not_specialize=['T'])
310def chunk_fwd_kernel_o(
311 q,
312 k,
313 v,
314 h,
315 g,
316 g_gamma,
317 o,
318 cu_seqlens,
319 chunk_offsets,
320 scale,
321 T,
322 H: tl.constexpr,
323 N: tl.constexpr,
324 Hg: tl.constexpr,
325 K: tl.constexpr,
326 V: tl.constexpr,
327 BT: tl.constexpr,
328 BK: tl.constexpr,
329 BV: tl.constexpr,
330 USE_G: tl.constexpr,
331 USE_G_GAMMA: tl.constexpr,
332 IS_VARLEN: tl.constexpr,
333):
334 T_max = T
335 for i_v in range(tl.cdiv(V, BV)):
336 for i_n in range(N):
337 if IS_VARLEN:
338 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
339 cu_seqlens + i_n + 1
340 ).to(tl.int32)
341 T = eos - bos
342 NT = tl.cdiv(T, BT)
343 boh = tl.load(chunk_offsets + i_n).to(tl.int64)
344 else:
345 bos, eos = i_n * T, i_n * T + T
346 NT = tl.cdiv(T, BT)
347 boh = i_n * NT
348
349 core_id = tl.program_id(0)
350 total_cores = tl.num_programs(0)
351 base_chunks_per_pid = NT // total_cores
352 remainder = NT % total_cores
353
354 if core_id < remainder:
355 chunks_this_pid = base_chunks_per_pid + 1
356 start_idx = core_id * chunks_this_pid
357 else:
358 chunks_this_pid = base_chunks_per_pid
359 start_idx = core_id * base_chunks_per_pid + remainder
360
361 # offset calculation
362 for i_h in range(0, H):
363 q_offset = (bos * Hg + i_h // (H // Hg)) * K
364 k_offset = (bos * Hg + i_h // (H // Hg)) * K
365 v_offset = (bos * H + i_h) * V
366 o_offset = (bos * H + i_h) * V
367
368 for i_t in range(start_idx, start_idx + chunks_this_pid):
369 i_tg = boh + i_t
370 h_base = h + (i_tg * H + i_h).to(tl.int64) * K * V
371 b_o = tl.zeros([BT, BV], dtype=tl.float32)
372 b_A = tl.zeros([BT, BT], dtype=tl.float32)
373 for i_k in range(tl.cdiv(K, BK)):
374 p_q = tl.make_block_ptr(
375 q + q_offset, (T, K), (Hg * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)
376 )
377 p_k = tl.make_block_ptr(
378 k + k_offset, (K, T), (1, Hg * K), (i_k * BK, i_t * BT), (BK, BT), (0, 1)
379 )
380 p_h = tl.make_block_ptr(
381 h_base, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0)
382 )
383 b_q = tl.load(p_q, boundary_check=(0, 1))
384 b_k = tl.load(p_k, boundary_check=(0, 1))
385 b_h = tl.load(p_h, boundary_check=(0, 1))
386
387 # [BT, BK] @ [BK, BV] -> [BT, BV]
388 b_o += tl.dot(b_q, b_h)
389 # [BT, BK] @ [BK, BT] -> [BT, BT]
390 b_A += tl.dot(b_q, b_k)
391
392 if USE_G:
393 if IS_VARLEN:
394 p_g = tl.make_block_ptr(g + bos + i_h * T_max, (T,), (1,), (i_t * BT,), (BT,), (0,))
395 else:
396 p_g = tl.make_block_ptr(g + bos * H + i_h * T_max, (T,), (1,), (i_t * BT,), (BT,), (0,))
397 b_g = tl.load(p_g, boundary_check=(0,))
398 b_o = b_o * exp(b_g)[:, None]
399 b_A = b_A * exp(b_g[:, None] - b_g[None, :])
400 if USE_G_GAMMA:
401 b_gamma = tl.load(g_gamma + i_h)
402 b_g = b_gamma * (tl.arange(0, BT) + 1)
403
404 o_i = tl.arange(0, BT)
405 m_A = o_i[:, None] >= o_i[None, :]
406 b_A = tl.where(m_A, b_A, 0)
407
408 p_v = tl.make_block_ptr(
409 v + v_offset, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
410 )
411 p_o = tl.make_block_ptr(
412 o + o_offset, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
413 )
414 b_v = tl.load(p_v, boundary_check=(0, 1))
415
416 # to fix mma -> mma layout conversion
417 # already solved by triton v3.2 or higher
418 b_o = b_o * scale + tl.dot(b_A.to(b_v.dtype), b_v) * scale
419 tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
420
421
422def chunk_bwd_dqkwg(
423 q: torch.Tensor,
424 k: torch.Tensor,
425 v: torch.Tensor,
426 do: torch.Tensor,
427 h: torch.Tensor,
428 dh: torch.Tensor,
429 g: Optional[torch.Tensor] = None,
430 g_gamma: Optional[torch.Tensor] = None,
431 dv: Optional[torch.Tensor] = None,
432 w: Optional[torch.Tensor] = None,
433 cu_seqlens: Optional[torch.LongTensor] = None,
434 chunk_size: int = 64,
435 scale: float = 1.0,
436) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
437 B, T, H, K, V = *k.shape, v.shape[-1]
438 BT = min(chunk_size, max(16, triton.next_power_of_2(T)))
439 chunk_indices = prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None
440 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
441
442 BK = 128 if cu_seqlens is None else 64
443 BV = 64
444 NK = triton.cdiv(K, BK)
445 dq = torch.empty_like(q)
446 dk = torch.empty_like(k)
447 g = g.transpose(1, 2).contiguous()
448 dg = torch.empty(NK, *g.shape, dtype=torch.float32, device=g.device) if g is not None else None
449 dw = torch.empty_like(w) if w is not None else None
450 grid = (NT, B)
451
452 if cu_seqlens is None:
453 if NT * BT == T:
454 g_ = g.reshape(B, H, NT, BT)
455 g_diff = g_[:, :, :, :, None] - g_[:, :, :, None, :]
456 g_diff = g_diff.clamp(-60, 60).exp()
457 g_diff[:, :, :] *= torch.tril(torch.ones(BT, BT), diagonal=0).to(g.device)
458 else:
459 diff = NT * BT - T
460 g_ = torch.cat((g, torch.zeros(B, H, diff).to(g.device)), dim=-1).reshape(B, H, NT, BT)
461 g_diff = g_[:, :, :, :, None] - g_[:, :, :, None, :]
462 g_diff = g_diff.clamp(-60, 60).exp()
463 g_diff[:, :, :] *= torch.tril(torch.ones(BT, BT), diagonal=0).to(g.device)
464 bias = torch.arange(0, BT).to(g.device)
465 o_t = (NT - 1) * BT + bias
466 m_t = o_t < T
467 m_A = (m_t[:, None] & m_t)
468 g_diff[:, :, -1] *= m_A
469 else:
470 g_diff = None
471
472 chunk_bwd_kernel_dqkwg[grid](
473 q=q,
474 k=k,
475 v=v,
476 h=h,
477 g=g,
478 g_gamma=g_gamma,
479 do=do,
480 dh=dh,
481 dv=dv,
482 w=w,
483 dw=dw,
484 dq=dq,
485 dk=dk,
486 dg=dg,
487 cu_seqlens=cu_seqlens,
488 chunk_indices=chunk_indices,
489 scale=scale,
490 B=B,
491 T=T,
492 H=H,
493 K=K,
494 V=V,
495 BT=BT,
496 BK=BK,
497 BV=BV,
498 gdiff=g_diff,
499 )
500
501 if dg is not None:
502 dg = dg.sum(0)
503 dg = dg.transpose(1, 2).contiguous()
504 return dq, dk, dw, dg
505
506
507def chunk_bwd_dv_local(
508 q: torch.Tensor,
509 k: torch.Tensor,
510 do: torch.Tensor,
511 g: Optional[torch.Tensor] = None,
512 g_gamma: Optional[torch.Tensor] = None,
513 scale: float = None,
514 cu_seqlens: Optional[torch.LongTensor] = None,
515 chunk_size: int = 64
516) -> torch.Tensor:
517 B, T, H, K, V = *k.shape, do.shape[-1]
518 BT = min(chunk_size, max(16, triton.next_power_of_2(T)))
519 chunk_indices = prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None
520
521 BK = 128
522 BV = 128
523 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
524
525 g = g.transpose(1, 2).contiguous()
526 dv = torch.empty_like(do)
527 grid = (NT, B)
528 chunk_bwd_kernel_dv_local[grid](
529 q=q,
530 k=k,
531 g=g,
532 g_gamma=g_gamma,
533 do=do,
534 dv=dv,
535 cu_seqlens=cu_seqlens,
536 chunk_indices=chunk_indices,
537 scale=scale,
538 T=T,
539 H=H,
540 K=K,
541 V=V,
542 BT=BT,
543 BK=BK,
544 BV=BV,
545 )
546 return dv
547
548
549def chunk_fwd_o(
550 q: torch.Tensor,
551 k: torch.Tensor,
552 v: torch.Tensor,
553 h: torch.Tensor,
554 g: Optional[torch.Tensor] = None,
555 g_gamma: Optional[torch.Tensor] = None,
556 scale: Optional[float] = None,
557 cu_seqlens: Optional[torch.LongTensor] = None,
558 chunk_size: int = 64
559) -> torch.Tensor:
560 B, T, Hg, K, V = *q.shape, v.shape[-1]
561 H = v.shape[-2]
562 BT = min(chunk_size, max(16, triton.next_power_of_2(T)))
563 chunk_indices = (
564 prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None
565 )
566 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
567 if scale is None:
568 scale = k.shape[-1] ** -0.5
569
570 output = torch.empty_like(v)
571 if cu_seqlens is None:
572 N, chunk_offsets = B, None
573 else:
574 N, chunk_offsets = (
575 len(cu_seqlens) - 1,
576 prepare_chunk_offsets(cu_seqlens, BT),
577 )
578
579 g = g.transpose(1, 2).contiguous()
580 h = h.contiguous()
581 CV_kernel_num = 24
582 chunk_fwd_kernel_o[(CV_kernel_num,)](
583 q,
584 k,
585 v,
586 h,
587 g,
588 g_gamma,
589 output,
590 cu_seqlens,
591 chunk_offsets,
592 scale,
593 T=T,
594 H=H,
595 N=N,
596 Hg=Hg,
597 K=K,
598 V=V,
599 BT=BT,
600 BK=128,
601 BV=128,
602 )
603 return output
604
605bwd_chunk_dqkwg = chunk_bwd_dqkwg
606bwd_chunk_dv_local = chunk_bwd_dv_local