1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
16
17# pylint: disable=line-too-long,missing-public-type-hints,missing-public-docstring
18# pylint: disable=used-before-assignment,unsupported-binary-operation,unused-argument
19# pylint: disable=invalid-name,missing-module-docstring,missing-function-docstring
20
21from typing import Optional, Tuple
22
23import torch
24import triton
25import triton.language as tl
26
27from .utils import prepare_chunk_indices, prepare_chunk_offsets, get_autotune_config, get_npu_properties
28
29CUBE_CORE_NUM = get_npu_properties()['num_aicore']
30
31
32@triton.heuristics({
33 'USE_G': lambda args: args['g'] is not None,
34 'USE_GK': lambda args: args['gk'] is not None,
35 'USE_INITIAL_STATE': lambda args: args['h0'] is not None,
36 'STORE_FINAL_STATE': lambda args: args['ht'] is not None,
37 'SAVE_NEW_VALUE': lambda args: args['v_new'] is not None,
38 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
39})
40@triton.autotune(
41 configs=get_autotune_config(multibuffer_list=(False,)),
42 key=['H', 'K', 'V', 'BT'],
43)
44@triton.jit(do_not_specialize=['T'])
45def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
46 k,
47 v,
48 w,
49 v_new,
50 g,
51 gk,
52 h,
53 h0,
54 ht,
55 cu_seqlens,
56 chunk_offsets,
57 T,
58 H: tl.constexpr,
59 K: tl.constexpr,
60 V: tl.constexpr,
61 BT: tl.constexpr,
62 BV: tl.constexpr,
63 NT: tl.constexpr,
64 USE_G: tl.constexpr,
65 USE_GK: tl.constexpr,
66 USE_INITIAL_STATE: tl.constexpr,
67 STORE_FINAL_STATE: tl.constexpr,
68 SAVE_NEW_VALUE: tl.constexpr,
69 IS_VARLEN: tl.constexpr,
70):
71 T_all = T
72 NT_all = NT
73 i_v, i_nh = tl.program_id(0), tl.program_id(1)
74 i_n, i_h = i_nh // H, i_nh % H
75 if IS_VARLEN:
76 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
77 T = eos - bos
78 NT = tl.cdiv(T, BT)
79 boh = tl.load(chunk_offsets + i_n).to(tl.int32)
80 else:
81 bos, eos = i_n * T, i_n * T + T
82 NT = tl.cdiv(T, BT)
83 boh = i_n * NT
84
85 # Initialize hidden states
86 b_h1 = tl.zeros([64, BV], dtype=tl.float32)
87 if K > 64:
88 b_h2 = tl.zeros([64, BV], dtype=tl.float32)
89 if K > 128:
90 b_h3 = tl.zeros([64, BV], dtype=tl.float32)
91 if K > 192:
92 b_h4 = tl.zeros([64, BV], dtype=tl.float32)
93
94 if IS_VARLEN:
95 v = v + (i_h * T_all + bos) * V
96 k = k + (i_h * T_all + bos) * K
97 w = w + (i_h * T_all + bos) * K
98 if USE_G:
99 g = g + i_h * T_all + bos
100 h = h + (i_h * NT_all + boh) * K * V
101 if SAVE_NEW_VALUE:
102 v_new_base = v_new + (i_h * T_all + bos) * V
103 else:
104 v = v + (i_n * H + i_h) * T * V
105 k = k + (i_n * H + i_h) * T * K
106 w = w + (i_n * H + i_h) * T * K
107 if USE_G:
108 g = g + (i_n * H + i_h) * T
109 h = h + (i_n * H + i_h) * NT * K * V
110 if SAVE_NEW_VALUE:
111 v_new_base = v_new + (i_n * H + i_h) * T * V
112
113 if USE_INITIAL_STATE:
114 h0_ptr = h0 + i_nh * K * V
115 if STORE_FINAL_STATE:
116 ht_ptr = ht + i_nh * K * V
117
118 # Load initial state
119 if USE_INITIAL_STATE:
120 p_h0_1 = tl.make_block_ptr(h0_ptr, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0))
121 b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32)
122 if K > 64:
123 p_h0_2 = tl.make_block_ptr(h0_ptr, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0))
124 b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32)
125 if K > 128:
126 p_h0_3 = tl.make_block_ptr(h0_ptr, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0))
127 b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32)
128 if K > 192:
129 p_h0_4 = tl.make_block_ptr(h0_ptr, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0))
130 b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32)
131
132 # Main recurrence over chunks
133 for i_t in range(NT):
134 # Store current hidden state h_t
135 p_h1 = tl.make_block_ptr(h + i_t * K * V, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0))
136 tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1))
137 if K > 64:
138 p_h2 = tl.make_block_ptr(h + i_t * K * V, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0))
139 tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1))
140 if K > 128:
141 p_h3 = tl.make_block_ptr(h + i_t * K * V, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0))
142 tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1))
143 if K > 192:
144 p_h4 = tl.make_block_ptr(h + i_t * K * V, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0))
145 tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1))
146
147 # Compute v_residual = v - w @ h
148 p_w = tl.make_block_ptr(w, (T, K), (K, 1), (i_t * BT, 0), (BT, 64), (1, 0))
149 b_w = tl.load(p_w, boundary_check=(0, 1))
150 b_v = tl.dot(b_w, b_h1.to(b_w.dtype))
151 if K > 64:
152 p_w = tl.make_block_ptr(w, (T, K), (K, 1), (i_t * BT, 64), (BT, 64), (1, 0))
153 b_w = tl.load(p_w, boundary_check=(0, 1))
154 b_v += tl.dot(b_w, b_h2.to(b_w.dtype))
155 if K > 128:
156 p_w = tl.make_block_ptr(w, (T, K), (K, 1), (i_t * BT, 128), (BT, 64), (1, 0))
157 b_w = tl.load(p_w, boundary_check=(0, 1))
158 b_v += tl.dot(b_w, b_h3.to(b_w.dtype))
159 if K > 192:
160 p_w = tl.make_block_ptr(w, (T, K), (K, 1), (i_t * BT, 192), (BT, 64), (1, 0))
161 b_w = tl.load(p_w, boundary_check=(0, 1))
162 b_v += tl.dot(b_w, b_h4.to(b_w.dtype))
163
164 p_v = tl.make_block_ptr(v, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
165 b_v = tl.load(p_v, boundary_check=(0, 1)) - b_v
166
167 if SAVE_NEW_VALUE:
168 p_v_new = tl.make_block_ptr(v_new_base, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
169 tl.store(p_v_new, b_v.to(p_v_new.dtype.element_ty), boundary_check=(0, 1))
170
171 last_idx = min((i_t + 1) * BT, T) - 1
172
173 # Apply output gate g
174 if USE_G:
175 m_t = (i_t * BT + tl.arange(0, BT)).to(tl.float32) < T
176 b_g_last = tl.load(g + last_idx)
177 p_g = tl.make_block_ptr(g, (T,), (1,), (i_t * BT,), (BT,), (0,))
178 b_g = tl.load(p_g, boundary_check=(0,))
179 b_v *= (m_t * tl.exp(b_g_last - b_g))[:, None]
180 b_g_last_exp = tl.exp(b_g_last)
181 b_h1 *= b_g_last_exp
182 if K > 64:
183 b_h2 *= b_g_last_exp
184 if K > 128:
185 b_h3 *= b_g_last_exp
186 if K > 192:
187 b_h4 *= b_g_last_exp
188
189 # Apply key gate gk
190 if USE_GK:
191 o_k1 = tl.arange(0, 64)
192 if IS_VARLEN:
193 gk_base_ptr = gk + (i_h * T_all + bos) * K
194 else:
195 gk_base_ptr = gk + (i_n * H + i_h) * T * K
196 b_gk_last1 = tl.load(gk_base_ptr + last_idx * K + o_k1, mask=(o_k1 < K), other=0.)
197 b_h1 *= tl.exp(b_gk_last1)[:, None]
198 if K > 64:
199 o_k2 = 64 + o_k1
200 b_gk_last2 = tl.load(gk_base_ptr + last_idx * K + o_k2, mask=(o_k2 < K), other=0.)
201 b_h2 *= tl.exp(b_gk_last2)[:, None]
202 if K > 128:
203 o_k3 = 128 + o_k1
204 b_gk_last3 = tl.load(gk_base_ptr + last_idx * K + o_k3, mask=(o_k3 < K), other=0.)
205 b_h3 *= tl.exp(b_gk_last3)[:, None]
206 if K > 192:
207 o_k4 = 192 + o_k1
208 b_gk_last4 = tl.load(gk_base_ptr + last_idx * K + o_k4, mask=(o_k4 < K), other=0.)
209 b_h4 *= tl.exp(b_gk_last4)[:, None]
210
211 b_v = b_v.to(k.dtype.element_ty)
212
213 # Update hidden state: h += k @ v
214 p_k = tl.make_block_ptr(k, (K, T), (1, K), (0, i_t * BT), (64, BT), (0, 1))
215 b_k = tl.load(p_k, boundary_check=(0, 1))
216 if USE_GK:
217 p_gk = tl.make_block_ptr(gk_base_ptr, (K, T), (1, K), (0, i_t * BT), (64, BT), (0, 1))
218 b_k = (b_k * tl.exp(b_gk_last1[:, None] - tl.load(p_gk, boundary_check=(0, 1)))).to(b_k.dtype)
219 b_h1 += tl.dot(b_k, b_v)
220
221 if K > 64:
222 p_k = tl.make_block_ptr(k, (K, T), (1, K), (64, i_t * BT), (64, BT), (0, 1))
223 b_k = tl.load(p_k, boundary_check=(0, 1))
224 if USE_GK:
225 p_gk = tl.make_block_ptr(gk_base_ptr, (K, T), (1, K), (64, i_t * BT), (64, BT), (0, 1))
226 b_k = (b_k * tl.exp(b_gk_last2[:, None] - tl.load(p_gk, boundary_check=(0, 1)))).to(b_k.dtype)
227 b_h2 += tl.dot(b_k, b_v)
228
229 if K > 128:
230 p_k = tl.make_block_ptr(k, (K, T), (1, K), (128, i_t * BT), (64, BT), (0, 1))
231 b_k = tl.load(p_k, boundary_check=(0, 1))
232 if USE_GK:
233 p_gk = tl.make_block_ptr(gk_base_ptr, (K, T), (1, K), (128, i_t * BT), (64, BT), (0, 1))
234 b_k = (b_k * tl.exp(b_gk_last3[:, None] - tl.load(p_gk, boundary_check=(0, 1)))).to(b_k.dtype)
235 b_h3 += tl.dot(b_k, b_v)
236
237 if K > 192:
238 p_k = tl.make_block_ptr(k, (K, T), (1, K), (192, i_t * BT), (64, BT), (0, 1))
239 b_k = tl.load(p_k, boundary_check=(0, 1))
240 if USE_GK:
241 p_gk = tl.make_block_ptr(gk_base_ptr, (K, T), (1, K), (192, i_t * BT), (64, BT), (0, 1))
242 b_k = (b_k * tl.exp(b_gk_last4[:, None] - tl.load(p_gk, boundary_check=(0, 1)))).to(b_k.dtype)
243 b_h4 += tl.dot(b_k, b_v)
244
245 # Store final state
246 if STORE_FINAL_STATE:
247 p_ht = tl.make_block_ptr(ht_ptr, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0))
248 tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
249 if K > 64:
250 p_ht = tl.make_block_ptr(ht_ptr, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0))
251 tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
252 if K > 128:
253 p_ht = tl.make_block_ptr(ht_ptr, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0))
254 tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
255 if K > 192:
256 p_ht = tl.make_block_ptr(ht_ptr, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0))
257 tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
258
259
260def chunk_gated_delta_rule_fwd_h(
261 k: torch.Tensor,
262 w: torch.Tensor,
263 u: torch.Tensor,
264 g: Optional[torch.Tensor] = None,
265 gk: Optional[torch.Tensor] = None,
266 initial_state: Optional[torch.Tensor] = None,
267 output_final_state: bool = False,
268 chunk_size: int = 64, # default:64
269 save_new_value: bool = True,
270 cu_seqlens: Optional[torch.LongTensor] = None,
271) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
272 B, T, H, K, V = *k.shape, u.shape[-1]
273 BT = chunk_size
274
275 chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) if cu_seqlens is not None else None
276 # N: the actual number of sequences in the batch with either equal or variable lengths
277 if cu_seqlens is None:
278 N, NT, chunk_offsets = B, triton.cdiv(T, BT), None
279 else:
280 N, NT, chunk_offsets = len(cu_seqlens) - 1, len(chunk_indices), prepare_chunk_offsets(cu_seqlens, BT)
281 if K > 256:
282 raise ValueError("current kernel does not support head dimension larger than 256.")
283
284 h = k.new_empty(B, NT, H, K, V).permute(0, 2, 1, 3, 4).contiguous()
285 final_state = k.new_empty(N, H, K, V, dtype=torch.float32) if output_final_state else None
286
287 BV = 128
288
289 v_new = torch.empty_like(u).permute(0, 2, 1, 3).contiguous() if save_new_value else None
290 k = k.permute(0, 2, 1, 3).contiguous()
291 w = w.permute(0, 2, 1, 3).contiguous()
292 u = u.permute(0, 2, 1, 3).contiguous()
293 g = g.permute(0, 2, 1).contiguous() if g is not None else None
294 # Match the head-major layout used by the forward kernel's token strides.
295 gk = gk.permute(0, 2, 1, 3).contiguous() if gk is not None else None
296 chunk_gated_delta_rule_fwd_kernel_h_blockdim64[(triton.cdiv(V, BV), N * H)](
297 k=k,
298 v=u,
299 w=w,
300 v_new=v_new,
301 g=g,
302 gk=gk,
303 h=h,
304 h0=initial_state,
305 ht=final_state,
306 cu_seqlens=cu_seqlens,
307 chunk_offsets=chunk_offsets,
308 T=T,
309 H=H,
310 K=K,
311 V=V,
312 BT=BT,
313 BV=BV,
314 NT=NT,
315 )
316 h = h.permute(0, 2, 1, 3, 4).contiguous()
317 v_new = v_new.permute(0, 2, 1, 3).contiguous() if v_new is not None else None
318 return h, v_new, final_state
319
320
321@triton.heuristics({
322 'USE_G': lambda args: args['g'] is not None,
323 'USE_GK': lambda args: args['gk'] is not None,
324 'USE_INITIAL_STATE': lambda args: args['dh0'] is not None,
325 'USE_FINAL_STATE_GRADIENT': lambda args: args['dht'] is not None,
326 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None,
327})
328# Preserve all cache keys without interpreting them as automatic tiling axes.
329@triton.runtime.autotuner.autotune(
330 configs=get_autotune_config(multibuffer_list=(True, False)),
331 key=['H', 'K', 'V', 'BT', 'BV', 'USE_G', 'IS_VARLEN'],
332)
333@triton.jit(do_not_specialize=['T'])
334def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64(
335 q,
336 k,
337 w,
338 g,
339 gk,
340 dht,
341 dh0,
342 do,
343 dh,
344 dv,
345 dv2,
346 cu_seqlens,
347 chunk_offsets,
348 scale,
349 T,
350 H: tl.constexpr,
351 K: tl.constexpr,
352 V: tl.constexpr,
353 BT: tl.constexpr,
354 BV: tl.constexpr,
355 USE_G: tl.constexpr,
356 USE_GK: tl.constexpr,
357 USE_INITIAL_STATE: tl.constexpr,
358 USE_FINAL_STATE_GRADIENT: tl.constexpr,
359 IS_VARLEN: tl.constexpr,
360):
361 T_all = T
362 i_v, i_nh = tl.program_id(0), tl.program_id(1)
363 i_n, i_h = i_nh // H, i_nh % H
364 if IS_VARLEN:
365 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32)
366 T = eos - bos
367 NT = tl.cdiv(T, BT)
368 boh = tl.load(chunk_offsets + i_n).to(tl.int32)
369 else:
370 bos, eos = i_n * T, i_n * T + T
371 NT = tl.cdiv(T, BT)
372 boh = i_n * NT
373
374 b_dh1 = tl.zeros([64, BV], dtype=tl.float32)
375 if K > 64:
376 b_dh2 = tl.zeros([64, BV], dtype=tl.float32)
377 if K > 128:
378 b_dh3 = tl.zeros([64, BV], dtype=tl.float32)
379 if K > 192:
380 b_dh4 = tl.zeros([64, BV], dtype=tl.float32)
381
382 q += (bos * H + i_h) * K
383 k += (bos * H + i_h) * K
384 w += (bos * H + i_h) * K
385 do += (bos * H + i_h) * V
386 dv += (bos * H + i_h) * V
387 dv2 += (bos * H + i_h) * V
388 dh += (boh * H + i_h) * K * V
389 if USE_GK:
390 gk += (bos * H + i_h) * K
391
392 if USE_INITIAL_STATE:
393 dh0 += i_nh * K * V
394 if USE_FINAL_STATE_GRADIENT:
395 dht += i_nh * K * V
396
397 stride_v = H * V
398 stride_h = H * K * V
399 stride_k = H * K
400
401 if USE_FINAL_STATE_GRADIENT:
402 p_dht1 = tl.make_block_ptr(dht, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0))
403 b_dh1 += tl.load(p_dht1, boundary_check=(0, 1))
404 if K > 64:
405 p_dht2 = tl.make_block_ptr(dht, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0))
406 b_dh2 += tl.load(p_dht2, boundary_check=(0, 1))
407 if K > 128:
408 p_dht3 = tl.make_block_ptr(dht, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0))
409 b_dh3 += tl.load(p_dht3, boundary_check=(0, 1))
410 if K > 192:
411 p_dht4 = tl.make_block_ptr(dht, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0))
412 b_dh4 += tl.load(p_dht4, boundary_check=(0, 1))
413
414 for i_t in range(NT - 1, -1, -1):
415 p_dh1 = tl.make_block_ptr(dh + i_t * stride_h, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0))
416 tl.store(p_dh1, b_dh1.to(p_dh1.dtype.element_ty), boundary_check=(0, 1))
417 if K > 64:
418 p_dh2 = tl.make_block_ptr(dh + i_t * stride_h, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0))
419 tl.store(p_dh2, b_dh2.to(p_dh2.dtype.element_ty), boundary_check=(0, 1))
420 if K > 128:
421 p_dh3 = tl.make_block_ptr(dh + i_t * stride_h, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0))
422 tl.store(p_dh3, b_dh3.to(p_dh3.dtype.element_ty), boundary_check=(0, 1))
423 if K > 192:
424 p_dh4 = tl.make_block_ptr(dh + i_t * stride_h, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0))
425 tl.store(p_dh4, b_dh4.to(p_dh4.dtype.element_ty), boundary_check=(0, 1))
426
427 last_idx = min((i_t + 1) * BT, T) - 1
428 if USE_G:
429 if IS_VARLEN:
430 bos_g = i_h * T_all + bos
431 else:
432 bos_g = (i_n * H + i_h) * T_all
433 bg_last = tl.load(g + bos_g + last_idx)
434 bg_last_exp = tl.exp(bg_last)
435 p_g = tl.make_block_ptr(base=g + bos_g, shape=(T,), strides=(1,), offsets=(i_t * BT,), block_shape=(BT,), order=(0,))
436 b_g = tl.load(p_g, boundary_check=(0,))
437 b_g_exp = tl.exp(b_g)
438
439 p_dv = tl.make_block_ptr(dv, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
440 p_dv2 = tl.make_block_ptr(dv2, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
441 p_do = tl.make_block_ptr(do, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0))
442
443 b_do = tl.load(p_do, boundary_check=(0, 1))
444
445 # Update dv
446 p_k = tl.make_block_ptr(k, (T, K), (stride_k, 1), (i_t * BT, 0), (BT, 64), (1, 0))
447 b_k = tl.load(p_k, boundary_check=(0, 1))
448 if USE_GK:
449 o_k1 = tl.arange(0, 64)
450 b_gk_last1 = tl.load(gk + last_idx * H * K + o_k1, mask=(o_k1 < K), other=0.)
451 b_dv = tl.dot(b_k, b_dh1.to(b_k.dtype))
452
453 if K > 64:
454 p_k = tl.make_block_ptr(k, (T, K), (stride_k, 1), (i_t * BT, 64), (BT, 64), (1, 0))
455 b_k = tl.load(p_k, boundary_check=(0, 1))
456 if USE_GK:
457 o_k2 = 64 + o_k1
458 b_gk_last2 = tl.load(gk + last_idx * H * K + o_k2, mask=(o_k2 < K), other=0.)
459 b_dv += tl.dot(b_k, b_dh2.to(b_k.dtype))
460
461 if K > 128:
462 p_k = tl.make_block_ptr(k, (T, K), (stride_k, 1), (i_t * BT, 128), (BT, 64), (1, 0))
463 b_k = tl.load(p_k, boundary_check=(0, 1))
464 if USE_GK:
465 o_k3 = 128 + o_k1
466 b_gk_last3 = tl.load(gk + last_idx * H * K + o_k3, mask=(o_k3 < K), other=0.)
467 b_dv += tl.dot(b_k, b_dh3.to(b_k.dtype))
468
469 if K > 192:
470 p_k = tl.make_block_ptr(k, (T, K), (stride_k, 1), (i_t * BT, 192), (BT, 64), (1, 0))
471 b_k = tl.load(p_k, boundary_check=(0, 1))
472 if USE_GK:
473 o_k4 = 192 + o_k1
474 b_gk_last4 = tl.load(gk + last_idx * H * K + o_k4, mask=(o_k4 < K), other=0.)
475 b_dv += tl.dot(b_k, b_dh4.to(b_k.dtype))
476
477 if USE_G:
478 m_t = (i_t * BT + tl.arange(0, BT)).to(tl.float32) < T
479 b_dv *= (m_t * tl.exp(bg_last - b_g))[:, None]
480 b_dv += tl.load(p_dv, boundary_check=(0, 1))
481
482 tl.store(p_dv2, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1))
483 # Update dh
484 p_w = tl.make_block_ptr(w, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1))
485 p_q = tl.make_block_ptr(q, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1))
486 b_w = tl.load(p_w, boundary_check=(0, 1))
487 b_q = tl.load(p_q, boundary_check=(0, 1))
488 if USE_G:
489 b_dh1 *= bg_last_exp
490 b_q = b_q * b_g_exp[None, :]
491 if USE_GK:
492 b_dh1 *= tl.exp(b_gk_last1[:, None])
493 b_dh1 += tl.dot(b_q.to(b_q.dtype), b_do.to(b_q.dtype)) * scale - tl.dot(b_w, b_dv.to(b_w.dtype))
494 if K > 64:
495 p_q = tl.make_block_ptr(q, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1))
496 p_w = tl.make_block_ptr(w, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1))
497 b_q = tl.load(p_q, boundary_check=(0, 1))
498 b_w = tl.load(p_w, boundary_check=(0, 1))
499 if USE_G:
500 b_dh2 *= bg_last_exp
501 b_q = b_q * b_g_exp[None, :]
502 if USE_GK:
503 b_dh2 *= tl.exp(b_gk_last2[:, None])
504 b_dh2 += tl.dot(b_q.to(b_q.dtype), b_do.to(b_q.dtype)) * scale - tl.dot(b_w, b_dv.to(b_w.dtype))
505 if K > 128:
506 p_q = tl.make_block_ptr(q, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1))
507 p_w = tl.make_block_ptr(w, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1))
508 b_q = tl.load(p_q, boundary_check=(0, 1))
509 b_w = tl.load(p_w, boundary_check=(0, 1))
510 if USE_G:
511 b_dh3 *= bg_last_exp
512 b_q = b_q * b_g_exp[None, :]
513 if USE_GK:
514 b_dh3 *= tl.exp(b_gk_last3[:, None])
515 b_dh3 += tl.dot(b_q.to(b_q.dtype), b_do.to(b_q.dtype)) * scale - tl.dot(b_w, b_dv.to(b_w.dtype))
516 if K > 192:
517 p_q = tl.make_block_ptr(q, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1))
518 p_w = tl.make_block_ptr(w, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1))
519 b_q = tl.load(p_q, boundary_check=(0, 1))
520 b_w = tl.load(p_w, boundary_check=(0, 1))
521 if USE_G:
522 b_dh4 *= bg_last_exp
523 b_q = b_q * b_g_exp[None, :]
524 if USE_GK:
525 b_dh4 *= tl.exp(b_gk_last4[:, None])
526 b_dh4 += tl.dot(b_q.to(b_q.dtype), b_do.to(b_q.dtype)) * scale - tl.dot(b_w, b_dv.to(b_w.dtype))
527
528 if USE_INITIAL_STATE:
529 p_dh0 = tl.make_block_ptr(dh0, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0))
530 tl.store(p_dh0, b_dh1.to(p_dh0.dtype.element_ty), boundary_check=(0, 1))
531 if K > 64:
532 p_dh1 = tl.make_block_ptr(dh0, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0))
533 tl.store(p_dh1, b_dh2.to(p_dh1.dtype.element_ty), boundary_check=(0, 1))
534 if K > 128:
535 p_dh2 = tl.make_block_ptr(dh0, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0))
536 tl.store(p_dh2, b_dh3.to(p_dh2.dtype.element_ty), boundary_check=(0, 1))
537 if K > 192:
538 p_dh3 = tl.make_block_ptr(dh0, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0))
539 tl.store(p_dh3, b_dh4.to(p_dh3.dtype.element_ty), boundary_check=(0, 1))
540
541
542def chunk_gated_delta_rule_bwd_dhu(
543 q: torch.Tensor,
544 k: torch.Tensor,
545 w: torch.Tensor,
546 do: torch.Tensor,
547 dv: torch.Tensor,
548 g: torch.Tensor | None = None,
549 gk: torch.Tensor | None = None,
550 h0: torch.Tensor | None = None,
551 dht: torch.Tensor | None = None,
552 scale: float | None = None,
553 cu_seqlens: torch.LongTensor | None = None,
554 chunk_size: int = 64, # SY: remove this argument and force chunk size 64?
555 chunk_indices: torch.LongTensor | None = None,
556 use_exp2: bool = False,
557) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
558 B, T, H, K, V = *q.shape, do.shape[-1]
559 # N: the actual number of sequences in the batch with either equal or variable lengths
560 BT = 64
561 if K > 256:
562 raise ValueError("current kernel does not support head dimension being larger than 256.")
563
564 if chunk_indices is None and cu_seqlens is not None:
565 chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size)
566 if cu_seqlens is None:
567 N, NT, chunk_offsets = B, triton.cdiv(T, BT), None
568 else:
569 N, NT, chunk_offsets = len(cu_seqlens) - 1, len(chunk_indices), prepare_chunk_offsets(cu_seqlens, BT)
570
571 dh = q.new_empty(B, NT, H, K, V)
572 dh0 = torch.empty_like(h0, dtype=torch.float32) if h0 is not None else None
573 dv2 = torch.empty_like(dv)
574
575 BV = 128
576
577 g = g.permute(0, 2, 1).contiguous()
578
579 chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64[(triton.cdiv(V, BV), N * H)](
580 q=q,
581 k=k,
582 w=w,
583 g=g,
584 gk=gk,
585 dht=dht,
586 dh0=dh0,
587 do=do,
588 dh=dh,
589 dv=dv,
590 dv2=dv2,
591 cu_seqlens=cu_seqlens,
592 chunk_offsets=chunk_offsets,
593 scale=scale,
594 T=T,
595 H=H,
596 K=K,
597 V=V,
598 BT=BT,
599 BV=BV,
600 )
601 return dh, dh0, dv2