Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / _triton / gated_delta_net / chunk_delta_h.py: 0%

370 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang 

16 

17# pylint: disable=line-too-long,missing-public-type-hints,missing-public-docstring 

18# pylint: disable=used-before-assignment,unsupported-binary-operation,unused-argument 

19# pylint: disable=invalid-name,missing-module-docstring,missing-function-docstring 

20 

21from typing import Optional, Tuple 

22 

23import torch 

24import triton 

25import triton.language as tl 

26 

27from .utils import prepare_chunk_indices, prepare_chunk_offsets, get_autotune_config, get_npu_properties 

28 

29CUBE_CORE_NUM = get_npu_properties()['num_aicore'] 

30 

31 

32@triton.heuristics({ 

33 'USE_G': lambda args: args['g'] is not None, 

34 'USE_GK': lambda args: args['gk'] is not None, 

35 'USE_INITIAL_STATE': lambda args: args['h0'] is not None, 

36 'STORE_FINAL_STATE': lambda args: args['ht'] is not None, 

37 'SAVE_NEW_VALUE': lambda args: args['v_new'] is not None, 

38 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None, 

39}) 

40@triton.autotune( 

41 configs=get_autotune_config(multibuffer_list=(False,)), 

42 key=['H', 'K', 'V', 'BT'], 

43) 

44@triton.jit(do_not_specialize=['T']) 

45def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( 

46 k, 

47 v, 

48 w, 

49 v_new, 

50 g, 

51 gk, 

52 h, 

53 h0, 

54 ht, 

55 cu_seqlens, 

56 chunk_offsets, 

57 T, 

58 H: tl.constexpr, 

59 K: tl.constexpr, 

60 V: tl.constexpr, 

61 BT: tl.constexpr, 

62 BV: tl.constexpr, 

63 NT: tl.constexpr, 

64 USE_G: tl.constexpr, 

65 USE_GK: tl.constexpr, 

66 USE_INITIAL_STATE: tl.constexpr, 

67 STORE_FINAL_STATE: tl.constexpr, 

68 SAVE_NEW_VALUE: tl.constexpr, 

69 IS_VARLEN: tl.constexpr, 

70): 

71 T_all = T 

72 NT_all = NT 

73 i_v, i_nh = tl.program_id(0), tl.program_id(1) 

74 i_n, i_h = i_nh // H, i_nh % H 

75 if IS_VARLEN: 

76 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

77 T = eos - bos 

78 NT = tl.cdiv(T, BT) 

79 boh = tl.load(chunk_offsets + i_n).to(tl.int32) 

80 else: 

81 bos, eos = i_n * T, i_n * T + T 

82 NT = tl.cdiv(T, BT) 

83 boh = i_n * NT 

84 

85 # Initialize hidden states 

86 b_h1 = tl.zeros([64, BV], dtype=tl.float32) 

87 if K > 64: 

88 b_h2 = tl.zeros([64, BV], dtype=tl.float32) 

89 if K > 128: 

90 b_h3 = tl.zeros([64, BV], dtype=tl.float32) 

91 if K > 192: 

92 b_h4 = tl.zeros([64, BV], dtype=tl.float32) 

93 

94 if IS_VARLEN: 

95 v = v + (i_h * T_all + bos) * V 

96 k = k + (i_h * T_all + bos) * K 

97 w = w + (i_h * T_all + bos) * K 

98 if USE_G: 

99 g = g + i_h * T_all + bos 

100 h = h + (i_h * NT_all + boh) * K * V 

101 if SAVE_NEW_VALUE: 

102 v_new_base = v_new + (i_h * T_all + bos) * V 

103 else: 

104 v = v + (i_n * H + i_h) * T * V 

105 k = k + (i_n * H + i_h) * T * K 

106 w = w + (i_n * H + i_h) * T * K 

107 if USE_G: 

108 g = g + (i_n * H + i_h) * T 

109 h = h + (i_n * H + i_h) * NT * K * V 

110 if SAVE_NEW_VALUE: 

111 v_new_base = v_new + (i_n * H + i_h) * T * V 

112 

113 if USE_INITIAL_STATE: 

114 h0_ptr = h0 + i_nh * K * V 

115 if STORE_FINAL_STATE: 

116 ht_ptr = ht + i_nh * K * V 

117 

118 # Load initial state 

119 if USE_INITIAL_STATE: 

120 p_h0_1 = tl.make_block_ptr(h0_ptr, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) 

121 b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32) 

122 if K > 64: 

123 p_h0_2 = tl.make_block_ptr(h0_ptr, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) 

124 b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32) 

125 if K > 128: 

126 p_h0_3 = tl.make_block_ptr(h0_ptr, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) 

127 b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32) 

128 if K > 192: 

129 p_h0_4 = tl.make_block_ptr(h0_ptr, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) 

130 b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32) 

131 

132 # Main recurrence over chunks 

133 for i_t in range(NT): 

134 # Store current hidden state h_t 

135 p_h1 = tl.make_block_ptr(h + i_t * K * V, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) 

136 tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1)) 

137 if K > 64: 

138 p_h2 = tl.make_block_ptr(h + i_t * K * V, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) 

139 tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1)) 

140 if K > 128: 

141 p_h3 = tl.make_block_ptr(h + i_t * K * V, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) 

142 tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1)) 

143 if K > 192: 

144 p_h4 = tl.make_block_ptr(h + i_t * K * V, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) 

145 tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1)) 

146 

147 # Compute v_residual = v - w @ h 

148 p_w = tl.make_block_ptr(w, (T, K), (K, 1), (i_t * BT, 0), (BT, 64), (1, 0)) 

149 b_w = tl.load(p_w, boundary_check=(0, 1)) 

150 b_v = tl.dot(b_w, b_h1.to(b_w.dtype)) 

151 if K > 64: 

152 p_w = tl.make_block_ptr(w, (T, K), (K, 1), (i_t * BT, 64), (BT, 64), (1, 0)) 

153 b_w = tl.load(p_w, boundary_check=(0, 1)) 

154 b_v += tl.dot(b_w, b_h2.to(b_w.dtype)) 

155 if K > 128: 

156 p_w = tl.make_block_ptr(w, (T, K), (K, 1), (i_t * BT, 128), (BT, 64), (1, 0)) 

157 b_w = tl.load(p_w, boundary_check=(0, 1)) 

158 b_v += tl.dot(b_w, b_h3.to(b_w.dtype)) 

159 if K > 192: 

160 p_w = tl.make_block_ptr(w, (T, K), (K, 1), (i_t * BT, 192), (BT, 64), (1, 0)) 

161 b_w = tl.load(p_w, boundary_check=(0, 1)) 

162 b_v += tl.dot(b_w, b_h4.to(b_w.dtype)) 

163 

164 p_v = tl.make_block_ptr(v, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

165 b_v = tl.load(p_v, boundary_check=(0, 1)) - b_v 

166 

167 if SAVE_NEW_VALUE: 

168 p_v_new = tl.make_block_ptr(v_new_base, (T, V), (V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

169 tl.store(p_v_new, b_v.to(p_v_new.dtype.element_ty), boundary_check=(0, 1)) 

170 

171 last_idx = min((i_t + 1) * BT, T) - 1 

172 

173 # Apply output gate g 

174 if USE_G: 

175 m_t = (i_t * BT + tl.arange(0, BT)).to(tl.float32) < T 

176 b_g_last = tl.load(g + last_idx) 

177 p_g = tl.make_block_ptr(g, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

178 b_g = tl.load(p_g, boundary_check=(0,)) 

179 b_v *= (m_t * tl.exp(b_g_last - b_g))[:, None] 

180 b_g_last_exp = tl.exp(b_g_last) 

181 b_h1 *= b_g_last_exp 

182 if K > 64: 

183 b_h2 *= b_g_last_exp 

184 if K > 128: 

185 b_h3 *= b_g_last_exp 

186 if K > 192: 

187 b_h4 *= b_g_last_exp 

188 

189 # Apply key gate gk 

190 if USE_GK: 

191 o_k1 = tl.arange(0, 64) 

192 if IS_VARLEN: 

193 gk_base_ptr = gk + (i_h * T_all + bos) * K 

194 else: 

195 gk_base_ptr = gk + (i_n * H + i_h) * T * K 

196 b_gk_last1 = tl.load(gk_base_ptr + last_idx * K + o_k1, mask=(o_k1 < K), other=0.) 

197 b_h1 *= tl.exp(b_gk_last1)[:, None] 

198 if K > 64: 

199 o_k2 = 64 + o_k1 

200 b_gk_last2 = tl.load(gk_base_ptr + last_idx * K + o_k2, mask=(o_k2 < K), other=0.) 

201 b_h2 *= tl.exp(b_gk_last2)[:, None] 

202 if K > 128: 

203 o_k3 = 128 + o_k1 

204 b_gk_last3 = tl.load(gk_base_ptr + last_idx * K + o_k3, mask=(o_k3 < K), other=0.) 

205 b_h3 *= tl.exp(b_gk_last3)[:, None] 

206 if K > 192: 

207 o_k4 = 192 + o_k1 

208 b_gk_last4 = tl.load(gk_base_ptr + last_idx * K + o_k4, mask=(o_k4 < K), other=0.) 

209 b_h4 *= tl.exp(b_gk_last4)[:, None] 

210 

211 b_v = b_v.to(k.dtype.element_ty) 

212 

213 # Update hidden state: h += k @ v 

214 p_k = tl.make_block_ptr(k, (K, T), (1, K), (0, i_t * BT), (64, BT), (0, 1)) 

215 b_k = tl.load(p_k, boundary_check=(0, 1)) 

216 if USE_GK: 

217 p_gk = tl.make_block_ptr(gk_base_ptr, (K, T), (1, K), (0, i_t * BT), (64, BT), (0, 1)) 

218 b_k = (b_k * tl.exp(b_gk_last1[:, None] - tl.load(p_gk, boundary_check=(0, 1)))).to(b_k.dtype) 

219 b_h1 += tl.dot(b_k, b_v) 

220 

221 if K > 64: 

222 p_k = tl.make_block_ptr(k, (K, T), (1, K), (64, i_t * BT), (64, BT), (0, 1)) 

223 b_k = tl.load(p_k, boundary_check=(0, 1)) 

224 if USE_GK: 

225 p_gk = tl.make_block_ptr(gk_base_ptr, (K, T), (1, K), (64, i_t * BT), (64, BT), (0, 1)) 

226 b_k = (b_k * tl.exp(b_gk_last2[:, None] - tl.load(p_gk, boundary_check=(0, 1)))).to(b_k.dtype) 

227 b_h2 += tl.dot(b_k, b_v) 

228 

229 if K > 128: 

230 p_k = tl.make_block_ptr(k, (K, T), (1, K), (128, i_t * BT), (64, BT), (0, 1)) 

231 b_k = tl.load(p_k, boundary_check=(0, 1)) 

232 if USE_GK: 

233 p_gk = tl.make_block_ptr(gk_base_ptr, (K, T), (1, K), (128, i_t * BT), (64, BT), (0, 1)) 

234 b_k = (b_k * tl.exp(b_gk_last3[:, None] - tl.load(p_gk, boundary_check=(0, 1)))).to(b_k.dtype) 

235 b_h3 += tl.dot(b_k, b_v) 

236 

237 if K > 192: 

238 p_k = tl.make_block_ptr(k, (K, T), (1, K), (192, i_t * BT), (64, BT), (0, 1)) 

239 b_k = tl.load(p_k, boundary_check=(0, 1)) 

240 if USE_GK: 

241 p_gk = tl.make_block_ptr(gk_base_ptr, (K, T), (1, K), (192, i_t * BT), (64, BT), (0, 1)) 

242 b_k = (b_k * tl.exp(b_gk_last4[:, None] - tl.load(p_gk, boundary_check=(0, 1)))).to(b_k.dtype) 

243 b_h4 += tl.dot(b_k, b_v) 

244 

245 # Store final state 

246 if STORE_FINAL_STATE: 

247 p_ht = tl.make_block_ptr(ht_ptr, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) 

248 tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) 

249 if K > 64: 

250 p_ht = tl.make_block_ptr(ht_ptr, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) 

251 tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) 

252 if K > 128: 

253 p_ht = tl.make_block_ptr(ht_ptr, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) 

254 tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) 

255 if K > 192: 

256 p_ht = tl.make_block_ptr(ht_ptr, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) 

257 tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) 

258 

259 

260def chunk_gated_delta_rule_fwd_h( 

261 k: torch.Tensor, 

262 w: torch.Tensor, 

263 u: torch.Tensor, 

264 g: Optional[torch.Tensor] = None, 

265 gk: Optional[torch.Tensor] = None, 

266 initial_state: Optional[torch.Tensor] = None, 

267 output_final_state: bool = False, 

268 chunk_size: int = 64, # default:64 

269 save_new_value: bool = True, 

270 cu_seqlens: Optional[torch.LongTensor] = None, 

271) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: 

272 B, T, H, K, V = *k.shape, u.shape[-1] 

273 BT = chunk_size 

274 

275 chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) if cu_seqlens is not None else None 

276 # N: the actual number of sequences in the batch with either equal or variable lengths 

277 if cu_seqlens is None: 

278 N, NT, chunk_offsets = B, triton.cdiv(T, BT), None 

279 else: 

280 N, NT, chunk_offsets = len(cu_seqlens) - 1, len(chunk_indices), prepare_chunk_offsets(cu_seqlens, BT) 

281 if K > 256: 

282 raise ValueError("current kernel does not support head dimension larger than 256.") 

283 

284 h = k.new_empty(B, NT, H, K, V).permute(0, 2, 1, 3, 4).contiguous() 

285 final_state = k.new_empty(N, H, K, V, dtype=torch.float32) if output_final_state else None 

286 

287 BV = 128 

288 

289 v_new = torch.empty_like(u).permute(0, 2, 1, 3).contiguous() if save_new_value else None 

290 k = k.permute(0, 2, 1, 3).contiguous() 

291 w = w.permute(0, 2, 1, 3).contiguous() 

292 u = u.permute(0, 2, 1, 3).contiguous() 

293 g = g.permute(0, 2, 1).contiguous() if g is not None else None 

294 # Match the head-major layout used by the forward kernel's token strides. 

295 gk = gk.permute(0, 2, 1, 3).contiguous() if gk is not None else None 

296 chunk_gated_delta_rule_fwd_kernel_h_blockdim64[(triton.cdiv(V, BV), N * H)]( 

297 k=k, 

298 v=u, 

299 w=w, 

300 v_new=v_new, 

301 g=g, 

302 gk=gk, 

303 h=h, 

304 h0=initial_state, 

305 ht=final_state, 

306 cu_seqlens=cu_seqlens, 

307 chunk_offsets=chunk_offsets, 

308 T=T, 

309 H=H, 

310 K=K, 

311 V=V, 

312 BT=BT, 

313 BV=BV, 

314 NT=NT, 

315 ) 

316 h = h.permute(0, 2, 1, 3, 4).contiguous() 

317 v_new = v_new.permute(0, 2, 1, 3).contiguous() if v_new is not None else None 

318 return h, v_new, final_state 

319 

320 

321@triton.heuristics({ 

322 'USE_G': lambda args: args['g'] is not None, 

323 'USE_GK': lambda args: args['gk'] is not None, 

324 'USE_INITIAL_STATE': lambda args: args['dh0'] is not None, 

325 'USE_FINAL_STATE_GRADIENT': lambda args: args['dht'] is not None, 

326 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None, 

327}) 

328# Preserve all cache keys without interpreting them as automatic tiling axes. 

329@triton.runtime.autotuner.autotune( 

330 configs=get_autotune_config(multibuffer_list=(True, False)), 

331 key=['H', 'K', 'V', 'BT', 'BV', 'USE_G', 'IS_VARLEN'], 

332) 

333@triton.jit(do_not_specialize=['T']) 

334def chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64( 

335 q, 

336 k, 

337 w, 

338 g, 

339 gk, 

340 dht, 

341 dh0, 

342 do, 

343 dh, 

344 dv, 

345 dv2, 

346 cu_seqlens, 

347 chunk_offsets, 

348 scale, 

349 T, 

350 H: tl.constexpr, 

351 K: tl.constexpr, 

352 V: tl.constexpr, 

353 BT: tl.constexpr, 

354 BV: tl.constexpr, 

355 USE_G: tl.constexpr, 

356 USE_GK: tl.constexpr, 

357 USE_INITIAL_STATE: tl.constexpr, 

358 USE_FINAL_STATE_GRADIENT: tl.constexpr, 

359 IS_VARLEN: tl.constexpr, 

360): 

361 T_all = T 

362 i_v, i_nh = tl.program_id(0), tl.program_id(1) 

363 i_n, i_h = i_nh // H, i_nh % H 

364 if IS_VARLEN: 

365 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

366 T = eos - bos 

367 NT = tl.cdiv(T, BT) 

368 boh = tl.load(chunk_offsets + i_n).to(tl.int32) 

369 else: 

370 bos, eos = i_n * T, i_n * T + T 

371 NT = tl.cdiv(T, BT) 

372 boh = i_n * NT 

373 

374 b_dh1 = tl.zeros([64, BV], dtype=tl.float32) 

375 if K > 64: 

376 b_dh2 = tl.zeros([64, BV], dtype=tl.float32) 

377 if K > 128: 

378 b_dh3 = tl.zeros([64, BV], dtype=tl.float32) 

379 if K > 192: 

380 b_dh4 = tl.zeros([64, BV], dtype=tl.float32) 

381 

382 q += (bos * H + i_h) * K 

383 k += (bos * H + i_h) * K 

384 w += (bos * H + i_h) * K 

385 do += (bos * H + i_h) * V 

386 dv += (bos * H + i_h) * V 

387 dv2 += (bos * H + i_h) * V 

388 dh += (boh * H + i_h) * K * V 

389 if USE_GK: 

390 gk += (bos * H + i_h) * K 

391 

392 if USE_INITIAL_STATE: 

393 dh0 += i_nh * K * V 

394 if USE_FINAL_STATE_GRADIENT: 

395 dht += i_nh * K * V 

396 

397 stride_v = H * V 

398 stride_h = H * K * V 

399 stride_k = H * K 

400 

401 if USE_FINAL_STATE_GRADIENT: 

402 p_dht1 = tl.make_block_ptr(dht, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) 

403 b_dh1 += tl.load(p_dht1, boundary_check=(0, 1)) 

404 if K > 64: 

405 p_dht2 = tl.make_block_ptr(dht, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) 

406 b_dh2 += tl.load(p_dht2, boundary_check=(0, 1)) 

407 if K > 128: 

408 p_dht3 = tl.make_block_ptr(dht, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) 

409 b_dh3 += tl.load(p_dht3, boundary_check=(0, 1)) 

410 if K > 192: 

411 p_dht4 = tl.make_block_ptr(dht, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) 

412 b_dh4 += tl.load(p_dht4, boundary_check=(0, 1)) 

413 

414 for i_t in range(NT - 1, -1, -1): 

415 p_dh1 = tl.make_block_ptr(dh + i_t * stride_h, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) 

416 tl.store(p_dh1, b_dh1.to(p_dh1.dtype.element_ty), boundary_check=(0, 1)) 

417 if K > 64: 

418 p_dh2 = tl.make_block_ptr(dh + i_t * stride_h, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) 

419 tl.store(p_dh2, b_dh2.to(p_dh2.dtype.element_ty), boundary_check=(0, 1)) 

420 if K > 128: 

421 p_dh3 = tl.make_block_ptr(dh + i_t * stride_h, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) 

422 tl.store(p_dh3, b_dh3.to(p_dh3.dtype.element_ty), boundary_check=(0, 1)) 

423 if K > 192: 

424 p_dh4 = tl.make_block_ptr(dh + i_t * stride_h, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) 

425 tl.store(p_dh4, b_dh4.to(p_dh4.dtype.element_ty), boundary_check=(0, 1)) 

426 

427 last_idx = min((i_t + 1) * BT, T) - 1 

428 if USE_G: 

429 if IS_VARLEN: 

430 bos_g = i_h * T_all + bos 

431 else: 

432 bos_g = (i_n * H + i_h) * T_all 

433 bg_last = tl.load(g + bos_g + last_idx) 

434 bg_last_exp = tl.exp(bg_last) 

435 p_g = tl.make_block_ptr(base=g + bos_g, shape=(T,), strides=(1,), offsets=(i_t * BT,), block_shape=(BT,), order=(0,)) 

436 b_g = tl.load(p_g, boundary_check=(0,)) 

437 b_g_exp = tl.exp(b_g) 

438 

439 p_dv = tl.make_block_ptr(dv, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

440 p_dv2 = tl.make_block_ptr(dv2, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

441 p_do = tl.make_block_ptr(do, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

442 

443 b_do = tl.load(p_do, boundary_check=(0, 1)) 

444 

445 # Update dv 

446 p_k = tl.make_block_ptr(k, (T, K), (stride_k, 1), (i_t * BT, 0), (BT, 64), (1, 0)) 

447 b_k = tl.load(p_k, boundary_check=(0, 1)) 

448 if USE_GK: 

449 o_k1 = tl.arange(0, 64) 

450 b_gk_last1 = tl.load(gk + last_idx * H * K + o_k1, mask=(o_k1 < K), other=0.) 

451 b_dv = tl.dot(b_k, b_dh1.to(b_k.dtype)) 

452 

453 if K > 64: 

454 p_k = tl.make_block_ptr(k, (T, K), (stride_k, 1), (i_t * BT, 64), (BT, 64), (1, 0)) 

455 b_k = tl.load(p_k, boundary_check=(0, 1)) 

456 if USE_GK: 

457 o_k2 = 64 + o_k1 

458 b_gk_last2 = tl.load(gk + last_idx * H * K + o_k2, mask=(o_k2 < K), other=0.) 

459 b_dv += tl.dot(b_k, b_dh2.to(b_k.dtype)) 

460 

461 if K > 128: 

462 p_k = tl.make_block_ptr(k, (T, K), (stride_k, 1), (i_t * BT, 128), (BT, 64), (1, 0)) 

463 b_k = tl.load(p_k, boundary_check=(0, 1)) 

464 if USE_GK: 

465 o_k3 = 128 + o_k1 

466 b_gk_last3 = tl.load(gk + last_idx * H * K + o_k3, mask=(o_k3 < K), other=0.) 

467 b_dv += tl.dot(b_k, b_dh3.to(b_k.dtype)) 

468 

469 if K > 192: 

470 p_k = tl.make_block_ptr(k, (T, K), (stride_k, 1), (i_t * BT, 192), (BT, 64), (1, 0)) 

471 b_k = tl.load(p_k, boundary_check=(0, 1)) 

472 if USE_GK: 

473 o_k4 = 192 + o_k1 

474 b_gk_last4 = tl.load(gk + last_idx * H * K + o_k4, mask=(o_k4 < K), other=0.) 

475 b_dv += tl.dot(b_k, b_dh4.to(b_k.dtype)) 

476 

477 if USE_G: 

478 m_t = (i_t * BT + tl.arange(0, BT)).to(tl.float32) < T 

479 b_dv *= (m_t * tl.exp(bg_last - b_g))[:, None] 

480 b_dv += tl.load(p_dv, boundary_check=(0, 1)) 

481 

482 tl.store(p_dv2, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1)) 

483 # Update dh 

484 p_w = tl.make_block_ptr(w, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1)) 

485 p_q = tl.make_block_ptr(q, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1)) 

486 b_w = tl.load(p_w, boundary_check=(0, 1)) 

487 b_q = tl.load(p_q, boundary_check=(0, 1)) 

488 if USE_G: 

489 b_dh1 *= bg_last_exp 

490 b_q = b_q * b_g_exp[None, :] 

491 if USE_GK: 

492 b_dh1 *= tl.exp(b_gk_last1[:, None]) 

493 b_dh1 += tl.dot(b_q.to(b_q.dtype), b_do.to(b_q.dtype)) * scale - tl.dot(b_w, b_dv.to(b_w.dtype)) 

494 if K > 64: 

495 p_q = tl.make_block_ptr(q, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1)) 

496 p_w = tl.make_block_ptr(w, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1)) 

497 b_q = tl.load(p_q, boundary_check=(0, 1)) 

498 b_w = tl.load(p_w, boundary_check=(0, 1)) 

499 if USE_G: 

500 b_dh2 *= bg_last_exp 

501 b_q = b_q * b_g_exp[None, :] 

502 if USE_GK: 

503 b_dh2 *= tl.exp(b_gk_last2[:, None]) 

504 b_dh2 += tl.dot(b_q.to(b_q.dtype), b_do.to(b_q.dtype)) * scale - tl.dot(b_w, b_dv.to(b_w.dtype)) 

505 if K > 128: 

506 p_q = tl.make_block_ptr(q, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1)) 

507 p_w = tl.make_block_ptr(w, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1)) 

508 b_q = tl.load(p_q, boundary_check=(0, 1)) 

509 b_w = tl.load(p_w, boundary_check=(0, 1)) 

510 if USE_G: 

511 b_dh3 *= bg_last_exp 

512 b_q = b_q * b_g_exp[None, :] 

513 if USE_GK: 

514 b_dh3 *= tl.exp(b_gk_last3[:, None]) 

515 b_dh3 += tl.dot(b_q.to(b_q.dtype), b_do.to(b_q.dtype)) * scale - tl.dot(b_w, b_dv.to(b_w.dtype)) 

516 if K > 192: 

517 p_q = tl.make_block_ptr(q, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1)) 

518 p_w = tl.make_block_ptr(w, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1)) 

519 b_q = tl.load(p_q, boundary_check=(0, 1)) 

520 b_w = tl.load(p_w, boundary_check=(0, 1)) 

521 if USE_G: 

522 b_dh4 *= bg_last_exp 

523 b_q = b_q * b_g_exp[None, :] 

524 if USE_GK: 

525 b_dh4 *= tl.exp(b_gk_last4[:, None]) 

526 b_dh4 += tl.dot(b_q.to(b_q.dtype), b_do.to(b_q.dtype)) * scale - tl.dot(b_w, b_dv.to(b_w.dtype)) 

527 

528 if USE_INITIAL_STATE: 

529 p_dh0 = tl.make_block_ptr(dh0, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) 

530 tl.store(p_dh0, b_dh1.to(p_dh0.dtype.element_ty), boundary_check=(0, 1)) 

531 if K > 64: 

532 p_dh1 = tl.make_block_ptr(dh0, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) 

533 tl.store(p_dh1, b_dh2.to(p_dh1.dtype.element_ty), boundary_check=(0, 1)) 

534 if K > 128: 

535 p_dh2 = tl.make_block_ptr(dh0, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) 

536 tl.store(p_dh2, b_dh3.to(p_dh2.dtype.element_ty), boundary_check=(0, 1)) 

537 if K > 192: 

538 p_dh3 = tl.make_block_ptr(dh0, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) 

539 tl.store(p_dh3, b_dh4.to(p_dh3.dtype.element_ty), boundary_check=(0, 1)) 

540 

541 

542def chunk_gated_delta_rule_bwd_dhu( 

543 q: torch.Tensor, 

544 k: torch.Tensor, 

545 w: torch.Tensor, 

546 do: torch.Tensor, 

547 dv: torch.Tensor, 

548 g: torch.Tensor | None = None, 

549 gk: torch.Tensor | None = None, 

550 h0: torch.Tensor | None = None, 

551 dht: torch.Tensor | None = None, 

552 scale: float | None = None, 

553 cu_seqlens: torch.LongTensor | None = None, 

554 chunk_size: int = 64, # SY: remove this argument and force chunk size 64? 

555 chunk_indices: torch.LongTensor | None = None, 

556 use_exp2: bool = False, 

557) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 

558 B, T, H, K, V = *q.shape, do.shape[-1] 

559 # N: the actual number of sequences in the batch with either equal or variable lengths 

560 BT = 64 

561 if K > 256: 

562 raise ValueError("current kernel does not support head dimension being larger than 256.") 

563 

564 if chunk_indices is None and cu_seqlens is not None: 

565 chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) 

566 if cu_seqlens is None: 

567 N, NT, chunk_offsets = B, triton.cdiv(T, BT), None 

568 else: 

569 N, NT, chunk_offsets = len(cu_seqlens) - 1, len(chunk_indices), prepare_chunk_offsets(cu_seqlens, BT) 

570 

571 dh = q.new_empty(B, NT, H, K, V) 

572 dh0 = torch.empty_like(h0, dtype=torch.float32) if h0 is not None else None 

573 dv2 = torch.empty_like(dv) 

574 

575 BV = 128 

576 

577 g = g.permute(0, 2, 1).contiguous() 

578 

579 chunk_gated_delta_rule_bwd_kernel_dhu_blockdim64[(triton.cdiv(V, BV), N * H)]( 

580 q=q, 

581 k=k, 

582 w=w, 

583 g=g, 

584 gk=gk, 

585 dht=dht, 

586 dh0=dh0, 

587 do=do, 

588 dh=dh, 

589 dv=dv, 

590 dv2=dv2, 

591 cu_seqlens=cu_seqlens, 

592 chunk_offsets=chunk_offsets, 

593 scale=scale, 

594 T=T, 

595 H=H, 

596 K=K, 

597 V=V, 

598 BT=BT, 

599 BV=BV, 

600 ) 

601 return dh, dh0, dv2