Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / _triton / gated_delta_net / chunk_o.py: 0%

298 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang 

16 

17# pylint: disable=missing-public-type-hints,missing-public-docstring,disallowed-name 

18# pylint: disable=invalid-name,missing-module-docstring,missing-function-docstring 

19# pylint: disable=unused-variable,too-many-nested-blocks 

20 

21from typing import Optional, Tuple 

22 

23import torch 

24import triton 

25import triton.language as tl 

26 

27from .utils import prepare_chunk_indices, exp, prepare_chunk_offsets 

28 

29 

30@triton.heuristics({ 

31 'USE_G': lambda args: args['g'] is not None, 

32 'USE_G_GAMMA': lambda args: args['g_gamma'] is not None, 

33 'USE_DW': lambda args: args['dw'] is not None, 

34 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None, 

35}) 

36@triton.jit(do_not_specialize=['T']) 

37def chunk_bwd_kernel_dqkwg( 

38 q, 

39 k, 

40 v, 

41 h, 

42 g, 

43 g_gamma, 

44 do, 

45 dh, 

46 dq, 

47 dk, 

48 dg, 

49 w, 

50 dv, 

51 dw, 

52 cu_seqlens, 

53 chunk_indices, 

54 scale, 

55 B: tl.constexpr, 

56 T, 

57 H: tl.constexpr, 

58 K: tl.constexpr, 

59 V: tl.constexpr, 

60 BT: tl.constexpr, 

61 BK: tl.constexpr, 

62 BV: tl.constexpr, 

63 USE_G: tl.constexpr, 

64 USE_G_GAMMA: tl.constexpr, 

65 USE_DW: tl.constexpr, 

66 IS_VARLEN: tl.constexpr, 

67 gdiff, 

68): 

69 i_t, i_b = tl.program_id(0), tl.program_id(1) 

70 T_max = T 

71 if IS_VARLEN: 

72 i_tg = i_t 

73 i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32) 

74 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

75 total = B * T_max 

76 T = eos - bos 

77 else: 

78 NT = tl.cdiv(T, BT) 

79 i_tg = i_b * NT + i_t 

80 bos, eos = i_b * T, i_b * T + T 

81 total = B * T_max 

82 

83 NK = tl.cdiv(K, BK) 

84 for i_k in range(NK): 

85 if USE_G: 

86 dg_k = dg + i_k * total * H 

87 

88 for i_h in range(H): 

89 v_h = v + (bos * H + i_h) * V 

90 do_h = do + (bos * H + i_h) * V 

91 h_h = h + (i_tg * H + i_h).to(tl.int64) * K * V 

92 dh_h = dh + (i_tg * H + i_h).to(tl.int64) * K * V 

93 q_h = q + (bos * H + i_h) * K 

94 k_h = k + (bos * H + i_h) * K 

95 dq_h = dq + (bos * H + i_h) * K 

96 dk_h = dk + (bos * H + i_h) * K 

97 

98 if USE_DW: 

99 w_h = w + (bos * H + i_h) * K 

100 dw_h = dw + (bos * H + i_h) * K 

101 dv_h = dv + (bos * H + i_h) * V 

102 

103 if USE_G: 

104 if IS_VARLEN: 

105 dg_h = dg_k + i_h * T_max + bos 

106 g_h = g + i_h * T_max + bos 

107 else: 

108 dg_h = dg_k + (i_b * H + i_h) * T_max 

109 g_h = g + (i_b * H + i_h) * T_max 

110 b_dg_last = tl.zeros([1, ], dtype=tl.float32) 

111 

112 if USE_G_GAMMA: 

113 b_gamma = tl.load(g_gamma + i_h) 

114 b_g = b_gamma * (tl.arange(0, BT) + 1) 

115 b_g_last = b_gamma * min(BT, T - i_t * BT) 

116 

117 b_dq = tl.zeros([BT, BK], dtype=tl.float32) 

118 b_dk = tl.zeros([BT, BK], dtype=tl.float32) 

119 b_ds = tl.zeros([BT, BT], dtype=tl.float32) 

120 b_dw = tl.zeros([BT, BK], dtype=tl.float32) if USE_DW else None 

121 

122 for i_v in range(tl.cdiv(V, BV)): 

123 p_v = tl.make_block_ptr(v_h, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

124 p_do = tl.make_block_ptr(do_h, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

125 p_h = tl.make_block_ptr(h_h, (V, K), (1, V), (i_v * BV, i_k * BK), (BV, BK), (0, 1)) 

126 p_dh = tl.make_block_ptr(dh_h, (V, K), (1, V), (i_v * BV, i_k * BK), (BV, BK), (0, 1)) 

127 

128 b_v = tl.load(p_v, boundary_check=(0, 1)) 

129 b_do = tl.load(p_do, boundary_check=(0, 1)) 

130 b_h = tl.load(p_h, boundary_check=(0, 1)) 

131 b_dh = tl.load(p_dh, boundary_check=(0, 1)) 

132 

133 if USE_G: 

134 b_dg_last += (tl.sum(b_h * b_dh)) 

135 

136 b_ds += tl.dot(b_do, tl.trans(b_v)) 

137 b_dq += tl.dot(b_do, b_h.to(b_do.dtype)) 

138 b_dk += tl.dot(b_v, b_dh.to(b_v.dtype)) 

139 

140 if USE_DW: 

141 p_dv = tl.make_block_ptr(dv_h, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

142 b_dv = tl.load(p_dv, boundary_check=(0, 1)) 

143 b_dw += tl.dot(b_dv.to(b_v.dtype), b_h.to(b_v.dtype)) 

144 

145 if USE_DW: 

146 p_dw = tl.make_block_ptr(dw_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

147 tl.store(p_dw, -b_dw.to(p_dw.dtype.element_ty), boundary_check=(0, 1)) 

148 

149 tl.debug_barrier() 

150 

151 p_q = tl.make_block_ptr(q_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

152 p_k = tl.make_block_ptr(k_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

153 b_q = tl.load(p_q, boundary_check=(0, 1)) 

154 b_k = tl.load(p_k, boundary_check=(0, 1)) 

155 

156 p_dq = tl.make_block_ptr(dq_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

157 p_dk = tl.make_block_ptr(dk_h, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

158 

159 o_t = i_t * BT + tl.arange(0, BT) 

160 m_t = o_t < T 

161 m_A = (o_t[:, None] >= o_t[None, :]) & (m_t[:, None] & m_t) 

162 

163 if USE_G: 

164 b_dg = tl.zeros([BT, ], dtype=tl.float32) 

165 p_g = tl.make_block_ptr(g_h, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

166 b_g = tl.load(p_g, boundary_check=(0,)) 

167 b_g_last = tl.load(g_h + (min(i_t * BT + BT, T) - 1) * 1) 

168 b_dg_last *= tl.exp(b_g_last) 

169 

170 b_dq = b_dq * tl.exp(b_g)[:, None] * scale 

171 b_dg += tl.sum(b_dq * b_q, axis=1) 

172 

173 b_dk = b_dk * tl.where(m_t, tl.exp(-b_g + b_g_last), 0)[:, None] 

174 b_dg -= tl.sum(b_k * b_dk, axis=1) 

175 b_dg_last += tl.sum(b_dk * b_k) 

176 

177 if IS_VARLEN: 

178 b_ds = tl.where(m_A, b_ds * exp(b_g[:, None] - b_g[None, :]), 0) * scale 

179 else: 

180 p_gdiff = tl.make_block_ptr(gdiff + i_b * H * NT * BT * BT + i_h * NT * BT * BT + i_t * BT * BT, 

181 (BT, BT), (BT, 1), (0, 0), (BT, BT), (1, 0)) 

182 gdiff_ = tl.load(p_gdiff) 

183 b_ds = b_ds * gdiff_ * scale 

184 

185 b_ds2 = b_ds * tl.dot(b_q, tl.trans(b_k)) 

186 b_dg += tl.sum(b_ds2, axis=1) 

187 b_dg -= tl.sum(b_ds2, axis=0) 

188 

189 b_ds = b_ds.to(b_k.dtype) 

190 b_dq += tl.dot(b_ds, b_k) 

191 b_dk += tl.dot(tl.trans(b_ds), b_q) 

192 p_dg = tl.make_block_ptr(dg_h, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

193 

194 last_index_local = min(BT, T - i_t * BT) - 1 

195 if last_index_local >= 0: 

196 is_last_mask = tl.arange(0, BT) == last_index_local 

197 b_dg = tl.where(is_last_mask, b_dg + b_dg_last, b_dg) 

198 else: 

199 pass 

200 

201 tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1)) 

202 tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1)) 

203 tl.store(p_dg, b_dg.to(p_dg.dtype.element_ty), boundary_check=(0,)) 

204 

205 elif USE_G_GAMMA: 

206 b_dq = b_dq * exp(b_g)[:, None] * scale 

207 b_dk = b_dk * tl.where(m_t, exp(-b_g + b_g_last), 0)[:, None] 

208 b_ds = tl.where(m_A, b_ds * exp(b_g[:, None] - b_g[None, :]), 0) * scale 

209 b_ds = b_ds.to(b_k.dtype) 

210 b_dq += tl.dot(b_ds, b_k) 

211 b_dk += tl.dot(tl.trans(b_ds), b_q) 

212 tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1)) 

213 tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1)) 

214 

215 else: 

216 b_ds = tl.where(m_A, b_ds, 0) 

217 b_ds = b_ds.to(b_k.dtype) 

218 b_dq += tl.dot(b_ds, b_k) 

219 b_dk += tl.dot(tl.trans(b_ds), b_q) * scale 

220 b_dq *= scale 

221 tl.store(p_dq, b_dq.to(p_dq.dtype.element_ty), boundary_check=(0, 1)) 

222 tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1)) 

223 

224 

225@triton.heuristics({ 

226 'USE_G': lambda args: args['g'] is not None, 

227 'USE_G_GAMMA': lambda args: args['g_gamma'] is not None, 

228 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None, 

229}) 

230@triton.jit(do_not_specialize=['T']) 

231def chunk_bwd_kernel_dv_local( 

232 q, 

233 k, 

234 g, 

235 g_gamma, 

236 do, 

237 dv, 

238 cu_seqlens, 

239 chunk_indices, 

240 scale, 

241 T, 

242 H: tl.constexpr, 

243 K: tl.constexpr, 

244 V: tl.constexpr, 

245 BT: tl.constexpr, 

246 BK: tl.constexpr, 

247 BV: tl.constexpr, 

248 USE_G: tl.constexpr, 

249 USE_G_GAMMA: tl.constexpr, 

250 IS_VARLEN: tl.constexpr, 

251): 

252 i_t, i_b = tl.program_id(0), tl.program_id(1) 

253 T_max = T 

254 

255 if IS_VARLEN: 

256 i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32) 

257 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

258 T = eos - bos 

259 else: 

260 bos, eos = i_b * T, i_b * T + T 

261 

262 for i_h in range(H): 

263 offset_kh = (bos * H + i_h) * K 

264 offset_vh = (bos * H + i_h) * V 

265 

266 b_A = tl.zeros([BT, BT], dtype=tl.float32) 

267 for i_k in range(tl.cdiv(K, BK)): 

268 p_k = tl.make_block_ptr(k + offset_kh, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

269 p_q = tl.make_block_ptr(q + offset_kh, (K, T), (1, H * K), (i_k * BK, i_t * BT), (BK, BT), (0, 1)) 

270 b_q = tl.load(p_q, boundary_check=(0, 1)) 

271 b_k = tl.load(p_k, boundary_check=(0, 1)) 

272 b_A += tl.dot(b_k, b_q) 

273 

274 if USE_G: 

275 if IS_VARLEN: 

276 offset_g = i_h * T_max + bos 

277 else: 

278 offset_g = i_b * H * T_max + i_h * T_max 

279 

280 p_g = tl.make_block_ptr(g + offset_g, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

281 b_g = tl.load(p_g, boundary_check=(0,)) 

282 

283 if USE_G_GAMMA: 

284 b_gamma = tl.load(g_gamma + i_h) 

285 b_g = b_gamma * (tl.arange(0, BT) + 1) 

286 

287 o_t = i_t * BT + tl.arange(0, BT) 

288 m_t = o_t < T 

289 m_A = (o_t[:, None] <= o_t[None, :]) & (m_t[:, None] & m_t) 

290 

291 if USE_G: 

292 b_A = tl.where(m_A, b_A * tl.exp(b_g[None, :] - b_g[:, None]) * scale, 0).to(do.dtype.element_ty) 

293 else: 

294 b_A = tl.where(m_A, b_A * scale, 0).to(do.dtype.element_ty) 

295 

296 for i_v in range(tl.cdiv(V, BV)): 

297 p_do = tl.make_block_ptr(do + offset_vh, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

298 p_dv = tl.make_block_ptr(dv + offset_vh, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

299 b_do = tl.load(p_do, boundary_check=(0, 1)) 

300 b_dv = tl.dot(b_A.to(b_do.dtype), b_do) 

301 tl.store(p_dv, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1)) 

302 

303 

304@triton.heuristics({ 

305 'USE_G': lambda args: args['g'] is not None, 

306 'USE_G_GAMMA': lambda args: args['g_gamma'] is not None, 

307 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None 

308}) 

309@triton.jit(do_not_specialize=['T']) 

310def chunk_fwd_kernel_o( 

311 q, 

312 k, 

313 v, 

314 h, 

315 g, 

316 g_gamma, 

317 o, 

318 cu_seqlens, 

319 chunk_offsets, 

320 scale, 

321 T, 

322 H: tl.constexpr, 

323 N: tl.constexpr, 

324 Hg: tl.constexpr, 

325 K: tl.constexpr, 

326 V: tl.constexpr, 

327 BT: tl.constexpr, 

328 BK: tl.constexpr, 

329 BV: tl.constexpr, 

330 USE_G: tl.constexpr, 

331 USE_G_GAMMA: tl.constexpr, 

332 IS_VARLEN: tl.constexpr, 

333): 

334 T_max = T 

335 for i_v in range(tl.cdiv(V, BV)): 

336 for i_n in range(N): 

337 if IS_VARLEN: 

338 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( 

339 cu_seqlens + i_n + 1 

340 ).to(tl.int32) 

341 T = eos - bos 

342 NT = tl.cdiv(T, BT) 

343 boh = tl.load(chunk_offsets + i_n).to(tl.int64) 

344 else: 

345 bos, eos = i_n * T, i_n * T + T 

346 NT = tl.cdiv(T, BT) 

347 boh = i_n * NT 

348 

349 core_id = tl.program_id(0) 

350 total_cores = tl.num_programs(0) 

351 base_chunks_per_pid = NT // total_cores 

352 remainder = NT % total_cores 

353 

354 if core_id < remainder: 

355 chunks_this_pid = base_chunks_per_pid + 1 

356 start_idx = core_id * chunks_this_pid 

357 else: 

358 chunks_this_pid = base_chunks_per_pid 

359 start_idx = core_id * base_chunks_per_pid + remainder 

360 

361 # offset calculation 

362 for i_h in range(0, H): 

363 q_offset = (bos * Hg + i_h // (H // Hg)) * K 

364 k_offset = (bos * Hg + i_h // (H // Hg)) * K 

365 v_offset = (bos * H + i_h) * V 

366 o_offset = (bos * H + i_h) * V 

367 

368 for i_t in range(start_idx, start_idx + chunks_this_pid): 

369 i_tg = boh + i_t 

370 h_base = h + (i_tg * H + i_h).to(tl.int64) * K * V 

371 b_o = tl.zeros([BT, BV], dtype=tl.float32) 

372 b_A = tl.zeros([BT, BT], dtype=tl.float32) 

373 for i_k in range(tl.cdiv(K, BK)): 

374 p_q = tl.make_block_ptr( 

375 q + q_offset, (T, K), (Hg * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0) 

376 ) 

377 p_k = tl.make_block_ptr( 

378 k + k_offset, (K, T), (1, Hg * K), (i_k * BK, i_t * BT), (BK, BT), (0, 1) 

379 ) 

380 p_h = tl.make_block_ptr( 

381 h_base, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0) 

382 ) 

383 b_q = tl.load(p_q, boundary_check=(0, 1)) 

384 b_k = tl.load(p_k, boundary_check=(0, 1)) 

385 b_h = tl.load(p_h, boundary_check=(0, 1)) 

386 

387 # [BT, BK] @ [BK, BV] -> [BT, BV] 

388 b_o += tl.dot(b_q, b_h) 

389 # [BT, BK] @ [BK, BT] -> [BT, BT] 

390 b_A += tl.dot(b_q, b_k) 

391 

392 if USE_G: 

393 if IS_VARLEN: 

394 p_g = tl.make_block_ptr(g + bos + i_h * T_max, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

395 else: 

396 p_g = tl.make_block_ptr(g + bos * H + i_h * T_max, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

397 b_g = tl.load(p_g, boundary_check=(0,)) 

398 b_o = b_o * exp(b_g)[:, None] 

399 b_A = b_A * exp(b_g[:, None] - b_g[None, :]) 

400 if USE_G_GAMMA: 

401 b_gamma = tl.load(g_gamma + i_h) 

402 b_g = b_gamma * (tl.arange(0, BT) + 1) 

403 

404 o_i = tl.arange(0, BT) 

405 m_A = o_i[:, None] >= o_i[None, :] 

406 b_A = tl.where(m_A, b_A, 0) 

407 

408 p_v = tl.make_block_ptr( 

409 v + v_offset, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) 

410 ) 

411 p_o = tl.make_block_ptr( 

412 o + o_offset, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) 

413 ) 

414 b_v = tl.load(p_v, boundary_check=(0, 1)) 

415 

416 # to fix mma -> mma layout conversion 

417 # already solved by triton v3.2 or higher 

418 b_o = b_o * scale + tl.dot(b_A.to(b_v.dtype), b_v) * scale 

419 tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) 

420 

421 

422def chunk_bwd_dqkwg( 

423 q: torch.Tensor, 

424 k: torch.Tensor, 

425 v: torch.Tensor, 

426 do: torch.Tensor, 

427 h: torch.Tensor, 

428 dh: torch.Tensor, 

429 g: Optional[torch.Tensor] = None, 

430 g_gamma: Optional[torch.Tensor] = None, 

431 dv: Optional[torch.Tensor] = None, 

432 w: Optional[torch.Tensor] = None, 

433 cu_seqlens: Optional[torch.LongTensor] = None, 

434 chunk_size: int = 64, 

435 scale: float = 1.0, 

436) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 

437 B, T, H, K, V = *k.shape, v.shape[-1] 

438 BT = min(chunk_size, max(16, triton.next_power_of_2(T))) 

439 chunk_indices = prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None 

440 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) 

441 

442 BK = 128 if cu_seqlens is None else 64 

443 BV = 64 

444 NK = triton.cdiv(K, BK) 

445 dq = torch.empty_like(q) 

446 dk = torch.empty_like(k) 

447 g = g.transpose(1, 2).contiguous() 

448 dg = torch.empty(NK, *g.shape, dtype=torch.float32, device=g.device) if g is not None else None 

449 dw = torch.empty_like(w) if w is not None else None 

450 grid = (NT, B) 

451 

452 if cu_seqlens is None: 

453 if NT * BT == T: 

454 g_ = g.reshape(B, H, NT, BT) 

455 g_diff = g_[:, :, :, :, None] - g_[:, :, :, None, :] 

456 g_diff = g_diff.clamp(-60, 60).exp() 

457 g_diff[:, :, :] *= torch.tril(torch.ones(BT, BT), diagonal=0).to(g.device) 

458 else: 

459 diff = NT * BT - T 

460 g_ = torch.cat((g, torch.zeros(B, H, diff).to(g.device)), dim=-1).reshape(B, H, NT, BT) 

461 g_diff = g_[:, :, :, :, None] - g_[:, :, :, None, :] 

462 g_diff = g_diff.clamp(-60, 60).exp() 

463 g_diff[:, :, :] *= torch.tril(torch.ones(BT, BT), diagonal=0).to(g.device) 

464 bias = torch.arange(0, BT).to(g.device) 

465 o_t = (NT - 1) * BT + bias 

466 m_t = o_t < T 

467 m_A = (m_t[:, None] & m_t) 

468 g_diff[:, :, -1] *= m_A 

469 else: 

470 g_diff = None 

471 

472 chunk_bwd_kernel_dqkwg[grid]( 

473 q=q, 

474 k=k, 

475 v=v, 

476 h=h, 

477 g=g, 

478 g_gamma=g_gamma, 

479 do=do, 

480 dh=dh, 

481 dv=dv, 

482 w=w, 

483 dw=dw, 

484 dq=dq, 

485 dk=dk, 

486 dg=dg, 

487 cu_seqlens=cu_seqlens, 

488 chunk_indices=chunk_indices, 

489 scale=scale, 

490 B=B, 

491 T=T, 

492 H=H, 

493 K=K, 

494 V=V, 

495 BT=BT, 

496 BK=BK, 

497 BV=BV, 

498 gdiff=g_diff, 

499 ) 

500 

501 if dg is not None: 

502 dg = dg.sum(0) 

503 dg = dg.transpose(1, 2).contiguous() 

504 return dq, dk, dw, dg 

505 

506 

507def chunk_bwd_dv_local( 

508 q: torch.Tensor, 

509 k: torch.Tensor, 

510 do: torch.Tensor, 

511 g: Optional[torch.Tensor] = None, 

512 g_gamma: Optional[torch.Tensor] = None, 

513 scale: float = None, 

514 cu_seqlens: Optional[torch.LongTensor] = None, 

515 chunk_size: int = 64 

516) -> torch.Tensor: 

517 B, T, H, K, V = *k.shape, do.shape[-1] 

518 BT = min(chunk_size, max(16, triton.next_power_of_2(T))) 

519 chunk_indices = prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None 

520 

521 BK = 128 

522 BV = 128 

523 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) 

524 

525 g = g.transpose(1, 2).contiguous() 

526 dv = torch.empty_like(do) 

527 grid = (NT, B) 

528 chunk_bwd_kernel_dv_local[grid]( 

529 q=q, 

530 k=k, 

531 g=g, 

532 g_gamma=g_gamma, 

533 do=do, 

534 dv=dv, 

535 cu_seqlens=cu_seqlens, 

536 chunk_indices=chunk_indices, 

537 scale=scale, 

538 T=T, 

539 H=H, 

540 K=K, 

541 V=V, 

542 BT=BT, 

543 BK=BK, 

544 BV=BV, 

545 ) 

546 return dv 

547 

548 

549def chunk_fwd_o( 

550 q: torch.Tensor, 

551 k: torch.Tensor, 

552 v: torch.Tensor, 

553 h: torch.Tensor, 

554 g: Optional[torch.Tensor] = None, 

555 g_gamma: Optional[torch.Tensor] = None, 

556 scale: Optional[float] = None, 

557 cu_seqlens: Optional[torch.LongTensor] = None, 

558 chunk_size: int = 64 

559) -> torch.Tensor: 

560 B, T, Hg, K, V = *q.shape, v.shape[-1] 

561 H = v.shape[-2] 

562 BT = min(chunk_size, max(16, triton.next_power_of_2(T))) 

563 chunk_indices = ( 

564 prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None 

565 ) 

566 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) 

567 if scale is None: 

568 scale = k.shape[-1] ** -0.5 

569 

570 output = torch.empty_like(v) 

571 if cu_seqlens is None: 

572 N, chunk_offsets = B, None 

573 else: 

574 N, chunk_offsets = ( 

575 len(cu_seqlens) - 1, 

576 prepare_chunk_offsets(cu_seqlens, BT), 

577 ) 

578 

579 g = g.transpose(1, 2).contiguous() 

580 h = h.contiguous() 

581 CV_kernel_num = 24 

582 chunk_fwd_kernel_o[(CV_kernel_num,)]( 

583 q, 

584 k, 

585 v, 

586 h, 

587 g, 

588 g_gamma, 

589 output, 

590 cu_seqlens, 

591 chunk_offsets, 

592 scale, 

593 T=T, 

594 H=H, 

595 N=N, 

596 Hg=Hg, 

597 K=K, 

598 V=V, 

599 BT=BT, 

600 BK=128, 

601 BV=128, 

602 ) 

603 return output 

604 

605bwd_chunk_dqkwg = chunk_bwd_dqkwg 

606bwd_chunk_dv_local = chunk_bwd_dv_local