Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / _triton / gated_delta_net / chunk_scaled_dot_kkt.py: 0%

153 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang 

16 

17# pylint: disable=line-too-long,missing-public-type-hints,missing-public-docstring 

18# pylint: disable=unused-argument,invalid-name,missing-module-docstring 

19# pylint: disable=missing-function-docstring 

20 

21from typing import Optional 

22 

23import torch 

24import triton 

25import triton.language as tl 

26 

27from .utils import prepare_chunk_indices 

28 

29 

30@triton.heuristics({ 

31 'USE_G': lambda args: args['g'] is not None, 

32 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None, 

33}) 

34@triton.jit(do_not_specialize=['T', 'NT', 'TOTAL_TASKS']) 

35def chunk_scaled_dot_kkt_fwd_kernel( 

36 k, 

37 g, 

38 beta, 

39 A, 

40 cu_seqlens, 

41 chunk_indices, 

42 T, 

43 H: tl.constexpr, 

44 K: tl.constexpr, 

45 BT: tl.constexpr, 

46 BK: tl.constexpr, 

47 IS_VARLEN: tl.constexpr, 

48 USE_G: tl.constexpr, 

49 NT, 

50 B, 

51 TOTAL_TASKS, 

52): 

53 core_id = tl.program_id(0) 

54 num_blocks = tl.num_programs(0) 

55 T_max = T 

56 

57 base_tasks_per_block = TOTAL_TASKS // num_blocks 

58 remainder_tasks = TOTAL_TASKS % num_blocks 

59 

60 if core_id < remainder_tasks: 

61 tasks_this_core = base_tasks_per_block + 1 

62 start_idx = core_id * tasks_this_core 

63 else: 

64 tasks_this_core = base_tasks_per_block 

65 start_idx = core_id * base_tasks_per_block + remainder_tasks 

66 

67 for idx in range(start_idx, start_idx + tasks_this_core): 

68 i_b = idx // NT 

69 local_idx = idx % NT 

70 

71 if IS_VARLEN: 

72 i_n = tl.load(chunk_indices + local_idx * 2).to(tl.int32) 

73 i_t = tl.load(chunk_indices + local_idx * 2 + 1).to(tl.int32) 

74 bos = tl.load(cu_seqlens + i_n).to(tl.int32) 

75 eos = tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

76 T_local = eos - bos 

77 else: 

78 bos, eos = 0, T 

79 i_t = local_idx 

80 T_local = T 

81 

82 for i_h in range(H): 

83 k_batch_off = i_b * T_max * H * K 

84 beta_batch_off = i_b * H * T_max 

85 g_batch_off = i_b * H * T_max 

86 A_batch_off = i_b * T_max * H * BT 

87 

88 p_beta = tl.make_block_ptr(beta + beta_batch_off + bos + i_h * T_max, (T_local,), (1,), (i_t * BT,), (BT,), (0,)) 

89 b_beta = tl.load(p_beta, boundary_check=(0,)) 

90 

91 b_A = tl.zeros([BT, BT], dtype=tl.float32) 

92 for i_k in range(tl.cdiv(K, BK)): 

93 p_k = tl.make_block_ptr(k + k_batch_off + (bos * H + i_h) * K, (T_local, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

94 b_k = tl.load(p_k, boundary_check=(0, 1)) 

95 dot_product = tl.dot(b_k, tl.trans(b_k)) 

96 

97 o_t = i_t * BT + tl.arange(0, BT) 

98 o_t = o_t.to(tl.float32) 

99 T_mask = (o_t < T_local).to(tl.float32) 

100 

101 row_indices = tl.arange(0, BT)[:, None] 

102 col_indices = tl.arange(0, BT)[None, :] 

103 tril_mask = (row_indices > col_indices).to(tl.float32) 

104 tril_mask = tril_mask * T_mask[:, None] 

105 masked_dot = dot_product * tril_mask 

106 b_A += masked_dot 

107 

108 if USE_G: 

109 p_g = tl.make_block_ptr(g + g_batch_off + bos + i_h * T_max, (T_local,), (1,), (i_t * BT,), (BT,), (0,)) 

110 b_g = tl.load(p_g, boundary_check=(0,)) 

111 b_g_diff = b_g[:, None] - b_g[None, :] 

112 b_g_diff = tl.minimum(tl.maximum(b_g_diff, -50.0), 50.0) 

113 b_A *= tl.exp(b_g_diff) 

114 b_A *= b_beta[:, None] 

115 

116 p_A = tl.make_block_ptr(A + A_batch_off + (bos * H + i_h) * BT, (T_local, BT), (BT * H, 1), (i_t * BT, 0), (BT, BT), (1, 0)) 

117 tl.store(p_A, b_A.to(p_A.dtype.element_ty), boundary_check=(0, 1)) 

118 

119 

120@triton.heuristics({ 

121 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None 

122}) 

123@triton.autotune( 

124 configs=[ 

125 triton.Config({'BK': BK}) 

126 for BK in [32, 64] 

127 ], 

128 key=["BC"] 

129) 

130@triton.jit(do_not_specialize=['T']) 

131def chunk_scaled_dot_kkt_fwd_kernel_intra_sub_inter( 

132 k, 

133 g, 

134 beta, 

135 A, 

136 cu_seqlens, 

137 chunk_indices, 

138 T, 

139 H: tl.constexpr, 

140 K: tl.constexpr, 

141 BT: tl.constexpr, 

142 BC: tl.constexpr, 

143 BK: tl.constexpr, 

144 NC: tl.constexpr, 

145 IS_VARLEN: tl.constexpr, 

146): 

147 i_t, i_c, i_b = tl.program_id(0), tl.program_id(1), tl.program_id(2) 

148 i_i, i_j = i_c // NC, i_c % NC 

149 

150 for i_h in range(H): 

151 if IS_VARLEN: 

152 i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32) 

153 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

154 T_val = eos - bos 

155 else: 

156 bos, eos = i_b * T, i_b * T + T 

157 T_val = T 

158 

159 should_compute = (i_t * BT + i_i * BC < T_val) and (i_i > i_j) 

160 

161 if should_compute: 

162 k_ptr = k + (bos * H + i_h) * K 

163 g_ptr = g + (bos * H + i_h) * K 

164 A_ptr = A + (bos * H + i_h) * BT 

165 

166 p_beta = tl.make_block_ptr(beta + bos * H + i_h, (T_val,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,)) 

167 b_beta = tl.load(p_beta, boundary_check=(0,)) 

168 

169 b_A = tl.zeros([BC, BC], dtype=tl.float32) 

170 for i_k in range(tl.cdiv(K, BK)): 

171 p_k = tl.make_block_ptr(k_ptr, (T_val, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), 

172 (1, 0)) 

173 p_g = tl.make_block_ptr(g_ptr, (T_val, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), 

174 (1, 0)) 

175 b_kt = tl.make_block_ptr(k_ptr, (K, T_val), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), 

176 (0, 1)) 

177 p_gk = tl.make_block_ptr(g_ptr, (K, T_val), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), 

178 (0, 1)) 

179 

180 o_k = i_k * BK + tl.arange(0, BK) 

181 m_k = o_k < K 

182 b_gn = tl.load(g_ptr + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0) 

183 b_g = tl.load(p_g, boundary_check=(0, 1)) 

184 b_k = tl.load(p_k, boundary_check=(0, 1)) * tl.exp(b_g - b_gn[None, :]) 

185 b_gk = tl.load(p_gk, boundary_check=(0, 1)) 

186 b_kt = tl.load(b_kt, boundary_check=(0, 1)) * tl.exp(b_gn[:, None] - b_gk) 

187 b_A += tl.dot(b_k, b_kt) 

188 b_A *= b_beta[:, None] 

189 

190 p_A = tl.make_block_ptr(A_ptr, (T_val, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0)) 

191 tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1)) 

192 

193 

194@triton.heuristics({ 

195 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None 

196}) 

197@triton.jit(do_not_specialize=['T']) 

198def chunk_scaled_dot_kkt_fwd_kernel_intra_sub_intra( 

199 k, 

200 g, 

201 beta, 

202 A, 

203 cu_seqlens, 

204 chunk_indices, 

205 T, 

206 H: tl.constexpr, 

207 K: tl.constexpr, 

208 BT: tl.constexpr, 

209 BC: tl.constexpr, 

210 BK: tl.constexpr, 

211 IS_VARLEN: tl.constexpr, 

212): 

213 i_t, i_i, i_b = tl.program_id(0), tl.program_id(1), tl.program_id(2) 

214 

215 for i_h in range(H): 

216 if IS_VARLEN: 

217 i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32) 

218 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

219 T_val = eos - bos 

220 else: 

221 bos, eos = i_b * T, i_b * T + T 

222 T_val = T 

223 

224 should_compute = i_t * BT + i_i * BC < T_val 

225 

226 if should_compute: 

227 o_i = tl.arange(0, BC) 

228 o_k = tl.arange(0, BK) 

229 m_k = o_k < K 

230 m_A = (i_t * BT + i_i * BC + o_i) < T_val 

231 o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC 

232 

233 p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T_val, K), (H * K, 1), (i_t * BT + i_i * BC, 0), (BC, BK), 

234 (1, 0)) 

235 p_g = tl.make_block_ptr(g + (bos * H + i_h) * K, (T_val, K), (H * K, 1), (i_t * BT + i_i * BC, 0), (BC, BK), 

236 (1, 0)) 

237 p_beta = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h 

238 

239 b_k = tl.load(p_k, boundary_check=(0, 1)) * tl.load(p_beta, mask=m_A, other=0)[:, None] 

240 b_g = tl.load(p_g, boundary_check=(0, 1)) 

241 

242 p_kt = k + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k 

243 p_gk = g + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k 

244 

245 for j in range(0, min(BC, T_val - i_t * BT - i_i * BC)): 

246 b_kt = tl.load(p_kt, mask=m_k, other=0).to(tl.float32) 

247 b_gk = tl.load(p_gk, mask=m_k, other=0).to(tl.float32) 

248 b_A = tl.sum(b_k * b_kt[None, :] * tl.exp(b_g - b_gk[None, :]), 1) 

249 # 转化成f32 

250 o_i_tmp = o_i.to(tl.float32) 

251 b_A = tl.where(o_i_tmp > j, b_A, 0.) 

252 

253 tl.store(A + o_A + j, b_A, mask=m_A) 

254 p_kt += H * K 

255 p_gk += H * K 

256 

257 

258def chunk_scaled_dot_kkt_fwd( 

259 k: torch.Tensor, 

260 g: Optional[torch.Tensor] = None, 

261 gk: Optional[torch.Tensor] = None, 

262 beta: Optional[torch.Tensor] = None, 

263 cu_seqlens: Optional[torch.LongTensor] = None, 

264 chunk_size: int = 64, 

265 output_dtype: torch.dtype = torch.float32 

266) -> torch.Tensor: 

267 r""" 

268 Compute beta * K * K^T. 

269 

270 Args: 

271 k (torch.Tensor): 

272 The key tensor of shape `[B, T, H, K]`. 

273 beta (torch.Tensor): 

274 The beta tensor of shape `[B, T, H]`. 

275 g (torch.Tensor): 

276 The cumulative sum of the gate tensor of shape `[B, T, H]`. Default: `None`. 

277 gk (torch.Tensor): 

278 The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`. 

279 cu_seqlens (torch.LongTensor): 

280 The cumulative sequence lengths of the input tensor. 

281 Default: None 

282 chunk_size (int): 

283 The chunk size. Default: 64. 

284 output_dtype (torch.dtype): 

285 The dtype of the output tensor. Default: `torch.float32` 

286 

287 Returns: 

288 beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size. 

289 """ 

290 B, T, H, K = k.shape 

291 BT = chunk_size 

292 chunk_indices = prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None 

293 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) 

294 beta = beta.transpose(1, 2).contiguous() 

295 g = g.transpose(1, 2).contiguous() 

296 BK = 128 

297 kernel_num = 24 

298 

299 if gk is None: 

300 A = torch.empty(B, T, H, BT, device=k.device, dtype=output_dtype) 

301 chunk_scaled_dot_kkt_fwd_kernel[(kernel_num,)]( 

302 k=k, 

303 g=g, 

304 beta=beta, 

305 A=A, 

306 cu_seqlens=cu_seqlens, 

307 chunk_indices=chunk_indices, 

308 T=T, 

309 H=H, 

310 K=K, 

311 BT=BT, 

312 BK=BK, 

313 NT=NT, 

314 B=B, 

315 TOTAL_TASKS=B * NT, 

316 ) 

317 return A 

318 

319 BC = min(16, BT) 

320 NC = triton.cdiv(BT, BC) 

321 BK = max(triton.next_power_of_2(K), 16) 

322 A = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype) 

323 grid = (NT, NC * NC, B) 

324 chunk_scaled_dot_kkt_fwd_kernel_intra_sub_inter[grid]( 

325 k=k, 

326 g=gk, 

327 beta=beta, 

328 A=A, 

329 cu_seqlens=cu_seqlens, 

330 chunk_indices=chunk_indices, 

331 T=T, 

332 H=H, 

333 K=K, 

334 BT=BT, 

335 BC=BC, 

336 NC=NC, 

337 ) 

338 

339 grid = (NT, NC, B) 

340 chunk_scaled_dot_kkt_fwd_kernel_intra_sub_intra[grid]( 

341 k=k, 

342 g=gk, 

343 beta=beta, 

344 A=A, 

345 cu_seqlens=cu_seqlens, 

346 chunk_indices=chunk_indices, 

347 T=T, 

348 H=H, 

349 K=K, 

350 BT=BT, 

351 BC=BC, 

352 BK=BK, 

353 ) 

354 return A