Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / _triton / gated_delta_net / wy_fast.py: 0%

177 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang 

16# Copyright (c) 2026, Huawei Technologies Co., Ltd. All rights reserved. 

17 

18# pylint: disable=line-too-long,missing-public-type-hints,missing-public-docstring 

19# pylint: disable=invalid-name,missing-module-docstring,missing-function-docstring 

20 

21from typing import Optional, Tuple 

22 

23import torch 

24import triton 

25import triton.language as tl 

26 

27from .utils import prepare_chunk_indices, exp 

28 

29 

30@triton.heuristics({ 

31 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None 

32}) 

33@triton.jit(do_not_specialize=['T']) 

34def prepare_wy_repr_bwd_kernel( 

35 k, 

36 v, 

37 beta, 

38 g, 

39 A, 

40 dw, 

41 du, 

42 dk, 

43 dv, 

44 dbeta, 

45 dg, 

46 cu_seqlens, 

47 chunk_indices, 

48 T, 

49 B, 

50 H: tl.constexpr, 

51 K: tl.constexpr, 

52 V: tl.constexpr, 

53 NT: tl.constexpr, 

54 BT: tl.constexpr, 

55 BK: tl.constexpr, 

56 BV: tl.constexpr, 

57 IS_VARLEN: tl.constexpr 

58): 

59 core_id = tl.program_id(0) 

60 total_cores = tl.num_programs(0) 

61 T_max = T 

62 

63 base_chunks_per_pid = NT // total_cores 

64 remainder_chunks = NT % total_cores 

65 

66 if core_id < remainder_chunks: 

67 chunks_this_pid = base_chunks_per_pid + 1 

68 start_idx = core_id * chunks_this_pid 

69 else: 

70 chunks_this_pid = base_chunks_per_pid 

71 start_idx = core_id * chunks_this_pid + remainder_chunks 

72 

73 for idx in range(start_idx, start_idx + chunks_this_pid): 

74 for i_b in range(B): 

75 if IS_VARLEN: 

76 i_n, i_t = tl.load(chunk_indices + idx * 2).to(tl.int32), tl.load(chunk_indices + idx * 2 + 1).to(tl.int32) 

77 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

78 T = eos - bos 

79 else: 

80 i_t = idx 

81 bos, eos = i_b * T, i_b * T + T 

82 

83 o_t = i_t * BT + tl.arange(0, BT) 

84 m_t = o_t < T 

85 m_A = (o_t[:, None] > o_t[None, :]) & (m_t[:, None] & m_t) 

86 for i_h in range(0, H): 

87 if IS_VARLEN: 

88 offset = bos + i_h * T_max 

89 else: 

90 offset = bos * H + i_h * T_max 

91 

92 p_beta = tl.make_block_ptr(beta + offset, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

93 p_g = tl.make_block_ptr(g + offset, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

94 p_A = tl.make_block_ptr(A + (bos * H + i_h) * BT, (BT, T), (1, H * BT), (0, i_t * BT), (BT, BT), (0, 1)) 

95 

96 b_A = tl.load(p_A, boundary_check=(0, 1)) 

97 b_beta = tl.load(p_beta, boundary_check=(0,)) 

98 b_g = tl.load(p_g, boundary_check=(0,)) 

99 b_g_exp = tl.exp(b_g) 

100 

101 b_dbeta = tl.zeros([BT], dtype=tl.float32) 

102 b_dA = tl.zeros([BT, BT], dtype=tl.float32) 

103 b_dg = tl.zeros([BT], dtype=tl.float32) 

104 

105 for i_k in range(tl.cdiv(K, BK)): 

106 p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

107 p_dk = tl.make_block_ptr(dk + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

108 p_dw = tl.make_block_ptr(dw + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

109 b_k = tl.load(p_k, boundary_check=(0, 1)) 

110 b_k_beta_g = (b_k * b_beta[:, None] * b_g_exp[:, None]).to(b_k.dtype) 

111 b_dw = tl.load(p_dw, boundary_check=(0, 1)) 

112 b_dA += tl.dot(b_dw, tl.trans(b_k_beta_g)) 

113 b_dk_beta_g = tl.dot(b_A, b_dw) 

114 b_dk = b_dk_beta_g * b_beta[:, None] * b_g_exp[:, None] 

115 b_dbeta += tl.sum(b_dk_beta_g * b_k * b_g_exp[:, None], 1) 

116 b_dg += tl.sum(b_dk_beta_g * b_k * b_g_exp[:, None] * b_beta[:, None], 1) 

117 tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1)) 

118 

119 for i_v in range(tl.cdiv(V, BV)): 

120 p_v = tl.make_block_ptr(v + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

121 p_dv = tl.make_block_ptr(dv + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

122 p_du = tl.make_block_ptr(du + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

123 b_v = tl.load(p_v, boundary_check=(0, 1)) 

124 b_v_beta = (b_v * b_beta[:, None]).to(b_v.dtype) 

125 b_du = tl.load(p_du, boundary_check=(0, 1)) 

126 b_dA += tl.dot(b_du, tl.trans(b_v_beta)) 

127 b_dv_beta = tl.dot(b_A, b_du) 

128 b_dv = b_dv_beta * b_beta[:, None] 

129 b_dbeta += tl.sum(b_dv_beta * b_v, 1) 

130 tl.store(p_dv, b_dv.to(p_dv.dtype.element_ty), boundary_check=(0, 1)) 

131 

132 b_dA = tl.where(m_A, b_dA, 0) 

133 b_dA = tl.dot(b_dA.to(b_A.dtype), b_A) 

134 b_dA = tl.dot(b_A, b_dA.to(b_A.dtype)) 

135 b_dA = tl.where(m_A, -b_dA * exp(b_g[:, None] - b_g[None, :]), 0) 

136 b_dA = b_dA.to(k.dtype.element_ty) 

137 b_A = tl.zeros([BT, BT], dtype=tl.float32) 

138 

139 for i_k in range(tl.cdiv(K, BK)): 

140 p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

141 p_dk = tl.make_block_ptr(dk + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

142 b_k = tl.load(p_k, boundary_check=(0, 1)) 

143 b_dk = tl.load(p_dk, boundary_check=(0, 1)) 

144 b_k_beta = (b_k * b_beta[:, None]).to(b_k.dtype) 

145 b_A += tl.dot(b_k_beta, tl.trans(b_k)) 

146 b_dk_beta = tl.dot(b_dA, b_k) 

147 b_dbeta += tl.sum(b_dk_beta * b_k, 1) 

148 b_dk += tl.dot(tl.trans(b_dA), b_k_beta) 

149 b_dk += b_dk_beta * b_beta[:, None] 

150 tl.store(p_dk, b_dk.to(p_dk.dtype.element_ty), boundary_check=(0, 1)) 

151 

152 b_dA_A = b_dA * b_A 

153 b_dg += tl.sum(b_dA_A, axis=1) - tl.sum(b_dA_A, axis=0) 

154 p_dg = tl.make_block_ptr(dg + offset, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

155 p_dbeta = tl.make_block_ptr(dbeta + offset, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

156 tl.store(p_dg, b_dg.to(p_dg.dtype.element_ty), boundary_check=(0,)) 

157 tl.store(p_dbeta, b_dbeta.to(p_dbeta.dtype.element_ty), boundary_check=(0,)) 

158 

159 

160@triton.heuristics({ 

161 'USE_G': lambda args: args['g'] is not None, 

162 'USE_GK': lambda args: args['gk'] is not None, 

163 'IS_VARLEN': lambda args: args['cu_seqlens'] is not None 

164}) 

165@triton.jit(do_not_specialize=['T']) 

166def recompute_w_u_fwd_kernel( 

167 k, 

168 v, 

169 beta, 

170 w, 

171 u, 

172 A, 

173 g, 

174 gk, 

175 cu_seqlens, 

176 chunk_indices, 

177 T_tmp, 

178 B, 

179 H: tl.constexpr, 

180 K: tl.constexpr, 

181 V: tl.constexpr, 

182 NT: tl.constexpr, 

183 BT: tl.constexpr, 

184 BK: tl.constexpr, 

185 BV: tl.constexpr, 

186 USE_G: tl.constexpr, 

187 USE_GK: tl.constexpr, 

188 IS_VARLEN: tl.constexpr 

189): 

190 core_id = tl.program_id(0) 

191 total_cores = tl.num_programs(0) 

192 T_max = T_tmp 

193 

194 base_chunks_per_pid = NT // total_cores 

195 remainder_chunks = NT % total_cores 

196 

197 if core_id < remainder_chunks: 

198 chunks_this_pid = base_chunks_per_pid + 1 

199 start_idx = core_id * chunks_this_pid 

200 else: 

201 chunks_this_pid = base_chunks_per_pid 

202 start_idx = core_id * chunks_this_pid + remainder_chunks 

203 

204 for idx in range(start_idx, start_idx + chunks_this_pid): 

205 for i_b in range(B): 

206 for i_h in range(0, H): 

207 

208 if IS_VARLEN: 

209 i_n, i_t = tl.load(chunk_indices + idx * 2).to(tl.int32), tl.load(chunk_indices + idx * 2 + 1).to(tl.int32) 

210 bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(cu_seqlens + i_n + 1).to(tl.int32) 

211 offset = bos + i_h * T_max 

212 T = eos - bos 

213 else: 

214 T = T_tmp 

215 i_t = idx 

216 bos, eos = i_b * T, i_b * T + T 

217 offset = bos * H + i_h * T_max 

218 

219 p_beta = tl.make_block_ptr(beta + offset, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

220 b_beta = tl.load(p_beta, boundary_check=(0,)) 

221 

222 p_A = tl.make_block_ptr(A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)) 

223 b_A = tl.load(p_A, boundary_check=(0, 1)) 

224 

225 for i_v in range(tl.cdiv(V, BV)): 

226 p_v = tl.make_block_ptr(v + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

227 p_u = tl.make_block_ptr(u + (bos * H + i_h) * V, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)) 

228 b_v = tl.load(p_v, boundary_check=(0, 1)) 

229 b_vb = (b_v * b_beta[:, None]).to(b_v.dtype) 

230 b_u = tl.dot(b_A, b_vb, allow_tf32=False) 

231 tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1)) 

232 

233 if USE_G: 

234 p_g = tl.make_block_ptr(g + offset, (T,), (1,), (i_t * BT,), (BT,), (0,)) 

235 b_g = tl.exp(tl.load(p_g, boundary_check=(0,))) 

236 

237 for i_k in range(tl.cdiv(K, BK)): 

238 p_k = tl.make_block_ptr(k + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

239 p_w = tl.make_block_ptr(w + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

240 b_k = tl.load(p_k, boundary_check=(0, 1)) 

241 b_kb = b_k * b_beta[:, None] 

242 if USE_G: 

243 b_kb *= b_g[:, None] 

244 if USE_GK: 

245 p_gk = tl.make_block_ptr(gk + (bos * H + i_h) * K, (T, K), (H * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)) 

246 b_kb *= tl.exp(tl.load(p_gk, boundary_check=(0, 1))) 

247 b_w = tl.dot(b_A, b_kb.to(b_k.dtype)) 

248 tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1)) 

249 

250 

251def recompute_w_u_fwd( 

252 k: torch.Tensor, 

253 v: torch.Tensor, 

254 beta: torch.Tensor, 

255 A: torch.Tensor, 

256 g: Optional[torch.Tensor] = None, 

257 gk: Optional[torch.Tensor] = None, 

258 cu_seqlens: Optional[torch.LongTensor] = None, 

259) -> Tuple[torch.Tensor, torch.Tensor]: 

260 B, T, H, K, V = *k.shape, v.shape[-1] 

261 BT = A.shape[-1] 

262 BK = 128 

263 BV = 128 

264 

265 chunk_indices = prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None 

266 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) 

267 g = g.transpose(1, 2).contiguous() if g is not None else None 

268 beta = beta.transpose(1, 2).contiguous() 

269 

270 w = torch.empty_like(k) 

271 u = torch.empty_like(v) 

272 cv_kernel_num = 24 

273 recompute_w_u_fwd_kernel[(cv_kernel_num,)]( 

274 k=k, 

275 v=v, 

276 beta=beta, 

277 w=w, 

278 u=u, 

279 A=A, 

280 g=g, 

281 gk=gk, 

282 cu_seqlens=cu_seqlens, 

283 chunk_indices=chunk_indices, 

284 T_tmp=T, 

285 B=B, 

286 H=H, 

287 K=K, 

288 V=V, 

289 NT=NT, 

290 BT=BT, 

291 BK=BK, 

292 BV=BV, 

293 ) 

294 return w, u 

295 

296 

297def prepare_wy_repr_bwd( 

298 k: torch.Tensor, 

299 v: torch.Tensor, 

300 g: torch.Tensor, 

301 beta: torch.Tensor, 

302 A: torch.Tensor, 

303 dw: torch.Tensor, 

304 du: torch.Tensor, 

305 cu_seqlens: Optional[torch.LongTensor], 

306 chunk_size: int = 64, 

307) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 

308 B, T, H, K, V = *k.shape, v.shape[-1] 

309 BT = chunk_size 

310 chunk_indices = prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None 

311 NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) 

312 BK = 128 

313 BV = 128 

314 beta = beta.transpose(1, 2).contiguous() 

315 g = g.transpose(1, 2).contiguous() 

316 

317 dk = torch.empty_like(k) 

318 dv = torch.empty_like(v) 

319 dbeta = torch.empty_like(beta) 

320 dg = torch.empty_like(g) 

321 

322 cv_kernel_num = 24 

323 prepare_wy_repr_bwd_kernel[(cv_kernel_num,)]( 

324 k=k, 

325 v=v, 

326 beta=beta, 

327 g=g, 

328 A=A, 

329 dw=dw, 

330 du=du, 

331 dk=dk, 

332 dv=dv, 

333 dbeta=dbeta, 

334 dg=dg, 

335 cu_seqlens=cu_seqlens, 

336 chunk_indices=chunk_indices, 

337 T=T, 

338 B=B, 

339 H=H, 

340 K=K, 

341 V=V, 

342 NT=NT, 

343 BT=BT, 

344 BK=BK, 

345 BV=BV, 

346 ) 

347 

348 dbeta = dbeta.transpose(1, 2).contiguous() 

349 dg = dg.transpose(1, 2).contiguous() 

350 

351 return dk, dv, dbeta, dg 

352 

353 

354bwd_prepare_wy_repr = prepare_wy_repr_bwd 

355 

356fwd_recompute_w_u = recompute_w_u_fwd