Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / kimi_delta_attention_state_summary.py: 20%

79 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Affine KDA state summary built from prepared WY intermediates.""" 

16 

17 

18__all__ = [ 

19 "apply_kda_state_gradient_summary", 

20 "apply_kda_state_summary", 

21 "kda_state_gradient_summary_from_prepared", 

22 "kda_state_summary_forward_from_prepared", 

23] 

24 

25from typing import Optional 

26 

27import torch 

28import triton 

29 

30 

31def _validate_prepared_summary_inputs( 

32 key: torch.Tensor, 

33 w: torch.Tensor, 

34 u: torch.Tensor, 

35 gate: torch.Tensor, 

36 chunk_size: int, 

37) -> tuple[int, int, int, int, int]: 

38 """Validate the fixed Kimi K3 summary contract and return dimensions.""" 

39 if (key.ndim, w.ndim, u.ndim, gate.ndim) != (4, 4, 4, 4): 

40 raise ValueError("KDA prepared summary expects rank-4 key/w/u/gate tensors.") 

41 batch, sequence_length, heads, key_dim = key.shape 

42 value_dim = u.shape[-1] 

43 if key_dim != 128 or value_dim != 128 or chunk_size != 64: 

44 raise NotImplementedError( 

45 "Triton KDA state summary requires key_dim=value_dim=128 and " 

46 "chunk_size=64." 

47 ) 

48 if sequence_length <= 0: 

49 raise ValueError("State summary requires a nonempty sample fragment.") 

50 if w.shape != key.shape or gate.shape != key.shape: 

51 raise ValueError( 

52 "KDA prepared key, w, and gate must have identical shapes, got " 

53 f"key={tuple(key.shape)}, w={tuple(w.shape)}, gate={tuple(gate.shape)}." 

54 ) 

55 if u.shape[:3] != key.shape[:3]: 

56 raise ValueError( 

57 f"KDA prepared u prefix {tuple(u.shape[:3])} must match " 

58 f"key prefix {tuple(key.shape[:3])}." 

59 ) 

60 if not key.is_npu: 

61 raise RuntimeError("Triton KDA state summary requires Ascend NPU tensors.") 

62 return batch, sequence_length, heads, key_dim, value_dim 

63 

64 

65def _validate_gradient_summary_inputs( 

66 query: torch.Tensor, 

67 key: torch.Tensor, 

68 w: torch.Tensor, 

69 gate: torch.Tensor, 

70 grad_output: torch.Tensor, 

71 grad_value: torch.Tensor, 

72 chunk_size: int, 

73) -> tuple[int, int, int, int, int]: 

74 """Validate prepared tensors used by the reverse state wavefront.""" 

75 tensors = (query, key, w, gate, grad_output, grad_value) 

76 if any(tensor.ndim != 4 for tensor in tensors): 

77 raise ValueError( 

78 "KDA gradient summary expects rank-4 query/key/w/gate/do/dv tensors." 

79 ) 

80 batch, sequence_length, heads, key_dim = query.shape 

81 value_dim = grad_output.shape[-1] 

82 if key_dim != 128 or value_dim != 128 or chunk_size != 64: 

83 raise NotImplementedError( 

84 "Triton KDA state-gradient summary requires key_dim=value_dim=128 " 

85 "and chunk_size=64." 

86 ) 

87 if sequence_length <= 0: 

88 raise ValueError("State summary requires a nonempty sample fragment.") 

89 prepared_shape = (batch, sequence_length, heads, key_dim) 

90 value_shape = (batch, sequence_length, heads, value_dim) 

91 if key.shape != prepared_shape or w.shape != prepared_shape: 

92 raise ValueError( 

93 "KDA prepared query, key, and w must have identical shapes, got " 

94 f"query={tuple(query.shape)}, key={tuple(key.shape)}, " 

95 f"w={tuple(w.shape)}." 

96 ) 

97 if gate.shape != prepared_shape: 

98 raise ValueError( 

99 f"KDA prepared gate shape {tuple(gate.shape)} must match " 

100 f"query shape {prepared_shape}." 

101 ) 

102 if grad_output.shape != value_shape or grad_value.shape != value_shape: 

103 raise ValueError( 

104 "KDA grad_output and grad_value must have shape " 

105 f"{value_shape}, got do={tuple(grad_output.shape)}, " 

106 f"dv={tuple(grad_value.shape)}." 

107 ) 

108 if not query.is_npu: 

109 raise RuntimeError( 

110 "Triton KDA state-gradient summary requires Ascend NPU tensors." 

111 ) 

112 return batch, sequence_length, heads, key_dim, value_dim 

113 

114 

115def _launch_kda_state_summary( 

116 key: torch.Tensor, 

117 w: torch.Tensor, 

118 u: torch.Tensor, 

119 gate: torch.Tensor, 

120 *, 

121 chunk_size: int, 

122 output_transition: bool = True, 

123) -> tuple[torch.Tensor, Optional[torch.Tensor]]: 

124 """Launch the shared S/M summary used by the state CP protocols.""" 

125 return _launch_kda_paired_state_summary( 

126 key, 

127 w, 

128 u, 

129 gate, 

130 chunk_size=chunk_size, 

131 output_transition=output_transition, 

132 ) 

133 

134 

135def _launch_kda_state_gradient_summary( 

136 query: torch.Tensor, 

137 key: torch.Tensor, 

138 w: torch.Tensor, 

139 gate: torch.Tensor, 

140 grad_output: torch.Tensor, 

141 grad_value: torch.Tensor, 

142 scale: float, 

143 *, 

144 chunk_size: int, 

145 block_size: int = 64, 

146) -> torch.Tensor: 

147 """Launch the reverse-wavefront Triton-Ascend summary kernel.""" 

148 from ._triton.kimi_delta_attention.state_summary import ( # pylint: disable=import-outside-toplevel 

149 kda_state_grad_ext_kernel, 

150 ) 

151 

152 batch, sequence_length, heads, key_dim, value_dim = ( 

153 _validate_gradient_summary_inputs( 

154 query, 

155 key, 

156 w, 

157 gate, 

158 grad_output, 

159 grad_value, 

160 chunk_size, 

161 ) 

162 ) 

163 query, key, w, gate, grad_output, grad_value = ( 

164 tensor.contiguous() 

165 for tensor in (query, key, w, gate, grad_output, grad_value) 

166 ) 

167 grad_state_ext = torch.empty( 

168 batch, 

169 heads, 

170 key_dim, 

171 value_dim, 

172 dtype=torch.float32, 

173 device=query.device, 

174 ) 

175 kda_state_grad_ext_kernel[ 

176 (triton.cdiv(value_dim, block_size), batch * heads) 

177 ]( 

178 query, 

179 key, 

180 w, 

181 gate, 

182 grad_output, 

183 grad_value, 

184 grad_state_ext, 

185 float(scale), 

186 sequence_length, 

187 H=heads, 

188 H_TOTAL=heads, 

189 HEAD_OFFSET=0, 

190 K=key_dim, 

191 V=value_dim, 

192 BT=chunk_size, 

193 BV=block_size, 

194 num_warps=2, 

195 ) 

196 return grad_state_ext 

197 

198 

199@torch.compiler.disable 

200def _launch_kda_paired_state_summary( 

201 key: torch.Tensor, 

202 w: torch.Tensor, 

203 u: torch.Tensor, 

204 gate: torch.Tensor, 

205 *, 

206 chunk_size: int = 64, 

207 output_transition: bool = True, 

208) -> tuple[torch.Tensor, Optional[torch.Tensor]]: 

209 """Build Ascend FLA-paired ``S_ext`` and an explicit FP32 transfer ``M``.""" 

210 from ._triton.kimi_delta_attention.state_summary import ( # pylint: disable=import-outside-toplevel 

211 kda_split_state_summary_kernel, 

212 kda_state_transition_kernel, 

213 ) 

214 

215 batch, sequence_length, heads, key_dim, value_dim = ( 

216 _validate_prepared_summary_inputs(key, w, u, gate, chunk_size) 

217 ) 

218 key, w, u, gate = ( 

219 tensor.contiguous() for tensor in (key, w, u, gate) 

220 ) 

221 state_ext = torch.empty( 

222 batch, 

223 heads, 

224 key_dim, 

225 value_dim, 

226 dtype=torch.float32, 

227 device=key.device, 

228 ) 

229 transition = None 

230 if output_transition: 

231 transition = torch.empty( 

232 batch, 

233 heads, 

234 key_dim, 

235 key_dim, 

236 dtype=torch.float32, 

237 device=key.device, 

238 ) 

239 kda_split_state_summary_kernel[(1, batch * heads)]( 

240 key, 

241 w, 

242 u, 

243 gate, 

244 state_ext, 

245 transition if transition is not None else state_ext, 

246 sequence_length, 

247 H=heads, 

248 H_TOTAL=heads, 

249 HEAD_OFFSET=0, 

250 K=key_dim, 

251 V=value_dim, 

252 BT=chunk_size, 

253 BV=128, 

254 OUTPUT_MODE=1, 

255 num_warps=2, 

256 ) 

257 if output_transition: 

258 kda_state_transition_kernel[(triton.cdiv(key_dim, 64), batch * heads)]( 

259 key, 

260 w, 

261 gate, 

262 transition, 

263 sequence_length, 

264 H=heads, 

265 K=key_dim, 

266 BT=chunk_size, 

267 BV=64, 

268 num_warps=4, 

269 num_stages=2, 

270 ) 

271 return state_ext, transition 

272 

273 

274@torch.compiler.disable 

275def kda_state_summary_forward_from_prepared( 

276 key: torch.Tensor, 

277 w: torch.Tensor, 

278 u: torch.Tensor, 

279 gate: torch.Tensor, 

280 *, 

281 chunk_size: int = 64, 

282 output_transition: bool = True, 

283) -> tuple[torch.Tensor, Optional[torch.Tensor]]: 

284 """Build the local affine summary for state context parallelism. 

285 

286 ``S_ext`` follows Ascend FLA h's FP32 feedback. ``M`` composes explicit 

287 chunk transfers with FP32 feedback, following FLA CP. 

288 ``output_transition=False`` returns only S and skips M allocation/computation. 

289 It is valid for a zero-entry P2P endpoint whose backward never applies M. 

290 

291 Args: 

292 key: Prepared keys, shaped [batch, tokens, heads, key_dim]. 

293 w: Prepared feedback weights with the same shape as key. 

294 u: Prepared values, shaped [batch, tokens, heads, value_dim]. 

295 gate: Per-chunk cumulative log2 gates matching key. 

296 chunk_size: Number of tokens per local chunk; the kernel requires 64. 

297 output_transition: Whether the caller needs the local transition M. 

298 

299 Returns: 

300 The FP32 state S and the FP32 transition M, or None when M is omitted. 

301 """ 

302 return _launch_kda_state_summary( 

303 key, 

304 w, 

305 u, 

306 gate, 

307 chunk_size=chunk_size, 

308 output_transition=output_transition, 

309 ) 

310 

311 

312@torch.compiler.disable 

313def kda_state_gradient_summary_from_prepared( 

314 query: torch.Tensor, 

315 key: torch.Tensor, 

316 w: torch.Tensor, 

317 gate: torch.Tensor, 

318 grad_output: torch.Tensor, 

319 grad_value: torch.Tensor, 

320 scale: float, 

321 *, 

322 chunk_size: int = 64, 

323) -> torch.Tensor: 

324 """Return the local-loss contribution to the incoming KDA state gradient. 

325 

326 Match Ascend FLA ``dhu`` at zero terminal state: scale ``grad_output`` 

327 before its dot-input cast and preserve the two K64 slabs and subtraction 

328 grouping. This summary omits the full ``dh``/``dv`` outputs. All inputs 

329 are read-only; the affine application to a nonzero state is a separate step. 

330 

331 Args: 

332 query: Prepared gated queries [B,T,H,K]. 

333 key: Matching prepared gated keys. 

334 w: Prepared feedback weights matching key. 

335 gate: Per-chunk cumulative log2 gates matching key. 

336 grad_output: Original token-output gradients [B,T,H,V]. 

337 grad_value: Local value gradients before recurrent feedback. 

338 scale: Attention scaling factor. 

339 chunk_size: Tokens per local chunk; the kernel requires 64. 

340 

341 Returns: 

342 FP32 local contribution G, shaped [B,H,K,V]. 

343 """ 

344 return _launch_kda_state_gradient_summary( 

345 query, 

346 key, 

347 w, 

348 gate, 

349 grad_output, 

350 grad_value, 

351 scale, 

352 chunk_size=chunk_size, 

353 block_size=128, 

354 ) 

355 

356 

357def apply_kda_state_summary( 

358 state_ext: torch.Tensor, 

359 transition: Optional[torch.Tensor], 

360 initial_state: Optional[torch.Tensor], 

361) -> torch.Tensor: 

362 """Apply a prepared KDA state summary in FP32. 

363 

364 Args: 

365 state_ext: Local zero-entry state S. 

366 transition: Local transfer M; may be omitted for a zero entry. 

367 initial_state: Incoming FP32 state, or None for a zero entry. 

368 

369 Returns: 

370 The outgoing FP32 state. 

371 """ 

372 if initial_state is None: 

373 return state_ext 

374 # Model autocast must not silently quantize a communicated FP32 state. 

375 with torch.autocast(device_type=state_ext.device.type, enabled=False): 

376 return torch.matmul(transition.float(), initial_state.float()) + state_ext 

377 

378 

379def apply_kda_state_gradient_summary( 

380 grad_state_ext: torch.Tensor, 

381 transition: torch.Tensor, 

382 grad_final_state: Optional[torch.Tensor], 

383) -> torch.Tensor: 

384 """Apply the adjoint summary for the reverse P2P state wavefront.""" 

385 if grad_final_state is None: 

386 return grad_state_ext 

387 with torch.autocast(device_type=grad_state_ext.device.type, enabled=False): 

388 return ( 

389 torch.matmul(transition.float().transpose(-2, -1), grad_final_state.float()) 

390 + grad_state_ext 

391 )