Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / kimi_delta_attention.py: 14%

224 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Dense Triton-Ascend KDA execution helpers for Torch training.""" 

16from __future__ import annotations 

17 

18 

19__all__ = ["fused_chunk_kda", "fused_chunk_kda_p2p"] 

20 

21from typing import Any, Optional 

22 

23import torch 

24import torch.distributed as dist 

25 

26from hyper_parallel.components.functional.linear_attention_packed import PackedSequenceInfo, resolve_packed_sequence 

27 

28from .kimi_delta_attention_fla_adapter import get_fla_kda_staged_ops, run_fla_chunk_kda 

29from .kimi_delta_attention_state_summary import ( 

30 apply_kda_state_gradient_summary, 

31 apply_kda_state_summary, 

32 kda_state_gradient_summary_from_prepared, 

33 kda_state_summary_forward_from_prepared, 

34) 

35 

36 

37def _validate_local_shapes( 

38 query: torch.Tensor, 

39 key: torch.Tensor, 

40 value: torch.Tensor, 

41 gate: torch.Tensor, 

42 beta: torch.Tensor, 

43) -> tuple[int, int, int, int]: 

44 """Validate local KDA tensor shapes and return dimensions used by later checks.""" 

45 if (query.ndim, key.ndim, value.ndim, gate.ndim) != (4, 4, 4, 4): 

46 raise ValueError("Fused KDA expects rank-4 query/key/value/gate tensors.") 

47 if beta.ndim != 3: 

48 raise ValueError("Fused KDA expects a rank-3 beta tensor.") 

49 if query.shape != key.shape: 

50 raise ValueError("Fused KDA query and key must have identical shapes.") 

51 batch, sequence_length, num_query_heads, key_dim = query.shape 

52 num_value_heads, value_dim = value.shape[2:] 

53 if value.shape[:2] != (batch, sequence_length): 

54 raise ValueError("Fused KDA value must match query batch and sequence dimensions.") 

55 if gate.shape != (batch, sequence_length, num_value_heads, key_dim): 

56 raise ValueError("Fused KDA gate has an incompatible shape.") 

57 if beta.shape != (batch, sequence_length, num_value_heads): 

58 raise ValueError("Fused KDA beta has an incompatible shape.") 

59 if num_value_heads % num_query_heads: 

60 raise ValueError("Fused KDA value heads must be divisible by query heads.") 

61 return sequence_length, key_dim, value_dim, num_value_heads 

62 

63 

64def _validate_local_inputs( 

65 query: torch.Tensor, 

66 key: torch.Tensor, 

67 value: torch.Tensor, 

68 gate: torch.Tensor, 

69 beta: torch.Tensor, 

70 a_log: torch.Tensor, 

71 dt_bias: torch.Tensor, 

72 *, 

73 chunk_size: int, 

74 lower_bound: float, 

75) -> None: 

76 """Validate the fixed-shape local KDA backend before compiling kernels.""" 

77 _, key_dim, value_dim, num_value_heads = _validate_local_shapes( 

78 query, key, value, gate, beta 

79 ) 

80 if not ( 

81 query.dtype == key.dtype == value.dtype == gate.dtype == beta.dtype 

82 == torch.bfloat16 

83 ): 

84 raise TypeError("Fused KDA requires q/k/v/gate/beta tensors in bfloat16.") 

85 if a_log.dtype != torch.float32 or dt_bias.dtype != torch.float32: 

86 raise TypeError("Fused KDA requires a_log and dt_bias tensors in float32.") 

87 if key_dim != 128 or value_dim != 128 or chunk_size != 64: 

88 raise NotImplementedError( 

89 "Fused KDA currently requires key_dim=value_dim=128 and chunk_size=64." 

90 ) 

91 if a_log.numel() != num_value_heads: 

92 raise ValueError("Fused KDA a_log must contain one value per value head.") 

93 if dt_bias.numel() != num_value_heads * key_dim: 

94 raise ValueError("Fused KDA dt_bias must contain one value per gate channel.") 

95 if not -5.0 <= lower_bound < 0: 

96 raise ValueError("Fused KDA lower_bound must lie in [-5, 0).") 

97 if not query.is_npu: 

98 raise RuntimeError("The fused KDA backend requires Ascend NPU tensors.") 

99 if any( 

100 tensor.device != query.device 

101 for tensor in (key, value, gate, beta, a_log, dt_bias) 

102 ): 

103 raise ValueError("Fused KDA inputs must reside on the same NPU device.") 

104 get_fla_kda_staged_ops() 

105 

106 

107def _validate_p2p_inputs( 

108 query: torch.Tensor, 

109 key: torch.Tensor, 

110 value: torch.Tensor, 

111 gate: torch.Tensor, 

112 beta: torch.Tensor, 

113 a_log: torch.Tensor, 

114 dt_bias: torch.Tensor, 

115 *, 

116 chunk_size: int, 

117 lower_bound: float, 

118 cp_rank: int, 

119 cp_size: int, 

120) -> None: 

121 """Validate the fixed-shape KDA P2P backend before communication starts.""" 

122 _validate_local_inputs( 

123 query, 

124 key, 

125 value, 

126 gate, 

127 beta, 

128 a_log, 

129 dt_bias, 

130 chunk_size=chunk_size, 

131 lower_bound=lower_bound, 

132 ) 

133 if cp_size <= 0 or not 0 <= cp_rank < cp_size: 

134 raise ValueError(f"Invalid KDA P2P rank {cp_rank} for cp_size {cp_size}.") 

135 

136 

137def fused_chunk_kda( 

138 query: torch.Tensor, 

139 key: torch.Tensor, 

140 value: torch.Tensor, 

141 gate: torch.Tensor, 

142 beta: torch.Tensor, 

143 *, 

144 a_log: torch.Tensor, 

145 dt_bias: torch.Tensor, 

146 scale: Optional[float] = None, 

147 lower_bound: float = -5.0, 

148 chunk_size: int = 64, 

149 safe_gate: bool = True, 

150 cu_seqlens: Optional[torch.Tensor] = None, 

151) -> torch.Tensor: 

152 """Run the local Triton-Ascend KDA backend without CP communication. 

153 

154 Args: 

155 query: Local query tensor [B,T,H,K]. 

156 key: Matching local key tensor. 

157 value: Local value tensor [B,T,HV,V]. 

158 gate: Per-token gate logits [B,T,HV,K]. 

159 beta: Per-token beta logits [B,T,HV]. 

160 a_log: Decay parameter with one value per value head. 

161 dt_bias: Learned bias with one value per gate channel. 

162 scale: Optional attention scaling factor. 

163 lower_bound: Lower bound for the activated gate. 

164 chunk_size: Tokens per local recurrence chunk. 

165 safe_gate: Use the staged lower-bounded gate implementation. 

166 """ 

167 _validate_local_inputs( 

168 query, 

169 key, 

170 value, 

171 gate, 

172 beta, 

173 a_log, 

174 dt_bias, 

175 chunk_size=chunk_size, 

176 lower_bound=lower_bound, 

177 ) 

178 output, _ = run_fla_chunk_kda( 

179 query, 

180 key, 

181 value, 

182 gate, 

183 beta, 

184 a_log=a_log, 

185 dt_bias=dt_bias.reshape(-1), 

186 scale=scale, 

187 lower_bound=lower_bound, 

188 chunk_size=chunk_size, 

189 safe_gate=safe_gate, 

190 cu_seqlens=cu_seqlens, 

191 ) 

192 return output 

193 

194 

195def _prepare_packed_kda_summary(kg, w, u, gate, sequence_info, state_cp, send_next, chunk_size): 

196 """Skip summaries with no consumer when the invocation supports absent tensors.""" 

197 if state_cp is None and not send_next: 

198 return None, None 

199 state_shape = (kg.shape[0], u.shape[2], kg.shape[-1], u.shape[-1]) 

200 transition = None 

201 if getattr(state_cp, "needs_state_summary", True): 

202 tail = sequence_info.tail 

203 state, transition = kda_state_summary_forward_from_prepared( 

204 kg[:, tail], w[:, tail], u[:, tail], gate[:, tail], chunk_size=chunk_size, 

205 output_transition=(sequence_info.transfer_active 

206 and getattr(state_cp, "needs_transition_summary", True)), 

207 ) 

208 elif getattr(state_cp, "allows_absent_summaries", False): 

209 return None, None 

210 else: 

211 state = kg.new_zeros(state_shape, dtype=torch.float32) 

212 if state_cp is not None and transition is None: 

213 # Shape placeholders preserve existing collective validation; no M kernel runs. 

214 transition = kg.new_zeros((*state_shape[:-1], kg.shape[-1]), dtype=torch.float32) 

215 return state, transition 

216 

217 

218class _KDAStateP2PFunction(torch.autograd.Function): 

219 """Run shared local KDA and autograd with the selected state exchange.""" 

220 

221 @staticmethod 

222 def forward( # pylint: disable=arguments-differ,too-many-arguments,too-many-locals 

223 ctx: Any, 

224 query: torch.Tensor, 

225 key: torch.Tensor, 

226 value: torch.Tensor, 

227 gate_raw: torch.Tensor, 

228 beta_raw: torch.Tensor, 

229 a_log: torch.Tensor, 

230 dt_bias: torch.Tensor, 

231 scale: float, 

232 lower_bound: float, 

233 chunk_size: int, 

234 safe_gate: bool, 

235 cp_group: Any, 

236 prev_rank: int, 

237 next_rank: int, 

238 cp_rank: int, 

239 cp_size: int, 

240 state_cp: Any, 

241 sequence_info: Optional[PackedSequenceInfo], 

242 ) -> torch.Tensor: 

243 """Prepare local KDA, propagate its state, and produce token outputs. 

244 

245 Args: 

246 ctx: Autograd context holding tensors and communication metadata. 

247 query: Local query tensor [B,T,H,K]. 

248 key: Matching local key tensor. 

249 value: Local value tensor [B,T,HV,V]. 

250 gate_raw: Unactivated per-token gate logits. 

251 beta_raw: Unactivated per-token beta logits. 

252 a_log: Decay parameter with one value per value head. 

253 dt_bias: Learned bias with one value per gate channel. 

254 scale: Optional attention scaling factor. 

255 lower_bound: Lower bound for the activated gate. 

256 chunk_size: Tokens per local recurrence chunk. 

257 safe_gate: Use the staged lower-bounded gate implementation. 

258 cp_group: Precreated chronological CP process group. 

259 prev_rank: Global rank of the preceding sequence partition. 

260 next_rank: Global rank of the following sequence partition. 

261 cp_rank: Chronological index within the CP group. 

262 cp_size: Number of state partitions. 

263 state_cp: Executor returning an invocation cache; None selects P2P. 

264 """ 

265 ops = get_fla_kda_staged_ops() 

266 bind_sequence = getattr(state_cp, "bind_sequence", None) 

267 if sequence_info is not None and bind_sequence is not None: 

268 state_cp = bind_sequence( 

269 sequence_info, state_shape=(query.shape[0], value.shape[2], query.shape[-1], value.shape[-1])) 

270 plan = None if sequence_info is None else sequence_info.materialize(query.device, chunk_size) 

271 varlen = {} if plan is None else plan.kwargs() 

272 recurrent_varlen = dict(varlen) 

273 if plan is not None: 

274 recurrent_varlen["chunk_offsets"] = plan.chunk_offsets 

275 

276 # FLA consumes base-2 log gates: exp(x) = exp2(x / ln(2)). 

277 # Use the same conversion when recomputing gates in backward. 

278 rcp_ln2 = 1.4426950408889634 

279 

280 query, key, value, gate_raw, beta_raw = ( 

281 tensor.contiguous() 

282 for tensor in (query, key, value, gate_raw, beta_raw) 

283 ) 

284 query, query_rstd = ops.l2norm_fwd(query) 

285 key, key_rstd = ops.l2norm_fwd(key) 

286 beta = ops.fused_beta_sigmoid(beta_raw) 

287 gate = ops.kda_gate_chunk_cumsum( 

288 g=gate_raw, A_log=a_log, dt_bias=dt_bias, scale=rcp_ln2, chunk_size=chunk_size, 

289 lower_bound=lower_bound, 

290 **varlen, 

291 ) 

292 

293 state_shape = (query.shape[0], value.shape[2], query.shape[-1], value.shape[-1]) 

294 receive_previous = cp_rank > 0 and (sequence_info is None or sequence_info.continues_previous) 

295 send_next = cp_rank < cp_size - 1 and (sequence_info is None or sequence_info.continues_next) 

296 initial_state = None 

297 recv_handle = None 

298 if state_cp is None and receive_previous: 

299 initial_state = torch.empty(state_shape, device=query.device, dtype=torch.float32) 

300 recv_handle = dist.irecv(initial_state, src=prev_rank, group=cp_group) 

301 

302 w, u, _, kg, attention_qk, attention_kk = ops.chunk_kda_fwd_intra( 

303 q=query, k=key, v=value, gk=gate, beta=beta, scale=scale, chunk_size=chunk_size, safe_gate=safe_gate, 

304 disable_recompute=True, 

305 **varlen, 

306 ) 

307 # The gated query is unused here and is recomputed in backward. 

308 del _ 

309 state_ext = None 

310 transition = None 

311 if sequence_info is not None: 

312 state_ext, transition = _prepare_packed_kda_summary( 

313 kg, w, u, gate, sequence_info, state_cp, send_next, chunk_size, 

314 ) 

315 else: 

316 if state_cp is None and cp_rank == 0 and cp_size > 1: 

317 # Zero entry uses S directly, and rank 0 has no reverse-state recipient. 

318 state_ext, transition = kda_state_summary_forward_from_prepared( 

319 kg, w, u, gate, chunk_size=chunk_size, output_transition=False, 

320 ) 

321 elif state_cp is not None or cp_rank < cp_size - 1: 

322 state_ext, transition = kda_state_summary_forward_from_prepared(kg, w, u, gate, chunk_size=chunk_size) 

323 # A reset makes the outgoing summary independent of the incoming state. 

324 # Wait before apply only when it actually reads that state; local h still waits below. 

325 if recv_handle is not None and (sequence_info is None or sequence_info.transfer_active): 

326 recv_handle.wait() 

327 recv_handle = None 

328 

329 send_handle = None 

330 send_state = None 

331 state_cache = (transition,) if transition is not None else () 

332 if state_cp is not None: 

333 initial_state, state_cache = state_cp.forward(state_ext, transition) 

334 transition = None 

335 elif send_next: 

336 summary_initial = initial_state if transition is not None else None 

337 final_state = apply_kda_state_summary(state_ext, transition, summary_initial) 

338 send_state = final_state.contiguous() 

339 send_handle = dist.isend(send_state, dst=next_rank, group=cp_group) 

340 

341 if recv_handle is not None: 

342 recv_handle.wait() 

343 kernel_initial_state = initial_state if sequence_info is None else sequence_info.first_state(initial_state) 

344 states, value_new, _ = ops.chunk_gated_delta_rule_fwd_h( 

345 k=kg, w=w, u=u, gk=gate, initial_state=kernel_initial_state, 

346 output_final_state=False, chunk_size=chunk_size, 

347 **recurrent_varlen, 

348 ) 

349 # Local h was their last forward consumer on this stream. 

350 del w, u, kg, _ 

351 output = ops.chunk_gla_fwd_o_gk( 

352 q=query, v=value_new, g=gate, A=attention_qk, h=states, scale=scale, chunk_size=chunk_size, 

353 **varlen, 

354 ) 

355 if send_handle is not None: 

356 send_handle.wait() 

357 

358 saved_initial_state = initial_state 

359 if saved_initial_state is None: 

360 saved_initial_state = query.new_empty(0, dtype=torch.float32) 

361 packed_tensors = () if plan is None else ( 

362 plan.cu_seqlens, plan.chunk_indices, plan.chunk_offsets) 

363 ctx.save_for_backward( 

364 query, query_rstd, key, key_rstd, value, gate_raw, beta_raw, a_log, dt_bias, attention_qk, 

365 attention_kk, saved_initial_state, *state_cache, *packed_tensors, 

366 ) 

367 ctx.scale = scale 

368 ctx.lower_bound = lower_bound 

369 ctx.chunk_size = chunk_size 

370 ctx.safe_gate = safe_gate 

371 ctx.cp_group = cp_group 

372 ctx.prev_rank = prev_rank 

373 ctx.next_rank = next_rank 

374 ctx.cp_rank = cp_rank 

375 ctx.cp_size = cp_size 

376 ctx.state_cp = state_cp 

377 ctx.state_cache_count = len(state_cache) 

378 ctx.receive_previous = receive_previous 

379 ctx.send_next = send_next 

380 ctx.sequence_info = sequence_info 

381 return output.to(value.dtype) 

382 

383 @staticmethod 

384 def backward( # pylint: disable=arguments-differ,too-many-locals 

385 ctx: Any, 

386 grad_output: torch.Tensor, 

387 ) -> tuple[Any, ...]: 

388 """Reverse the state wavefront, then finish the fused local backward. 

389 

390 Args: 

391 ctx: Autograd context holding tensors and communication metadata. 

392 grad_output: Upstream gradient for the local token output. 

393 """ 

394 ops = get_fla_kda_staged_ops() 

395 

396 # FLA consumes base-2 log gates: exp(x) = exp2(x / ln(2)). 

397 # Use the same conversion when recomputing gates in backward. 

398 rcp_ln2 = 1.4426950408889634 

399 

400 ( 

401 query, query_rstd, key, key_rstd, value, gate_raw, beta_raw, a_log, dt_bias, 

402 attention_qk, attention_kk, saved_initial_state, *saved_tail, 

403 ) = ctx.saved_tensors 

404 state_cache = tuple(saved_tail[:ctx.state_cache_count]) 

405 packed_tensors = saved_tail[ctx.state_cache_count:] 

406 sequence_info = ctx.sequence_info 

407 cu_seqlens = chunk_indices = chunk_offsets = None 

408 if sequence_info is not None: 

409 cu_seqlens, chunk_indices, chunk_offsets = packed_tensors 

410 varlen = {} if sequence_info is None else {"cu_seqlens": cu_seqlens, "chunk_indices": chunk_indices} 

411 recurrent_varlen = dict(varlen) 

412 if sequence_info is not None: 

413 recurrent_varlen["chunk_offsets"] = chunk_offsets 

414 initial_state = saved_initial_state if saved_initial_state.numel() else None 

415 kernel_initial_state = initial_state if sequence_info is None else sequence_info.first_state(initial_state) 

416 grad_output = grad_output.contiguous() 

417 

418 grad_final_state = None 

419 recv_handle = None 

420 if ctx.state_cp is None and ctx.send_next: 

421 grad_final_state = torch.empty( 

422 (query.shape[0], value.shape[2], query.shape[-1], value.shape[-1]), device=query.device, 

423 dtype=torch.float32, 

424 ) 

425 recv_handle = dist.irecv(grad_final_state, src=ctx.next_rank, group=ctx.cp_group) 

426 

427 beta = ops.fused_beta_sigmoid(beta_raw) 

428 gate = ops.kda_gate_chunk_cumsum( 

429 g=gate_raw, A_log=a_log, dt_bias=dt_bias, scale=rcp_ln2, chunk_size=ctx.chunk_size, 

430 lower_bound=ctx.lower_bound, 

431 **varlen, 

432 ) 

433 w, u, query_gated, key_gated = ops.recompute_w_u_fwd( 

434 q=query, k=key, v=value, beta=beta, A=attention_kk, gk=gate, 

435 **varlen, 

436 ) 

437 states, value_new, _ = ops.chunk_gated_delta_rule_fwd_h( 

438 k=key_gated, w=w, u=u, gk=gate, initial_state=kernel_initial_state, output_final_state=False, 

439 chunk_size=ctx.chunk_size, 

440 **recurrent_varlen, 

441 ) 

442 grad_attention_qk, grad_value_local = ops.chunk_kda_bwd_dav( 

443 q=query, k=key, v=value_new, do=grad_output, A=attention_qk, scale=ctx.scale, 

444 chunk_size=ctx.chunk_size, 

445 **varlen, 

446 ) 

447 grad_state_ext = None 

448 if ctx.state_cp is not None or ctx.receive_previous: 

449 if sequence_info is not None and not sequence_info.continues_previous: 

450 if not getattr(ctx.state_cp, "allows_absent_summaries", False): 

451 grad_state_ext = query.new_zeros( 

452 (query.shape[0], value.shape[2], query.shape[-1], value.shape[-1]), dtype=torch.float32) 

453 else: 

454 head = slice(None) if sequence_info is None else sequence_info.head 

455 grad_state_ext = kda_state_gradient_summary_from_prepared( 

456 query_gated[:, head], key_gated[:, head], w[:, head], gate[:, head], 

457 grad_output[:, head], grad_value_local[:, head], ctx.scale, chunk_size=ctx.chunk_size, 

458 ) 

459 if recv_handle is not None and (sequence_info is None or sequence_info.transfer_active): 

460 recv_handle.wait() 

461 recv_handle = None 

462 send_handle = None 

463 send_state_gradient = None 

464 if ctx.state_cp is not None: 

465 grad_final_state = ctx.state_cp.backward(grad_state_ext, state_cache) 

466 elif ctx.receive_previous: 

467 transition = state_cache[0] if state_cache else None 

468 summary_final_gradient = grad_final_state if transition is not None else None 

469 grad_initial_state = apply_kda_state_gradient_summary(grad_state_ext, transition, summary_final_gradient) 

470 send_state_gradient = grad_initial_state.contiguous() 

471 send_handle = dist.isend(send_state_gradient, dst=ctx.prev_rank, group=ctx.cp_group) 

472 

473 if recv_handle is not None: 

474 recv_handle.wait() 

475 kernel_grad_final = grad_final_state if sequence_info is None else sequence_info.last_state(grad_final_state) 

476 grad_states, _, grad_value = ops.chunk_gated_delta_rule_bwd_dhu( 

477 q=query_gated, k=key_gated, w=w, gk=gate, h0=kernel_initial_state, dht=kernel_grad_final, do=grad_output, 

478 dv=grad_value_local, scale=ctx.scale, chunk_size=ctx.chunk_size, 

479 **recurrent_varlen, 

480 ) 

481 # These recomputed tensors have no consumers after dhu. Dropping local 

482 # references lets the allocator reuse their storage on this stream. 

483 del query_gated, key_gated, w, u, grad_value_local, _ 

484 ( 

485 grad_query, grad_key, grad_value, grad_beta, grad_gate, grad_attention_kk, 

486 ) = ops.chunk_kda_bwd_wy_dqkg_fused( 

487 q=query, k=key, v=value, v_new=value_new, g=gate, beta=beta, A=attention_kk, h=states, do=grad_output, 

488 dh=grad_states, dv=grad_value, scale=ctx.scale, chunk_size=ctx.chunk_size, 

489 **varlen, 

490 ) 

491 # Release recomputed states before intra backward allocates its outputs. 

492 del states, grad_states, value_new 

493 grad_query, grad_key, grad_beta, grad_gate = ops.chunk_kda_bwd_intra( 

494 q=query, k=key, g=gate, beta=beta, dAqk=grad_attention_qk, dAkk=grad_attention_kk, dq=grad_query, 

495 dk=grad_key, db=grad_beta, dg=grad_gate, chunk_size=ctx.chunk_size, safe_gate=ctx.safe_gate, 

496 **varlen, 

497 ) 

498 

499 num_query_heads = query.shape[2] 

500 num_value_heads = value.shape[2] 

501 if num_value_heads > num_query_heads: 

502 groups = num_value_heads // num_query_heads 

503 grad_query = grad_query.view(*grad_query.shape[:2], num_query_heads, groups, grad_query.shape[-1]).sum( 

504 dim=3, 

505 ) 

506 grad_key = grad_key.view(*grad_key.shape[:2], num_query_heads, groups, grad_key.shape[-1]).sum(dim=3) 

507 

508 grad_gate = ops.chunk_local_cumsum(grad_gate, chunk_size=ctx.chunk_size, reverse=True, **varlen) 

509 grad_gate, grad_a_log, grad_dt_bias = ops.kda_gate_bwd( 

510 g=gate_raw, A_log=a_log, dt_bias=dt_bias, dyg=grad_gate, lower_bound=ctx.lower_bound, 

511 ) 

512 grad_beta = ops.fused_beta_sigmoid_bwd(beta_raw, grad_beta) 

513 grad_query = ops.l2norm_bwd(query, query_rstd, grad_query) 

514 grad_key = ops.l2norm_bwd(key, key_rstd, grad_key) 

515 if send_handle is not None: 

516 send_handle.wait() 

517 

518 return ( 

519 grad_query, grad_key, grad_value, grad_gate, grad_beta, grad_a_log, grad_dt_bias, None, None, None, 

520 None, None, None, None, None, None, None, None, 

521 ) 

522 

523 

524@torch.compiler.disable 

525def fused_chunk_kda_p2p( 

526 query: torch.Tensor, 

527 key: torch.Tensor, 

528 value: torch.Tensor, 

529 gate: torch.Tensor, 

530 beta: torch.Tensor, 

531 *, 

532 a_log: torch.Tensor, 

533 dt_bias: torch.Tensor, 

534 cp_group: Any, 

535 prev_rank: int, 

536 next_rank: int, 

537 cp_rank: int, 

538 cp_size: int, 

539 scale: Optional[float] = None, 

540 lower_bound: float = -5.0, 

541 chunk_size: int = 64, 

542 safe_gate: bool = True, 

543 state_cp: Any = None, 

544 sequence_info: Optional[PackedSequenceInfo] = None, 

545) -> torch.Tensor: 

546 """Run one sequence-sharded KDA segment with a selected state CP executor. 

547 

548 The historical P2P entry name is retained for existing callers; passing 

549 state_cp selects gathered or recursive state exchange with the same local ops. 

550 

551 Args: 

552 query: Query input [B,T,H,K]. 

553 key: Matching key input. 

554 value: Value input [B,T,HV,V]. 

555 gate: Per-token gate logits. 

556 beta: Per-token beta logits. 

557 a_log: FP32 decay parameter per value head. 

558 dt_bias: FP32 gate-channel bias. 

559 cp_group: Existing chronological state process group. 

560 prev_rank: Global predecessor rank for P2P. 

561 next_rank: Global successor rank for P2P. 

562 cp_rank: Chronological index within the state group. 

563 cp_size: Number of state partitions. 

564 scale: Optional attention scale. 

565 lower_bound: Lower bound for the activated gate. 

566 chunk_size: Tokens per local chunk. 

567 safe_gate: Enable the staged safe-gate path. 

568 state_cp: State executor returning an invocation cache; None selects existing P2P. 

569 

570 Returns: 

571 Output tokens in the local sequence layout. 

572 """ 

573 _validate_p2p_inputs( 

574 query, 

575 key, 

576 value, 

577 gate, 

578 beta, 

579 a_log, 

580 dt_bias, 

581 chunk_size=chunk_size, 

582 lower_bound=lower_bound, 

583 cp_rank=cp_rank, 

584 cp_size=cp_size, 

585 ) 

586 if sequence_info is not None: 

587 if query.shape[0] != 1: 

588 raise ValueError("Packed state CP requires batch dimension one.") 

589 resolve_packed_sequence({"packed_sequence_info": sequence_info}, query.shape[1], 

590 cp_rank=cp_rank, cp_size=cp_size) 

591 effective_scale = query.shape[-1] ** -0.5 if scale is None else float(scale) 

592 return _KDAStateP2PFunction.apply( 

593 query, 

594 key, 

595 value, 

596 gate, 

597 beta, 

598 a_log, 

599 dt_bias.reshape(-1), 

600 effective_scale, 

601 float(lower_bound), 

602 chunk_size, 

603 safe_gate, 

604 cp_group, 

605 prev_rank, 

606 next_rank, 

607 cp_rank, 

608 cp_size, 

609 state_cp, 

610 sequence_info, 

611 )