Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / functional / kimi_delta_attention.py: 14%
224 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Dense Triton-Ascend KDA execution helpers for Torch training."""
16from __future__ import annotations
19__all__ = ["fused_chunk_kda", "fused_chunk_kda_p2p"]
21from typing import Any, Optional
23import torch
24import torch.distributed as dist
26from hyper_parallel.components.functional.linear_attention_packed import PackedSequenceInfo, resolve_packed_sequence
28from .kimi_delta_attention_fla_adapter import get_fla_kda_staged_ops, run_fla_chunk_kda
29from .kimi_delta_attention_state_summary import (
30 apply_kda_state_gradient_summary,
31 apply_kda_state_summary,
32 kda_state_gradient_summary_from_prepared,
33 kda_state_summary_forward_from_prepared,
34)
37def _validate_local_shapes(
38 query: torch.Tensor,
39 key: torch.Tensor,
40 value: torch.Tensor,
41 gate: torch.Tensor,
42 beta: torch.Tensor,
43) -> tuple[int, int, int, int]:
44 """Validate local KDA tensor shapes and return dimensions used by later checks."""
45 if (query.ndim, key.ndim, value.ndim, gate.ndim) != (4, 4, 4, 4):
46 raise ValueError("Fused KDA expects rank-4 query/key/value/gate tensors.")
47 if beta.ndim != 3:
48 raise ValueError("Fused KDA expects a rank-3 beta tensor.")
49 if query.shape != key.shape:
50 raise ValueError("Fused KDA query and key must have identical shapes.")
51 batch, sequence_length, num_query_heads, key_dim = query.shape
52 num_value_heads, value_dim = value.shape[2:]
53 if value.shape[:2] != (batch, sequence_length):
54 raise ValueError("Fused KDA value must match query batch and sequence dimensions.")
55 if gate.shape != (batch, sequence_length, num_value_heads, key_dim):
56 raise ValueError("Fused KDA gate has an incompatible shape.")
57 if beta.shape != (batch, sequence_length, num_value_heads):
58 raise ValueError("Fused KDA beta has an incompatible shape.")
59 if num_value_heads % num_query_heads:
60 raise ValueError("Fused KDA value heads must be divisible by query heads.")
61 return sequence_length, key_dim, value_dim, num_value_heads
64def _validate_local_inputs(
65 query: torch.Tensor,
66 key: torch.Tensor,
67 value: torch.Tensor,
68 gate: torch.Tensor,
69 beta: torch.Tensor,
70 a_log: torch.Tensor,
71 dt_bias: torch.Tensor,
72 *,
73 chunk_size: int,
74 lower_bound: float,
75) -> None:
76 """Validate the fixed-shape local KDA backend before compiling kernels."""
77 _, key_dim, value_dim, num_value_heads = _validate_local_shapes(
78 query, key, value, gate, beta
79 )
80 if not (
81 query.dtype == key.dtype == value.dtype == gate.dtype == beta.dtype
82 == torch.bfloat16
83 ):
84 raise TypeError("Fused KDA requires q/k/v/gate/beta tensors in bfloat16.")
85 if a_log.dtype != torch.float32 or dt_bias.dtype != torch.float32:
86 raise TypeError("Fused KDA requires a_log and dt_bias tensors in float32.")
87 if key_dim != 128 or value_dim != 128 or chunk_size != 64:
88 raise NotImplementedError(
89 "Fused KDA currently requires key_dim=value_dim=128 and chunk_size=64."
90 )
91 if a_log.numel() != num_value_heads:
92 raise ValueError("Fused KDA a_log must contain one value per value head.")
93 if dt_bias.numel() != num_value_heads * key_dim:
94 raise ValueError("Fused KDA dt_bias must contain one value per gate channel.")
95 if not -5.0 <= lower_bound < 0:
96 raise ValueError("Fused KDA lower_bound must lie in [-5, 0).")
97 if not query.is_npu:
98 raise RuntimeError("The fused KDA backend requires Ascend NPU tensors.")
99 if any(
100 tensor.device != query.device
101 for tensor in (key, value, gate, beta, a_log, dt_bias)
102 ):
103 raise ValueError("Fused KDA inputs must reside on the same NPU device.")
104 get_fla_kda_staged_ops()
107def _validate_p2p_inputs(
108 query: torch.Tensor,
109 key: torch.Tensor,
110 value: torch.Tensor,
111 gate: torch.Tensor,
112 beta: torch.Tensor,
113 a_log: torch.Tensor,
114 dt_bias: torch.Tensor,
115 *,
116 chunk_size: int,
117 lower_bound: float,
118 cp_rank: int,
119 cp_size: int,
120) -> None:
121 """Validate the fixed-shape KDA P2P backend before communication starts."""
122 _validate_local_inputs(
123 query,
124 key,
125 value,
126 gate,
127 beta,
128 a_log,
129 dt_bias,
130 chunk_size=chunk_size,
131 lower_bound=lower_bound,
132 )
133 if cp_size <= 0 or not 0 <= cp_rank < cp_size:
134 raise ValueError(f"Invalid KDA P2P rank {cp_rank} for cp_size {cp_size}.")
137def fused_chunk_kda(
138 query: torch.Tensor,
139 key: torch.Tensor,
140 value: torch.Tensor,
141 gate: torch.Tensor,
142 beta: torch.Tensor,
143 *,
144 a_log: torch.Tensor,
145 dt_bias: torch.Tensor,
146 scale: Optional[float] = None,
147 lower_bound: float = -5.0,
148 chunk_size: int = 64,
149 safe_gate: bool = True,
150 cu_seqlens: Optional[torch.Tensor] = None,
151) -> torch.Tensor:
152 """Run the local Triton-Ascend KDA backend without CP communication.
154 Args:
155 query: Local query tensor [B,T,H,K].
156 key: Matching local key tensor.
157 value: Local value tensor [B,T,HV,V].
158 gate: Per-token gate logits [B,T,HV,K].
159 beta: Per-token beta logits [B,T,HV].
160 a_log: Decay parameter with one value per value head.
161 dt_bias: Learned bias with one value per gate channel.
162 scale: Optional attention scaling factor.
163 lower_bound: Lower bound for the activated gate.
164 chunk_size: Tokens per local recurrence chunk.
165 safe_gate: Use the staged lower-bounded gate implementation.
166 """
167 _validate_local_inputs(
168 query,
169 key,
170 value,
171 gate,
172 beta,
173 a_log,
174 dt_bias,
175 chunk_size=chunk_size,
176 lower_bound=lower_bound,
177 )
178 output, _ = run_fla_chunk_kda(
179 query,
180 key,
181 value,
182 gate,
183 beta,
184 a_log=a_log,
185 dt_bias=dt_bias.reshape(-1),
186 scale=scale,
187 lower_bound=lower_bound,
188 chunk_size=chunk_size,
189 safe_gate=safe_gate,
190 cu_seqlens=cu_seqlens,
191 )
192 return output
195def _prepare_packed_kda_summary(kg, w, u, gate, sequence_info, state_cp, send_next, chunk_size):
196 """Skip summaries with no consumer when the invocation supports absent tensors."""
197 if state_cp is None and not send_next:
198 return None, None
199 state_shape = (kg.shape[0], u.shape[2], kg.shape[-1], u.shape[-1])
200 transition = None
201 if getattr(state_cp, "needs_state_summary", True):
202 tail = sequence_info.tail
203 state, transition = kda_state_summary_forward_from_prepared(
204 kg[:, tail], w[:, tail], u[:, tail], gate[:, tail], chunk_size=chunk_size,
205 output_transition=(sequence_info.transfer_active
206 and getattr(state_cp, "needs_transition_summary", True)),
207 )
208 elif getattr(state_cp, "allows_absent_summaries", False):
209 return None, None
210 else:
211 state = kg.new_zeros(state_shape, dtype=torch.float32)
212 if state_cp is not None and transition is None:
213 # Shape placeholders preserve existing collective validation; no M kernel runs.
214 transition = kg.new_zeros((*state_shape[:-1], kg.shape[-1]), dtype=torch.float32)
215 return state, transition
218class _KDAStateP2PFunction(torch.autograd.Function):
219 """Run shared local KDA and autograd with the selected state exchange."""
221 @staticmethod
222 def forward( # pylint: disable=arguments-differ,too-many-arguments,too-many-locals
223 ctx: Any,
224 query: torch.Tensor,
225 key: torch.Tensor,
226 value: torch.Tensor,
227 gate_raw: torch.Tensor,
228 beta_raw: torch.Tensor,
229 a_log: torch.Tensor,
230 dt_bias: torch.Tensor,
231 scale: float,
232 lower_bound: float,
233 chunk_size: int,
234 safe_gate: bool,
235 cp_group: Any,
236 prev_rank: int,
237 next_rank: int,
238 cp_rank: int,
239 cp_size: int,
240 state_cp: Any,
241 sequence_info: Optional[PackedSequenceInfo],
242 ) -> torch.Tensor:
243 """Prepare local KDA, propagate its state, and produce token outputs.
245 Args:
246 ctx: Autograd context holding tensors and communication metadata.
247 query: Local query tensor [B,T,H,K].
248 key: Matching local key tensor.
249 value: Local value tensor [B,T,HV,V].
250 gate_raw: Unactivated per-token gate logits.
251 beta_raw: Unactivated per-token beta logits.
252 a_log: Decay parameter with one value per value head.
253 dt_bias: Learned bias with one value per gate channel.
254 scale: Optional attention scaling factor.
255 lower_bound: Lower bound for the activated gate.
256 chunk_size: Tokens per local recurrence chunk.
257 safe_gate: Use the staged lower-bounded gate implementation.
258 cp_group: Precreated chronological CP process group.
259 prev_rank: Global rank of the preceding sequence partition.
260 next_rank: Global rank of the following sequence partition.
261 cp_rank: Chronological index within the CP group.
262 cp_size: Number of state partitions.
263 state_cp: Executor returning an invocation cache; None selects P2P.
264 """
265 ops = get_fla_kda_staged_ops()
266 bind_sequence = getattr(state_cp, "bind_sequence", None)
267 if sequence_info is not None and bind_sequence is not None:
268 state_cp = bind_sequence(
269 sequence_info, state_shape=(query.shape[0], value.shape[2], query.shape[-1], value.shape[-1]))
270 plan = None if sequence_info is None else sequence_info.materialize(query.device, chunk_size)
271 varlen = {} if plan is None else plan.kwargs()
272 recurrent_varlen = dict(varlen)
273 if plan is not None:
274 recurrent_varlen["chunk_offsets"] = plan.chunk_offsets
276 # FLA consumes base-2 log gates: exp(x) = exp2(x / ln(2)).
277 # Use the same conversion when recomputing gates in backward.
278 rcp_ln2 = 1.4426950408889634
280 query, key, value, gate_raw, beta_raw = (
281 tensor.contiguous()
282 for tensor in (query, key, value, gate_raw, beta_raw)
283 )
284 query, query_rstd = ops.l2norm_fwd(query)
285 key, key_rstd = ops.l2norm_fwd(key)
286 beta = ops.fused_beta_sigmoid(beta_raw)
287 gate = ops.kda_gate_chunk_cumsum(
288 g=gate_raw, A_log=a_log, dt_bias=dt_bias, scale=rcp_ln2, chunk_size=chunk_size,
289 lower_bound=lower_bound,
290 **varlen,
291 )
293 state_shape = (query.shape[0], value.shape[2], query.shape[-1], value.shape[-1])
294 receive_previous = cp_rank > 0 and (sequence_info is None or sequence_info.continues_previous)
295 send_next = cp_rank < cp_size - 1 and (sequence_info is None or sequence_info.continues_next)
296 initial_state = None
297 recv_handle = None
298 if state_cp is None and receive_previous:
299 initial_state = torch.empty(state_shape, device=query.device, dtype=torch.float32)
300 recv_handle = dist.irecv(initial_state, src=prev_rank, group=cp_group)
302 w, u, _, kg, attention_qk, attention_kk = ops.chunk_kda_fwd_intra(
303 q=query, k=key, v=value, gk=gate, beta=beta, scale=scale, chunk_size=chunk_size, safe_gate=safe_gate,
304 disable_recompute=True,
305 **varlen,
306 )
307 # The gated query is unused here and is recomputed in backward.
308 del _
309 state_ext = None
310 transition = None
311 if sequence_info is not None:
312 state_ext, transition = _prepare_packed_kda_summary(
313 kg, w, u, gate, sequence_info, state_cp, send_next, chunk_size,
314 )
315 else:
316 if state_cp is None and cp_rank == 0 and cp_size > 1:
317 # Zero entry uses S directly, and rank 0 has no reverse-state recipient.
318 state_ext, transition = kda_state_summary_forward_from_prepared(
319 kg, w, u, gate, chunk_size=chunk_size, output_transition=False,
320 )
321 elif state_cp is not None or cp_rank < cp_size - 1:
322 state_ext, transition = kda_state_summary_forward_from_prepared(kg, w, u, gate, chunk_size=chunk_size)
323 # A reset makes the outgoing summary independent of the incoming state.
324 # Wait before apply only when it actually reads that state; local h still waits below.
325 if recv_handle is not None and (sequence_info is None or sequence_info.transfer_active):
326 recv_handle.wait()
327 recv_handle = None
329 send_handle = None
330 send_state = None
331 state_cache = (transition,) if transition is not None else ()
332 if state_cp is not None:
333 initial_state, state_cache = state_cp.forward(state_ext, transition)
334 transition = None
335 elif send_next:
336 summary_initial = initial_state if transition is not None else None
337 final_state = apply_kda_state_summary(state_ext, transition, summary_initial)
338 send_state = final_state.contiguous()
339 send_handle = dist.isend(send_state, dst=next_rank, group=cp_group)
341 if recv_handle is not None:
342 recv_handle.wait()
343 kernel_initial_state = initial_state if sequence_info is None else sequence_info.first_state(initial_state)
344 states, value_new, _ = ops.chunk_gated_delta_rule_fwd_h(
345 k=kg, w=w, u=u, gk=gate, initial_state=kernel_initial_state,
346 output_final_state=False, chunk_size=chunk_size,
347 **recurrent_varlen,
348 )
349 # Local h was their last forward consumer on this stream.
350 del w, u, kg, _
351 output = ops.chunk_gla_fwd_o_gk(
352 q=query, v=value_new, g=gate, A=attention_qk, h=states, scale=scale, chunk_size=chunk_size,
353 **varlen,
354 )
355 if send_handle is not None:
356 send_handle.wait()
358 saved_initial_state = initial_state
359 if saved_initial_state is None:
360 saved_initial_state = query.new_empty(0, dtype=torch.float32)
361 packed_tensors = () if plan is None else (
362 plan.cu_seqlens, plan.chunk_indices, plan.chunk_offsets)
363 ctx.save_for_backward(
364 query, query_rstd, key, key_rstd, value, gate_raw, beta_raw, a_log, dt_bias, attention_qk,
365 attention_kk, saved_initial_state, *state_cache, *packed_tensors,
366 )
367 ctx.scale = scale
368 ctx.lower_bound = lower_bound
369 ctx.chunk_size = chunk_size
370 ctx.safe_gate = safe_gate
371 ctx.cp_group = cp_group
372 ctx.prev_rank = prev_rank
373 ctx.next_rank = next_rank
374 ctx.cp_rank = cp_rank
375 ctx.cp_size = cp_size
376 ctx.state_cp = state_cp
377 ctx.state_cache_count = len(state_cache)
378 ctx.receive_previous = receive_previous
379 ctx.send_next = send_next
380 ctx.sequence_info = sequence_info
381 return output.to(value.dtype)
383 @staticmethod
384 def backward( # pylint: disable=arguments-differ,too-many-locals
385 ctx: Any,
386 grad_output: torch.Tensor,
387 ) -> tuple[Any, ...]:
388 """Reverse the state wavefront, then finish the fused local backward.
390 Args:
391 ctx: Autograd context holding tensors and communication metadata.
392 grad_output: Upstream gradient for the local token output.
393 """
394 ops = get_fla_kda_staged_ops()
396 # FLA consumes base-2 log gates: exp(x) = exp2(x / ln(2)).
397 # Use the same conversion when recomputing gates in backward.
398 rcp_ln2 = 1.4426950408889634
400 (
401 query, query_rstd, key, key_rstd, value, gate_raw, beta_raw, a_log, dt_bias,
402 attention_qk, attention_kk, saved_initial_state, *saved_tail,
403 ) = ctx.saved_tensors
404 state_cache = tuple(saved_tail[:ctx.state_cache_count])
405 packed_tensors = saved_tail[ctx.state_cache_count:]
406 sequence_info = ctx.sequence_info
407 cu_seqlens = chunk_indices = chunk_offsets = None
408 if sequence_info is not None:
409 cu_seqlens, chunk_indices, chunk_offsets = packed_tensors
410 varlen = {} if sequence_info is None else {"cu_seqlens": cu_seqlens, "chunk_indices": chunk_indices}
411 recurrent_varlen = dict(varlen)
412 if sequence_info is not None:
413 recurrent_varlen["chunk_offsets"] = chunk_offsets
414 initial_state = saved_initial_state if saved_initial_state.numel() else None
415 kernel_initial_state = initial_state if sequence_info is None else sequence_info.first_state(initial_state)
416 grad_output = grad_output.contiguous()
418 grad_final_state = None
419 recv_handle = None
420 if ctx.state_cp is None and ctx.send_next:
421 grad_final_state = torch.empty(
422 (query.shape[0], value.shape[2], query.shape[-1], value.shape[-1]), device=query.device,
423 dtype=torch.float32,
424 )
425 recv_handle = dist.irecv(grad_final_state, src=ctx.next_rank, group=ctx.cp_group)
427 beta = ops.fused_beta_sigmoid(beta_raw)
428 gate = ops.kda_gate_chunk_cumsum(
429 g=gate_raw, A_log=a_log, dt_bias=dt_bias, scale=rcp_ln2, chunk_size=ctx.chunk_size,
430 lower_bound=ctx.lower_bound,
431 **varlen,
432 )
433 w, u, query_gated, key_gated = ops.recompute_w_u_fwd(
434 q=query, k=key, v=value, beta=beta, A=attention_kk, gk=gate,
435 **varlen,
436 )
437 states, value_new, _ = ops.chunk_gated_delta_rule_fwd_h(
438 k=key_gated, w=w, u=u, gk=gate, initial_state=kernel_initial_state, output_final_state=False,
439 chunk_size=ctx.chunk_size,
440 **recurrent_varlen,
441 )
442 grad_attention_qk, grad_value_local = ops.chunk_kda_bwd_dav(
443 q=query, k=key, v=value_new, do=grad_output, A=attention_qk, scale=ctx.scale,
444 chunk_size=ctx.chunk_size,
445 **varlen,
446 )
447 grad_state_ext = None
448 if ctx.state_cp is not None or ctx.receive_previous:
449 if sequence_info is not None and not sequence_info.continues_previous:
450 if not getattr(ctx.state_cp, "allows_absent_summaries", False):
451 grad_state_ext = query.new_zeros(
452 (query.shape[0], value.shape[2], query.shape[-1], value.shape[-1]), dtype=torch.float32)
453 else:
454 head = slice(None) if sequence_info is None else sequence_info.head
455 grad_state_ext = kda_state_gradient_summary_from_prepared(
456 query_gated[:, head], key_gated[:, head], w[:, head], gate[:, head],
457 grad_output[:, head], grad_value_local[:, head], ctx.scale, chunk_size=ctx.chunk_size,
458 )
459 if recv_handle is not None and (sequence_info is None or sequence_info.transfer_active):
460 recv_handle.wait()
461 recv_handle = None
462 send_handle = None
463 send_state_gradient = None
464 if ctx.state_cp is not None:
465 grad_final_state = ctx.state_cp.backward(grad_state_ext, state_cache)
466 elif ctx.receive_previous:
467 transition = state_cache[0] if state_cache else None
468 summary_final_gradient = grad_final_state if transition is not None else None
469 grad_initial_state = apply_kda_state_gradient_summary(grad_state_ext, transition, summary_final_gradient)
470 send_state_gradient = grad_initial_state.contiguous()
471 send_handle = dist.isend(send_state_gradient, dst=ctx.prev_rank, group=ctx.cp_group)
473 if recv_handle is not None:
474 recv_handle.wait()
475 kernel_grad_final = grad_final_state if sequence_info is None else sequence_info.last_state(grad_final_state)
476 grad_states, _, grad_value = ops.chunk_gated_delta_rule_bwd_dhu(
477 q=query_gated, k=key_gated, w=w, gk=gate, h0=kernel_initial_state, dht=kernel_grad_final, do=grad_output,
478 dv=grad_value_local, scale=ctx.scale, chunk_size=ctx.chunk_size,
479 **recurrent_varlen,
480 )
481 # These recomputed tensors have no consumers after dhu. Dropping local
482 # references lets the allocator reuse their storage on this stream.
483 del query_gated, key_gated, w, u, grad_value_local, _
484 (
485 grad_query, grad_key, grad_value, grad_beta, grad_gate, grad_attention_kk,
486 ) = ops.chunk_kda_bwd_wy_dqkg_fused(
487 q=query, k=key, v=value, v_new=value_new, g=gate, beta=beta, A=attention_kk, h=states, do=grad_output,
488 dh=grad_states, dv=grad_value, scale=ctx.scale, chunk_size=ctx.chunk_size,
489 **varlen,
490 )
491 # Release recomputed states before intra backward allocates its outputs.
492 del states, grad_states, value_new
493 grad_query, grad_key, grad_beta, grad_gate = ops.chunk_kda_bwd_intra(
494 q=query, k=key, g=gate, beta=beta, dAqk=grad_attention_qk, dAkk=grad_attention_kk, dq=grad_query,
495 dk=grad_key, db=grad_beta, dg=grad_gate, chunk_size=ctx.chunk_size, safe_gate=ctx.safe_gate,
496 **varlen,
497 )
499 num_query_heads = query.shape[2]
500 num_value_heads = value.shape[2]
501 if num_value_heads > num_query_heads:
502 groups = num_value_heads // num_query_heads
503 grad_query = grad_query.view(*grad_query.shape[:2], num_query_heads, groups, grad_query.shape[-1]).sum(
504 dim=3,
505 )
506 grad_key = grad_key.view(*grad_key.shape[:2], num_query_heads, groups, grad_key.shape[-1]).sum(dim=3)
508 grad_gate = ops.chunk_local_cumsum(grad_gate, chunk_size=ctx.chunk_size, reverse=True, **varlen)
509 grad_gate, grad_a_log, grad_dt_bias = ops.kda_gate_bwd(
510 g=gate_raw, A_log=a_log, dt_bias=dt_bias, dyg=grad_gate, lower_bound=ctx.lower_bound,
511 )
512 grad_beta = ops.fused_beta_sigmoid_bwd(beta_raw, grad_beta)
513 grad_query = ops.l2norm_bwd(query, query_rstd, grad_query)
514 grad_key = ops.l2norm_bwd(key, key_rstd, grad_key)
515 if send_handle is not None:
516 send_handle.wait()
518 return (
519 grad_query, grad_key, grad_value, grad_gate, grad_beta, grad_a_log, grad_dt_bias, None, None, None,
520 None, None, None, None, None, None, None, None,
521 )
524@torch.compiler.disable
525def fused_chunk_kda_p2p(
526 query: torch.Tensor,
527 key: torch.Tensor,
528 value: torch.Tensor,
529 gate: torch.Tensor,
530 beta: torch.Tensor,
531 *,
532 a_log: torch.Tensor,
533 dt_bias: torch.Tensor,
534 cp_group: Any,
535 prev_rank: int,
536 next_rank: int,
537 cp_rank: int,
538 cp_size: int,
539 scale: Optional[float] = None,
540 lower_bound: float = -5.0,
541 chunk_size: int = 64,
542 safe_gate: bool = True,
543 state_cp: Any = None,
544 sequence_info: Optional[PackedSequenceInfo] = None,
545) -> torch.Tensor:
546 """Run one sequence-sharded KDA segment with a selected state CP executor.
548 The historical P2P entry name is retained for existing callers; passing
549 state_cp selects gathered or recursive state exchange with the same local ops.
551 Args:
552 query: Query input [B,T,H,K].
553 key: Matching key input.
554 value: Value input [B,T,HV,V].
555 gate: Per-token gate logits.
556 beta: Per-token beta logits.
557 a_log: FP32 decay parameter per value head.
558 dt_bias: FP32 gate-channel bias.
559 cp_group: Existing chronological state process group.
560 prev_rank: Global predecessor rank for P2P.
561 next_rank: Global successor rank for P2P.
562 cp_rank: Chronological index within the state group.
563 cp_size: Number of state partitions.
564 scale: Optional attention scale.
565 lower_bound: Lower bound for the activated gate.
566 chunk_size: Tokens per local chunk.
567 safe_gate: Enable the staged safe-gate path.
568 state_cp: State executor returning an invocation cache; None selects existing P2P.
570 Returns:
571 Output tokens in the local sequence layout.
572 """
573 _validate_p2p_inputs(
574 query,
575 key,
576 value,
577 gate,
578 beta,
579 a_log,
580 dt_bias,
581 chunk_size=chunk_size,
582 lower_bound=lower_bound,
583 cp_rank=cp_rank,
584 cp_size=cp_size,
585 )
586 if sequence_info is not None:
587 if query.shape[0] != 1:
588 raise ValueError("Packed state CP requires batch dimension one.")
589 resolve_packed_sequence({"packed_sequence_info": sequence_info}, query.shape[1],
590 cp_rank=cp_rank, cp_size=cp_size)
591 effective_scale = query.shape[-1] ** -0.5 if scale is None else float(scale)
592 return _KDAStateP2PFunction.apply(
593 query,
594 key,
595 value,
596 gate,
597 beta,
598 a_log,
599 dt_bias.reshape(-1),
600 effective_scale,
601 float(lower_bound),
602 chunk_size,
603 safe_gate,
604 cp_group,
605 prev_rank,
606 next_rank,
607 cp_rank,
608 cp_size,
609 state_cp,
610 sequence_info,
611 )