Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / kimi_delta_attention.py: 82%
372 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Pure PyTorch Kimi Delta Attention operators and training layer.
17The implementations in this module intentionally favor readable equations over
18performance. They are the numerical oracle shared by the standalone layer and
19context-parallel wrappers.
20"""
21# pylint: disable=invalid-name
22# This module mirrors the existing Torch model building blocks in this package.
23from __future__ import annotations
25__all__ = [
26 "KimiDeltaAttention",
27 "KimiRMSNormGated",
28 "chunk_kda",
29 "is_triton_kda_available",
30 "torch_apply_kda_state_summary",
31 "torch_chunk_kda",
32 "torch_compose_kda_state_summaries",
33 "torch_kda_gate",
34 "torch_kda_state_summary",
35 "torch_recurrent_kda",
36]
38from collections.abc import Sequence
39from typing import Any, Optional
41import torch
42from torch import nn
43from torch.nn import functional as F
45from hyper_parallel.components.functional.kimi_delta_attention_ascendc import ascendc_chunk_kda
46from hyper_parallel.components.functional.linear_attention_packed import (
47 packed_causal_conv1d, packed_reference, resolve_packed_sequence,
48)
50_KDA_BACKENDS = frozenset({"eager", "triton", "ascendc"})
51_TRITON_KDA_HEAD_DIM = 128
52_TRITON_KDA_CHUNK_SIZE = 64
55def _has_triton_kda_shapes(
56 query: torch.Tensor,
57 value: torch.Tensor,
58 gate: torch.Tensor,
59 beta: torch.Tensor,
60 a_log: torch.Tensor,
61 dt_bias: torch.Tensor,
62 chunk_size: int,
63) -> bool:
64 """Check the fixed dense shapes accepted by the Triton KDA path."""
65 batch_size, sequence_length, num_query_heads, key_dim = query.shape
66 num_value_heads, value_dim = value.shape[2:]
67 return all((
68 value.shape[:2] == (batch_size, sequence_length),
69 gate.shape == (batch_size, sequence_length, num_value_heads, key_dim),
70 beta.shape == (batch_size, sequence_length, num_value_heads),
71 num_value_heads % num_query_heads == 0,
72 a_log.numel() == num_value_heads,
73 dt_bias.numel() == num_value_heads * key_dim,
74 key_dim == value_dim == _TRITON_KDA_HEAD_DIM,
75 chunk_size == _TRITON_KDA_CHUNK_SIZE,
76 ))
79def _is_triton_kda_input_supported(
80 query: torch.Tensor,
81 key: Optional[torch.Tensor],
82 value: Optional[torch.Tensor],
83 gate: Optional[torch.Tensor],
84 beta: Optional[torch.Tensor],
85 a_log: Optional[torch.Tensor],
86 dt_bias: Optional[torch.Tensor],
87 *,
88 chunk_size: int,
89 lower_bound: Optional[float],
90) -> bool:
91 """Check the fixed Kimi K3 contract validated by the fused backend."""
92 operands = (key, value, gate, beta, a_log, dt_bias)
93 if any(tensor is None for tensor in operands):
94 return False
95 if lower_bound is None:
96 return False
98 basic_contract = (
99 query.device.type == "npu",
100 all(tensor.dtype == torch.bfloat16 for tensor in (query, key, value, gate, beta)),
101 all(tensor.dtype == torch.float32 for tensor in (a_log, dt_bias)),
102 all(tensor.ndim == 4 for tensor in (query, key, value, gate)),
103 beta.ndim == 3,
104 query.shape == key.shape,
105 )
106 if not all(basic_contract):
107 return False
109 shape_supported = _has_triton_kda_shapes(
110 query, value, gate, beta, a_log, dt_bias, chunk_size,
111 )
112 lower_bound_supported = -5.0 <= lower_bound < 0
113 if not (shape_supported and lower_bound_supported):
114 return False
115 return all(tensor.device == query.device for tensor in operands)
118def is_triton_kda_available(
119 query: Optional[torch.Tensor] = None,
120 key: Optional[torch.Tensor] = None,
121 value: Optional[torch.Tensor] = None,
122 gate: Optional[torch.Tensor] = None,
123 beta: Optional[torch.Tensor] = None,
124 a_log: Optional[torch.Tensor] = None,
125 dt_bias: Optional[torch.Tensor] = None,
126 *,
127 chunk_size: int = _TRITON_KDA_CHUNK_SIZE,
128 lower_bound: Optional[float] = -5.0,
129) -> bool:
130 """Return whether the validated Triton-Ascend KDA backend is available."""
131 if chunk_size != _TRITON_KDA_CHUNK_SIZE:
132 return False
133 if query is not None and not _is_triton_kda_input_supported(
134 query,
135 key,
136 value,
137 gate,
138 beta,
139 a_log,
140 dt_bias,
141 chunk_size=chunk_size,
142 lower_bound=lower_bound,
143 ):
144 return False
145 from hyper_parallel.components.functional.kimi_delta_attention_fla_adapter import ( # pylint: disable=import-outside-toplevel
146 is_fla_triton_kda_available,
147 )
148 return is_fla_triton_kda_available()
151def _validate_kda_inputs(
152 query: torch.Tensor,
153 key: torch.Tensor,
154 value: torch.Tensor,
155 gate: torch.Tensor,
156 beta: torch.Tensor,
157) -> tuple[int, int, int, int, int, int]:
158 """Validate dense token-first KDA inputs and return their dimensions."""
159 if (query.dim(), key.dim(), value.dim(), gate.dim()) != (4, 4, 4, 4):
160 raise ValueError("query, key, value, and gate must be rank-4 tensors.")
161 if beta.dim() != 3:
162 raise ValueError("beta must be a rank-3 tensor.")
163 if query.shape != key.shape:
164 raise ValueError("query and key must have identical shapes.")
166 batch_size, sequence_length, num_k_heads, k_head_dim = query.shape
167 value_batch, value_sequence, num_v_heads, v_head_dim = value.shape
168 if (value_batch, value_sequence) != (batch_size, sequence_length):
169 raise ValueError("value batch and sequence dimensions must match query.")
170 if gate.shape != (batch_size, sequence_length, num_v_heads, k_head_dim):
171 raise ValueError(
172 "gate must have shape [B, S, num_v_heads, k_head_dim], got "
173 f"{tuple(gate.shape)}."
174 )
175 if beta.shape != (batch_size, sequence_length, num_v_heads):
176 raise ValueError(
177 "beta must have shape [B, S, num_v_heads], got "
178 f"{tuple(beta.shape)}."
179 )
180 if num_v_heads % num_k_heads != 0:
181 raise ValueError("num_v_heads must be divisible by num_k_heads.")
182 return (
183 batch_size,
184 sequence_length,
185 num_k_heads,
186 num_v_heads,
187 k_head_dim,
188 v_head_dim,
189 )
192def _reshape_gate_bias(
193 parameter: torch.Tensor,
194 num_v_heads: int,
195 k_head_dim: int,
196 name: str,
197) -> torch.Tensor:
198 """Return a gate parameter as ``[num_v_heads, k_head_dim]``."""
199 if parameter.numel() != num_v_heads * k_head_dim:
200 raise ValueError(
201 f"{name} must contain {num_v_heads * k_head_dim} elements, "
202 f"got {parameter.numel()}."
203 )
204 return parameter.reshape(num_v_heads, k_head_dim)
207def torch_kda_gate(
208 gate_logits: torch.Tensor,
209 a_log: torch.Tensor,
210 dt_bias: torch.Tensor,
211 *,
212 lower_bound: Optional[float] = None,
213) -> torch.Tensor:
214 """Activate KDA log-decay logits.
216 ``lower_bound`` selects the Kimi K3 parameterization
217 ``g_min * sigmoid(exp(a_log) * (logits + bias))``. With no lower bound,
218 this function implements the earlier Kimi Linear negative-Softplus gate.
219 """
220 if gate_logits.dim() != 4:
221 raise ValueError("gate_logits must have shape [B, S, H, K].")
222 num_v_heads, k_head_dim = gate_logits.shape[-2:]
223 if a_log.numel() != num_v_heads:
224 raise ValueError(
225 f"a_log must contain {num_v_heads} elements, got {a_log.numel()}."
226 )
227 bias = _reshape_gate_bias(dt_bias, num_v_heads, k_head_dim, "dt_bias")
228 scale = a_log.float().exp().reshape(1, 1, num_v_heads, 1)
229 preactivation = gate_logits.float() + bias.float().reshape(
230 1, 1, num_v_heads, k_head_dim
231 )
232 if lower_bound is None:
233 return -scale * F.softplus(preactivation) # pylint: disable=not-callable
234 if lower_bound >= 0:
235 raise ValueError(f"lower_bound must be negative, got {lower_bound}.")
236 return float(lower_bound) * torch.sigmoid(scale * preactivation)
239def _prepare_kda_inputs(
240 query: torch.Tensor,
241 key: torch.Tensor,
242 value: torch.Tensor,
243 gate: torch.Tensor,
244 beta: torch.Tensor,
245 *,
246 a_log: Optional[torch.Tensor],
247 dt_bias: Optional[torch.Tensor],
248 scale: Optional[float],
249 use_qk_l2norm_in_kernel: bool,
250 use_gate_in_kernel: bool,
251 use_beta_sigmoid_in_kernel: bool,
252 lower_bound: Optional[float],
253) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, float]:
254 """Apply the common KDA input transformations in FP32."""
255 _, _, num_k_heads, num_v_heads, k_head_dim, _ = _validate_kda_inputs(
256 query, key, value, gate, beta
257 )
258 if use_qk_l2norm_in_kernel:
259 query = query * torch.rsqrt(
260 (query.float() * query.float()).sum(dim=-1, keepdim=True) + 1e-6
261 ).to(query.dtype)
262 key = key * torch.rsqrt(
263 (key.float() * key.float()).sum(dim=-1, keepdim=True) + 1e-6
264 ).to(key.dtype)
265 if use_gate_in_kernel:
266 if a_log is None or dt_bias is None:
267 raise ValueError("a_log and dt_bias are required when gate activation is enabled.")
268 gate = torch_kda_gate(
269 gate,
270 a_log,
271 dt_bias,
272 lower_bound=lower_bound,
273 )
274 if use_beta_sigmoid_in_kernel:
275 beta = beta.float().sigmoid()
277 value_groups = num_v_heads // num_k_heads
278 query = query.float().repeat_interleave(value_groups, dim=2)
279 key = key.float().repeat_interleave(value_groups, dim=2)
280 value = value.float()
281 gate = gate.float()
282 beta = beta.float()
283 effective_scale = k_head_dim ** -0.5 if scale is None else scale
284 return query, key, value, gate, beta, float(effective_scale)
287def _initial_kda_state(
288 value: torch.Tensor,
289 k_head_dim: int,
290 initial_state: Optional[torch.Tensor],
291) -> torch.Tensor:
292 """Build or validate a KDA state in FP32."""
293 expected_shape = (value.shape[0], value.shape[2], k_head_dim, value.shape[3])
294 if initial_state is None:
295 return value.new_zeros(expected_shape, dtype=torch.float32)
296 if tuple(initial_state.shape) != expected_shape:
297 raise ValueError(
298 f"initial_state must have shape {expected_shape}, got "
299 f"{tuple(initial_state.shape)}."
300 )
301 return initial_state.float()
304def torch_recurrent_kda(
305 query: torch.Tensor,
306 key: torch.Tensor,
307 value: torch.Tensor,
308 gate: torch.Tensor,
309 beta: torch.Tensor,
310 *,
311 a_log: Optional[torch.Tensor] = None,
312 dt_bias: Optional[torch.Tensor] = None,
313 scale: Optional[float] = None,
314 initial_state: Optional[torch.Tensor] = None,
315 output_final_state: bool = False,
316 use_qk_l2norm_in_kernel: bool = False,
317 use_gate_in_kernel: bool = False,
318 use_beta_sigmoid_in_kernel: bool = False,
319 lower_bound: Optional[float] = None,
320) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
321 """Evaluate the exact token-recurrent KDA equations with PyTorch ops."""
322 output_dtype = value.dtype
323 query, key, value, gate, beta, scale = _prepare_kda_inputs(
324 query,
325 key,
326 value,
327 gate,
328 beta,
329 a_log=a_log,
330 dt_bias=dt_bias,
331 scale=scale,
332 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
333 use_gate_in_kernel=use_gate_in_kernel,
334 use_beta_sigmoid_in_kernel=use_beta_sigmoid_in_kernel,
335 lower_bound=lower_bound,
336 )
337 state = _initial_kda_state(value, key.shape[-1], initial_state)
338 outputs = []
339 for token_idx in range(query.shape[1]):
340 query_i = query[:, token_idx]
341 key_i = key[:, token_idx]
342 value_i = value[:, token_idx]
343 gate_i = gate[:, token_idx]
344 beta_i = beta[:, token_idx]
346 state = state * gate_i.exp().unsqueeze(-1)
347 state_value = torch.einsum("bhk,bhkv->bhv", key_i, state)
348 residual = value_i - state_value
349 state = state + torch.einsum(
350 "bhk,bhv->bhkv",
351 key_i * beta_i.unsqueeze(-1),
352 residual,
353 )
354 outputs.append(torch.einsum("bhk,bhkv->bhv", query_i * scale, state))
356 output = torch.stack(outputs, dim=1).to(output_dtype)
357 return output, state if output_final_state else None
360def _inverse_unit_lower_triangular(strict_lower: torch.Tensor) -> torch.Tensor:
361 """Invert ``I + strict_lower`` using differentiable forward substitution."""
362 chunk_size = strict_lower.shape[-1]
363 rows = []
364 for row_idx in range(chunk_size):
365 base = -strict_lower[..., row_idx, :]
366 if rows:
367 previous = torch.stack(rows, dim=-2)
368 row = torch.einsum(
369 "...i,...ij->...j",
370 base[..., :row_idx],
371 previous,
372 )
373 else:
374 row = torch.zeros_like(base)
375 diagonal = F.one_hot( # pylint: disable=not-callable
376 torch.tensor(row_idx, device=strict_lower.device),
377 num_classes=chunk_size,
378 ).to(strict_lower.dtype)
379 rows.append(row + diagonal)
380 return torch.stack(rows, dim=-2)
383def _prepare_kda_wy(
384 query: torch.Tensor,
385 key: torch.Tensor,
386 value: torch.Tensor,
387 gate: torch.Tensor,
388 beta: torch.Tensor,
389 chunk_size: int,
390) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
391 """Build the cumulative gate, causal mask, and chunkwise WY factors."""
392 gate = gate.cumsum(dim=-2)
393 causal = torch.ones(
394 chunk_size,
395 chunk_size,
396 dtype=torch.bool,
397 device=query.device,
398 ).tril()
399 strict_causal = causal.logical_xor(
400 torch.eye(chunk_size, dtype=torch.bool, device=query.device)
401 )
403 gate_pair = gate.unsqueeze(-2) - gate.unsqueeze(-3)
404 key_pair = torch.einsum(
405 "...ik,...jk,...ijk->...ij",
406 key,
407 key,
408 gate_pair.exp(),
409 )
410 strict_lower = (
411 key_pair * beta.unsqueeze(-1)
412 ).masked_fill(~strict_causal, 0)
413 inverse = _inverse_unit_lower_triangular(strict_lower)
414 transform = inverse * beta.unsqueeze(-2)
415 w = transform @ (key * gate.exp())
416 u = transform @ value
418 return gate, causal, w, u
421def _run_kda_chunks(
422 query: torch.Tensor,
423 key: torch.Tensor,
424 gate: torch.Tensor,
425 w: torch.Tensor,
426 u: torch.Tensor,
427 state: torch.Tensor,
428 causal: torch.Tensor,
429 scale: float,
430 num_chunks: int,
431) -> tuple[list[torch.Tensor], torch.Tensor]:
432 """Apply chunk WY factors to the recurrent state in sequence order."""
433 output_chunks = []
434 for chunk_idx in range(num_chunks):
435 query_i = query[:, :, chunk_idx]
436 key_i = key[:, :, chunk_idx]
437 gate_i = gate[:, :, chunk_idx]
438 w_i = w[:, :, chunk_idx]
439 u_i = u[:, :, chunk_idx]
441 relative_gate = gate_i.unsqueeze(-2) - gate_i.unsqueeze(-3)
442 qk = torch.einsum(
443 "bhik,bhjk,bhijk->bhij",
444 query_i,
445 key_i,
446 relative_gate.exp(),
447 ).masked_fill(~causal, 0)
448 value_new = u_i - w_i @ state
449 output_inter = (query_i * gate_i.exp()) @ state
450 output_chunks.append(output_inter * scale + qk @ value_new * scale)
452 last_gate = gate_i[:, :, -1]
453 key_decay = key_i * (last_gate.unsqueeze(-2) - gate_i).exp()
454 state = state * last_gate.exp().unsqueeze(-1)
455 state = state + key_decay.transpose(-1, -2) @ value_new
457 return output_chunks, state
460def torch_chunk_kda(
461 query: torch.Tensor,
462 key: torch.Tensor,
463 value: torch.Tensor,
464 gate: torch.Tensor,
465 beta: torch.Tensor,
466 *,
467 a_log: Optional[torch.Tensor] = None,
468 dt_bias: Optional[torch.Tensor] = None,
469 scale: Optional[float] = None,
470 initial_state: Optional[torch.Tensor] = None,
471 output_final_state: bool = False,
472 use_qk_l2norm_in_kernel: bool = False,
473 use_gate_in_kernel: bool = False,
474 use_beta_sigmoid_in_kernel: bool = False,
475 lower_bound: Optional[float] = None,
476 chunk_size: int = 64,
477 cu_seqlens: Optional[torch.Tensor] = None,
478) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
479 """Evaluate chunkwise KDA using the UT/WY representation.
481 This implementation is algebraically independent from
482 :func:`torch_recurrent_kda`; comparing the two catches errors in the
483 chunk transform that a second token loop would hide.
484 """
485 if cu_seqlens is not None:
486 return packed_reference(
487 torch_chunk_kda, (query, key, value, gate, beta), cu_seqlens,
488 initial_state=initial_state, output_final_state=output_final_state,
489 a_log=a_log, dt_bias=dt_bias, scale=scale,
490 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, use_gate_in_kernel=use_gate_in_kernel,
491 use_beta_sigmoid_in_kernel=use_beta_sigmoid_in_kernel, lower_bound=lower_bound, chunk_size=chunk_size,
492 )
493 if chunk_size <= 0:
494 raise ValueError(f"chunk_size must be positive, got {chunk_size}.")
495 output_dtype = value.dtype
496 sequence_length = query.shape[1]
497 query, key, value, gate, beta, scale = _prepare_kda_inputs(
498 query,
499 key,
500 value,
501 gate,
502 beta,
503 a_log=a_log,
504 dt_bias=dt_bias,
505 scale=scale,
506 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
507 use_gate_in_kernel=use_gate_in_kernel,
508 use_beta_sigmoid_in_kernel=use_beta_sigmoid_in_kernel,
509 lower_bound=lower_bound,
510 )
511 batch_size, _, num_heads, k_head_dim = query.shape
512 v_head_dim = value.shape[-1]
513 pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size
514 query = F.pad(query, (0, 0, 0, 0, 0, pad_size))
515 key = F.pad(key, (0, 0, 0, 0, 0, pad_size))
516 value = F.pad(value, (0, 0, 0, 0, 0, pad_size))
517 gate = F.pad(gate, (0, 0, 0, 0, 0, pad_size))
518 beta = F.pad(beta, (0, 0, 0, pad_size))
519 num_chunks = query.shape[1] // chunk_size
521 def chunk_view(tensor: torch.Tensor) -> torch.Tensor:
522 """Rearrange a token-first tensor to head-first chunks."""
523 shape = tensor.shape
524 return tensor.reshape(
525 shape[0], num_chunks, chunk_size, shape[2], *shape[3:]
526 ).permute(0, 3, 1, 2, *range(4, tensor.dim() + 1))
528 query, key, value, gate, beta = [
529 chunk_view(tensor) for tensor in (query, key, value, gate, beta)
530 ]
531 gate, causal, w, u = _prepare_kda_wy(query, key, value, gate, beta, chunk_size)
533 state = _initial_kda_state(
534 value.permute(0, 2, 3, 1, 4).reshape(
535 batch_size, num_chunks * chunk_size, num_heads, v_head_dim
536 ),
537 k_head_dim,
538 initial_state,
539 )
540 output_chunks, state = _run_kda_chunks(query, key, gate, w, u, state, causal, scale, num_chunks)
542 output = torch.stack(output_chunks, dim=2)
543 output = output.permute(0, 2, 3, 1, 4).reshape(
544 batch_size,
545 num_chunks * chunk_size,
546 num_heads,
547 v_head_dim,
548 )
549 output = output[:, :sequence_length].to(output_dtype)
550 return output, state if output_final_state else None
553def _normalize_kda_backend(backend: str) -> str:
554 """Normalize the selected backend and reject unsupported names."""
555 backend = backend.lower()
556 if backend not in _KDA_BACKENDS:
557 raise ValueError(
558 f"unsupported KDA backend {backend!r}; "
559 f"expected one of {sorted(_KDA_BACKENDS)}."
560 )
561 return backend
564def _validate_fused_state_contract(initial_state, output_final_state, qk_norm, gate, beta, backend) -> None:
565 """Keep raw-input dispatch separate from the stateful CP backend contract."""
566 if initial_state is not None or output_final_state:
567 raise NotImplementedError(
568 f"KDA backend={backend!r} currently supports zero initial state without "
569 "returning a final state; stateful execution is owned by the CP backend."
570 )
571 if not (qk_norm and gate and beta):
572 raise NotImplementedError(
573 f"KDA backend={backend!r} requires fused Q/K normalization, gate activation, "
574 "and beta sigmoid."
575 )
578def chunk_kda(
579 query: torch.Tensor,
580 key: torch.Tensor,
581 value: torch.Tensor,
582 gate: torch.Tensor,
583 beta: torch.Tensor,
584 *,
585 a_log: Optional[torch.Tensor] = None,
586 dt_bias: Optional[torch.Tensor] = None,
587 scale: Optional[float] = None,
588 initial_state: Optional[torch.Tensor] = None,
589 output_final_state: bool = False,
590 use_qk_l2norm_in_kernel: bool = False,
591 use_gate_in_kernel: bool = False,
592 use_beta_sigmoid_in_kernel: bool = False,
593 lower_bound: Optional[float] = None,
594 chunk_size: int = 64,
595 backend: str = "eager",
596 cu_seqlens: Optional[torch.Tensor | Sequence[int]] = None,
597) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
598 """Dispatch dense or packed KDA to the explicitly selected local backend."""
599 if query.ndim == 3:
600 output, final = chunk_kda(
601 *(tensor.unsqueeze(0) for tensor in (query, key, value, gate, beta)),
602 a_log=a_log, dt_bias=dt_bias, scale=scale, initial_state=initial_state,
603 output_final_state=output_final_state, use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
604 use_gate_in_kernel=use_gate_in_kernel, use_beta_sigmoid_in_kernel=use_beta_sigmoid_in_kernel,
605 lower_bound=lower_bound, chunk_size=chunk_size, backend=backend, cu_seqlens=cu_seqlens)
606 return output.squeeze(0), final
607 backend = _normalize_kda_backend(backend)
608 if backend == "eager":
609 return torch_chunk_kda(
610 query,
611 key,
612 value,
613 gate,
614 beta,
615 a_log=a_log,
616 dt_bias=dt_bias,
617 scale=scale,
618 initial_state=initial_state,
619 output_final_state=output_final_state,
620 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
621 use_gate_in_kernel=use_gate_in_kernel,
622 use_beta_sigmoid_in_kernel=use_beta_sigmoid_in_kernel,
623 lower_bound=lower_bound,
624 chunk_size=chunk_size,
625 cu_seqlens=cu_seqlens,
626 )
627 _validate_fused_state_contract(initial_state, output_final_state, use_qk_l2norm_in_kernel,
628 use_gate_in_kernel, use_beta_sigmoid_in_kernel, backend)
629 if backend == "ascendc":
630 output = ascendc_chunk_kda(
631 query, key, value, gate, beta,
632 a_log=a_log.float() if a_log is not None else None,
633 dt_bias=dt_bias.float() if dt_bias is not None else None,
634 scale=scale, lower_bound=lower_bound, chunk_size=chunk_size, cu_seqlens=cu_seqlens,
635 )
636 return output, None
637 triton_a_log = a_log.float() if a_log is not None else None
638 triton_dt_bias = dt_bias.float() if dt_bias is not None else None
639 if not _is_triton_kda_input_supported(
640 query,
641 key,
642 value,
643 gate,
644 beta,
645 triton_a_log,
646 triton_dt_bias,
647 chunk_size=chunk_size,
648 lower_bound=lower_bound,
649 ):
650 raise RuntimeError(
651 "KDA backend='triton' received an unsupported input contract; it requires "
652 "q/k/v/gate/beta=bf16, a_log/dt_bias=fp32, "
653 "head_k_dim=head_v_dim=128, chunk_size=64, and a supported safe gate."
654 )
656 from hyper_parallel.components.functional.kimi_delta_attention import ( # pylint: disable=import-outside-toplevel
657 fused_chunk_kda,
658 )
660 output = fused_chunk_kda(
661 query,
662 key,
663 value,
664 gate,
665 beta,
666 a_log=triton_a_log,
667 dt_bias=triton_dt_bias,
668 scale=scale,
669 lower_bound=lower_bound,
670 chunk_size=chunk_size,
671 safe_gate=True,
672 cu_seqlens=cu_seqlens,
673 )
674 return output, None
677def _chunk_kda_summary_inputs(
678 key: torch.Tensor,
679 value: torch.Tensor,
680 gate: torch.Tensor,
681 beta: torch.Tensor,
682 chunk_size: int,
683) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
684 """Pad and convert token-first KDA tensors into head-first chunks."""
685 sequence_length = key.shape[1]
686 pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size
687 key = F.pad(key, (0, 0, 0, 0, 0, pad_size))
688 value = F.pad(value, (0, 0, 0, 0, 0, pad_size))
689 gate = F.pad(gate, (0, 0, 0, 0, 0, pad_size))
690 beta = F.pad(beta, (0, 0, 0, pad_size))
691 num_chunks = key.shape[1] // chunk_size
693 def chunk_view(tensor: torch.Tensor) -> torch.Tensor:
694 """Rearrange one token-first tensor to head-first chunks."""
695 shape = tensor.shape
696 return tensor.reshape(
697 shape[0], num_chunks, chunk_size, shape[2], *shape[3:]
698 ).permute(0, 3, 1, 2, *range(4, tensor.dim() + 1))
700 key, value, gate, beta = (
701 chunk_view(tensor) for tensor in (key, value, gate, beta)
702 )
703 return key, value, gate, beta, num_chunks
706def torch_kda_state_summary(
707 query: torch.Tensor,
708 key: torch.Tensor,
709 value: torch.Tensor,
710 gate: torch.Tensor,
711 beta: torch.Tensor,
712 *,
713 a_log: Optional[torch.Tensor] = None,
714 dt_bias: Optional[torch.Tensor] = None,
715 scale: Optional[float] = None,
716 use_qk_l2norm_in_kernel: bool = False,
717 use_gate_in_kernel: bool = False,
718 use_beta_sigmoid_in_kernel: bool = False,
719 lower_bound: Optional[float] = None,
720 chunk_size: int = 64,
721) -> tuple[torch.Tensor, torch.Tensor]:
722 """Summarize a local KDA segment as ``H_out = M @ H_in + S``.
724 The implementation follows the same chunkwise WY equations as
725 :func:`torch_chunk_kda`, but computes only the state-independent affine
726 transition. ``S`` has shape ``[B, HV, K, V]`` and ``M`` has shape
727 ``[B, HV, K, K]``. Query is accepted to share KDA input preprocessing and
728 grouped-value validation; the state transition itself is independent of
729 query and output scale.
730 """
731 if chunk_size <= 0:
732 raise ValueError(f"chunk_size must be positive, got {chunk_size}.")
733 _, key, value, gate, beta, _ = _prepare_kda_inputs(
734 query,
735 key,
736 value,
737 gate,
738 beta,
739 a_log=a_log,
740 dt_bias=dt_bias,
741 scale=scale,
742 use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
743 use_gate_in_kernel=use_gate_in_kernel,
744 use_beta_sigmoid_in_kernel=use_beta_sigmoid_in_kernel,
745 lower_bound=lower_bound,
746 )
747 key, value, gate, beta, num_chunks = _chunk_kda_summary_inputs(
748 key,
749 value,
750 gate,
751 beta,
752 chunk_size,
753 )
754 gate = gate.cumsum(dim=-2)
755 strict_causal = torch.ones(
756 chunk_size,
757 chunk_size,
758 dtype=torch.bool,
759 device=key.device,
760 ).tril(diagonal=-1)
761 gate_pair = gate.unsqueeze(-2) - gate.unsqueeze(-3)
762 key_pair = torch.einsum(
763 "...ik,...jk,...ijk->...ij",
764 key,
765 key,
766 gate_pair.exp(),
767 )
768 strict_lower = (key_pair * beta.unsqueeze(-1)).masked_fill(
769 ~strict_causal,
770 0,
771 )
772 inverse = _inverse_unit_lower_triangular(strict_lower)
773 transform = inverse * beta.unsqueeze(-2)
774 w = transform @ (key * gate.exp())
775 u = transform @ value
777 batch_size, num_heads, _, _, k_head_dim = key.shape
778 v_head_dim = value.shape[-1]
779 eye = torch.eye(
780 k_head_dim,
781 dtype=torch.float32,
782 device=key.device,
783 ).reshape(1, 1, k_head_dim, k_head_dim)
784 state_ext = torch.zeros(
785 batch_size,
786 num_heads,
787 k_head_dim,
788 v_head_dim,
789 dtype=torch.float32,
790 device=key.device,
791 )
792 transition = eye.expand(batch_size, num_heads, -1, -1).clone()
794 for chunk_idx in range(num_chunks):
795 key_i = key[:, :, chunk_idx]
796 gate_i = gate[:, :, chunk_idx]
797 w_i = w[:, :, chunk_idx]
798 u_i = u[:, :, chunk_idx]
799 last_gate = gate_i[:, :, -1]
800 key_decay = key_i * (last_gate.unsqueeze(-2) - gate_i).exp()
801 transition_i = torch.diag_embed(last_gate.exp())
802 transition_i = transition_i - key_decay.transpose(-1, -2) @ w_i
803 state_ext_i = key_decay.transpose(-1, -2) @ u_i
804 state_ext = transition_i @ state_ext + state_ext_i
805 transition = transition_i @ transition
807 return state_ext.float(), transition.float()
810def torch_apply_kda_state_summary(
811 state_ext: torch.Tensor,
812 transition: torch.Tensor,
813 initial_state: Optional[torch.Tensor],
814) -> torch.Tensor:
815 """Apply a KDA affine state summary to an incoming state."""
816 if initial_state is None:
817 return state_ext.float()
818 return (transition @ initial_state.to(transition) + state_ext).float()
821def torch_compose_kda_state_summaries(
822 first_state_ext: torch.Tensor,
823 first_transition: torch.Tensor,
824 second_state_ext: torch.Tensor,
825 second_transition: torch.Tensor,
826) -> tuple[torch.Tensor, torch.Tensor]:
827 """Compose two chronological KDA summaries, first then second."""
828 state_ext = second_transition @ first_state_ext + second_state_ext
829 transition = second_transition @ first_transition
830 return state_ext, transition
833class KimiRMSNormGated(nn.Module):
834 """Per-head RMSNorm followed by the sigmoid output gate used by KDA."""
836 def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
837 """Initialize the per-head normalization weight and epsilon."""
838 super().__init__()
839 self.weight = nn.Parameter(torch.ones(hidden_size))
840 self.variance_epsilon = eps
842 def forward(self, hidden_states: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
843 """Normalize in FP32 and apply a sigmoid gate in the input dtype."""
844 input_dtype = hidden_states.dtype
845 normalized = hidden_states.float()
846 variance = normalized.square().mean(dim=-1, keepdim=True)
847 normalized = normalized * torch.rsqrt(variance + self.variance_epsilon)
848 normalized = normalized * self.weight.float()
849 return (normalized * gate.float().sigmoid()).to(input_dtype)
852def _make_short_convolution(channels: int, kernel_size: int) -> nn.Conv1d:
853 """Create the depthwise causal convolution used by the training layer."""
854 convolution = nn.Conv1d(
855 channels,
856 channels,
857 kernel_size=kernel_size,
858 groups=channels,
859 padding=kernel_size - 1,
860 bias=False,
861 )
862 convolution.activation = "silu"
863 return convolution
866class KimiDeltaAttention(nn.Module):
867 """Training-only Kimi Delta Attention layer implemented with eager Torch.
869 The layer owns the projection, ShortConv, gate, normalization, and output
870 parameters used by Kimi K3. Accelerator selection is intentionally outside
871 this model module; context-parallel policies may replace ``forward`` while
872 preserving these parameters and state-dict names.
873 """
875 def __init__(
876 self,
877 hidden_size: int,
878 num_heads: int = 32,
879 num_v_heads: Optional[int] = None,
880 head_k_dim: int = 128,
881 head_v_dim: int = 128,
882 conv_kernel_size: int = 4,
883 chunk_size: int = 64,
884 gate_lower_bound: float = -5.0,
885 use_full_rank_gate: bool = True,
886 rms_norm_eps: float = 1e-6,
887 layer_idx: int = 0,
888 ) -> None:
889 """Initialize a configurable, training-only KDA attention layer."""
890 super().__init__()
891 if hidden_size <= 0 or num_heads <= 0:
892 raise ValueError("hidden_size and num_heads must be positive.")
893 if num_v_heads is None:
894 num_v_heads = num_heads
895 if num_v_heads <= 0 or num_v_heads % num_heads:
896 raise ValueError("num_v_heads must be positive and divisible by num_heads.")
897 if head_k_dim <= 0 or head_v_dim <= 0:
898 raise ValueError("KDA head dimensions must be positive.")
899 if conv_kernel_size <= 0 or chunk_size <= 0:
900 raise ValueError("conv_kernel_size and chunk_size must be positive.")
901 if gate_lower_bound >= 0:
902 raise ValueError("gate_lower_bound must be negative.")
904 self.hidden_size = hidden_size
905 self.num_heads = num_heads
906 self.num_k_heads = num_heads
907 self.num_v_heads = num_v_heads
908 self.head_k_dim = head_k_dim
909 self.head_v_dim = head_v_dim
910 self.head_dim = head_v_dim
911 self.key_dim = num_heads * head_k_dim
912 self.value_dim = num_v_heads * head_v_dim
913 self.conv_kernel_size = conv_kernel_size
914 self.conv_size = conv_kernel_size
915 self.chunk_size = chunk_size
916 self.layer_idx = layer_idx
917 self.gate_lower_bound = gate_lower_bound
918 self.lower_bound = gate_lower_bound
919 self.safe_gate = True
920 self.allow_neg_eigval = False
921 self.use_short_conv = True
922 self.use_full_rank_gate = use_full_rank_gate
924 self.q_proj = nn.Linear(hidden_size, self.key_dim, bias=False)
925 self.k_proj = nn.Linear(hidden_size, self.key_dim, bias=False)
926 self.v_proj = nn.Linear(hidden_size, self.value_dim, bias=False)
927 self.q_conv1d = _make_short_convolution(self.key_dim, conv_kernel_size)
928 self.k_conv1d = _make_short_convolution(self.key_dim, conv_kernel_size)
929 self.v_conv1d = _make_short_convolution(self.value_dim, conv_kernel_size)
931 self.A_log = nn.Parameter(torch.empty(num_v_heads, dtype=torch.float32))
932 self.dt_bias = nn.Parameter(
933 torch.zeros(num_v_heads * head_k_dim, dtype=torch.float32)
934 )
935 with torch.no_grad():
936 self.A_log.copy_(torch.empty_like(self.A_log).uniform_(1, 16).log())
938 self.f_a_proj = nn.Linear(hidden_size, head_k_dim, bias=False)
939 self.f_b_proj = nn.Linear(head_k_dim, num_v_heads * head_k_dim, bias=False)
940 self.b_proj = nn.Linear(hidden_size, num_v_heads, bias=False)
941 if use_full_rank_gate:
942 self.g_proj = nn.Linear(hidden_size, self.value_dim, bias=False)
943 else:
944 self.g_a_proj = nn.Linear(hidden_size, head_v_dim, bias=False)
945 self.g_b_proj = nn.Linear(head_v_dim, self.value_dim, bias=False)
946 self.o_norm = KimiRMSNormGated(head_v_dim, eps=rms_norm_eps)
947 self.o_proj = nn.Linear(self.value_dim, hidden_size, bias=False)
949 @staticmethod
950 def _run_short_convolution(
951 projected: torch.Tensor,
952 convolution: nn.Conv1d,
953 sequence_info=None,
954 ) -> torch.Tensor:
955 """Apply one causal depthwise convolution and trim its right padding."""
956 if sequence_info is not None:
957 return packed_causal_conv1d(projected, convolution, sequence_info)
958 sequence_length = projected.shape[1]
959 output = convolution(projected.transpose(1, 2))[:, :, :sequence_length]
960 return F.silu(output.transpose(1, 2))
962 def _output_gate(self, hidden_states: torch.Tensor) -> torch.Tensor:
963 """Project the layer input to the per-head output gate."""
964 if self.use_full_rank_gate:
965 return self.g_proj(hidden_states)
966 return self.g_b_proj(self.g_a_proj(hidden_states))
968 def forward(
969 self,
970 hidden_states: torch.Tensor,
971 attention_mask: Optional[torch.Tensor] = None,
972 *,
973 use_cache: bool = False,
974 output_attentions: bool = False,
975 **kwargs: Any,
976 ) -> torch.Tensor:
977 """Run dense or packed training-time KDA without recurrent cache."""
978 if use_cache or kwargs.get("cache_params") is not None:
979 raise NotImplementedError("KimiDeltaAttention does not support inference cache.")
980 if output_attentions:
981 raise NotImplementedError("KDA does not materialize attention weights.")
982 if attention_mask is not None and not bool(attention_mask.bool().all().item()):
983 raise NotImplementedError("Padded KDA batches are not implemented.")
985 is_flat_input = hidden_states.ndim == 2
986 if is_flat_input:
987 hidden_states = hidden_states.unsqueeze(0)
988 batch_size, sequence_length, _ = hidden_states.shape
989 sequence_info = resolve_packed_sequence(kwargs, sequence_length)
990 if sequence_info is not None and batch_size != 1:
991 raise ValueError("Packed KDA requires a flattened batch dimension of one.")
992 query = self._run_short_convolution(
993 self.q_proj(hidden_states), self.q_conv1d, sequence_info
994 ).reshape(batch_size, sequence_length, self.num_heads, self.head_k_dim)
995 key = self._run_short_convolution(
996 self.k_proj(hidden_states), self.k_conv1d, sequence_info
997 ).reshape(batch_size, sequence_length, self.num_heads, self.head_k_dim)
998 value = self._run_short_convolution(
999 self.v_proj(hidden_states), self.v_conv1d, sequence_info
1000 ).reshape(batch_size, sequence_length, self.num_v_heads, self.head_v_dim)
1001 gate = self.f_b_proj(self.f_a_proj(hidden_states)).reshape(
1002 batch_size, sequence_length, self.num_v_heads, self.head_k_dim
1003 )
1004 beta = self.b_proj(hidden_states).reshape(
1005 batch_size, sequence_length, self.num_v_heads
1006 )
1007 output, _ = torch_chunk_kda(
1008 query,
1009 key,
1010 value,
1011 gate,
1012 beta,
1013 a_log=self.A_log,
1014 dt_bias=self.dt_bias,
1015 initial_state=None,
1016 output_final_state=False,
1017 use_qk_l2norm_in_kernel=True,
1018 use_gate_in_kernel=True,
1019 use_beta_sigmoid_in_kernel=True,
1020 lower_bound=self.gate_lower_bound,
1021 chunk_size=self.chunk_size,
1022 cu_seqlens=None if sequence_info is None else sequence_info.local_offsets,
1023 )
1024 output_gate = self._output_gate(hidden_states).reshape(
1025 batch_size, sequence_length, self.num_v_heads, self.head_v_dim
1026 )
1027 output = self.o_norm(output, output_gate)
1028 output = self.o_proj(output.reshape(batch_size, sequence_length, self.value_dim))
1029 return output.squeeze(0) if is_flat_input else output