1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""KDA context-parallel executors: state CP, Ulysses and their combinations."""
16from __future__ import annotations
17
18__all__ = [
19 "KimiDeltaAttentionCoreCP",
20 "KimiDeltaAttentionLayerCP",
21 "KimiDeltaAttentionLayerP2PCP",
22 "KimiDeltaAttentionLayerUlyssesCP",
23 "KimiDeltaAttentionP2PCP",
24 "KimiDeltaAttentionUlyssesCP",
25]
26
27from typing import Any, Optional
28
29import torch
30import torch.distributed as dist
31from torch import nn
32from torch.nn import functional as F
33
34from hyper_parallel.components.functional.linear_attention_convolution import (
35 fused_causal_conv1d,
36)
37from hyper_parallel.components.functional.linear_attention_packed import (
38 PackedSequenceInfo, packed_causal_conv1d, resolve_packed_sequence,
39)
40
41from hyper_parallel.core.context_parallel.context_parallel import _ensure_1d
42from hyper_parallel.core.dtensor.device_mesh import DeviceMesh
43from hyper_parallel.core.dtensor.dtensor import DTensor
44from hyper_parallel.components.modules.kimi_delta_attention import (
45 chunk_kda,
46 torch_apply_kda_state_summary,
47 torch_kda_state_summary,
48)
49from hyper_parallel.core.utils import communication
50from hyper_parallel.distributed.context_parallel.kimi_delta_attention_state import (
51 build_kda_state_cp,
52 split_kda_mesh,
53)
54
55
56_KDA_BACKENDS = frozenset({"eager", "triton", "ascendc"})
57
58
59def _run_local_kda(
60 query: torch.Tensor,
61 key: torch.Tensor,
62 value: torch.Tensor,
63 gate: torch.Tensor,
64 beta: torch.Tensor,
65 *,
66 a_log: torch.Tensor,
67 dt_bias: torch.Tensor,
68 scale: Optional[float],
69 lower_bound: float,
70 chunk_size: int,
71 safe_gate: bool,
72 backend: str,
73 sequence_info: Optional[PackedSequenceInfo] = None,
74) -> torch.Tensor:
75 """Run the explicitly selected local KDA implementation."""
76 if backend in ("triton", "ascendc") and not safe_gate:
77 raise RuntimeError("The fused KDA backend requires the lower-bounded gate.")
78 cu_seqlens = None
79 if sequence_info is not None:
80 # Native APIs consume host metadata; do not materialize an NPU plan
81 # only to copy its offsets back to the host at every layer.
82 cu_seqlens = (sequence_info.local_offsets if backend == "ascendc" else
83 sequence_info.materialize(query.device, chunk_size).cu_seqlens)
84 output, _ = chunk_kda(
85 query,
86 key,
87 value,
88 gate,
89 beta,
90 a_log=a_log,
91 dt_bias=dt_bias,
92 scale=scale,
93 lower_bound=lower_bound,
94 chunk_size=chunk_size,
95 initial_state=None,
96 output_final_state=False,
97 use_qk_l2norm_in_kernel=True,
98 use_gate_in_kernel=True,
99 use_beta_sigmoid_in_kernel=True,
100 backend=backend,
101 cu_seqlens=cu_seqlens,
102 )
103 return output
104
105
106def _is_official_kimi_layer(module: nn.Module) -> bool:
107 """Whether ``module`` follows the official Kimi K3 projection contract."""
108 return hasattr(module, "f_a_proj") and hasattr(module, "f_b_proj")
109
110
111def _num_value_heads(module: nn.Module) -> int:
112 """Return the number of KDA value heads for either supported layer API."""
113 if hasattr(module, "num_v_heads"):
114 return int(module.num_v_heads)
115 return int(module.num_heads)
116
117
118def _value_head_dim(module: nn.Module) -> int:
119 """Return the KDA value head width for either supported layer API."""
120 if hasattr(module, "head_v_dim"):
121 return int(module.head_v_dim)
122 return int(module.head_dim)
123
124
125def _key_projection_dim(module: nn.Module) -> int:
126 """Return the flattened Q/K projection width."""
127 if hasattr(module, "key_dim"):
128 return int(module.key_dim)
129 return int(module.num_heads * module.head_k_dim)
130
131
132def _value_projection_dim(module: nn.Module) -> int:
133 """Return the flattened value projection width."""
134 if hasattr(module, "value_dim"):
135 return int(module.value_dim)
136 return int(_num_value_heads(module) * _value_head_dim(module))
137
138
139def _gate_lower_bound(module: nn.Module) -> Optional[float]:
140 """Return the lower bound used by the safe KDA gate parameterization."""
141 return getattr(module, "lower_bound", getattr(module, "gate_lower_bound", None))
142
143
144def _uses_safe_gate(module: nn.Module) -> bool:
145 """Return whether the layer enables the safe lower-bounded gate."""
146 return bool(getattr(module, "safe_gate", _gate_lower_bound(module) is not None))
147
148
149def _uses_short_conv(module: nn.Module) -> bool:
150 """Return whether Q/K/V pass through the layer's ShortConvolution modules."""
151 return bool(
152 getattr(
153 module,
154 "use_short_conv",
155 all(hasattr(module, name) for name in ("q_conv1d", "k_conv1d", "v_conv1d")),
156 )
157 )
158
159
160def _forget_gate_projection(module: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
161 """Run the module-specific state-decay gate projection."""
162 if hasattr(module, "f_proj"):
163 return module.f_proj(hidden_states)
164 return module.f_b_proj(module.f_a_proj(hidden_states))
165
166
167def _output_gate_projection(module: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
168 """Run the module-specific output-gate projection."""
169 if hasattr(module, "g_proj"):
170 return module.g_proj(hidden_states)
171 return module.g_b_proj(module.g_a_proj(hidden_states))
172
173
174def _format_layer_output(module: nn.Module, output: torch.Tensor):
175 """Preserve the original layer's tensor or three-tuple output contract."""
176 if _is_official_kimi_layer(module):
177 return output
178 return output, None, None
179
180
181def _prepare_layer_input(
182 hidden_states: torch.Tensor,
183 kwargs: dict[str, Any],
184 cp_rank: int,
185 cp_size: int,
186) -> tuple[torch.Tensor, Optional[PackedSequenceInfo], bool]:
187 """Normalize a local layer input and resolve this invocation's packed metadata."""
188 hidden_states = _local_cp_tensor(hidden_states)
189 # Restore the caller's layout on return; offsets define document boundaries.
190 is_flat_input = hidden_states.ndim == 2
191 if is_flat_input:
192 hidden_states = hidden_states.unsqueeze(0)
193 sequence_info = resolve_packed_sequence(
194 kwargs, hidden_states.shape[1], cp_rank=cp_rank, cp_size=cp_size)
195 if sequence_info is not None and hidden_states.shape[0] != 1:
196 raise ValueError("Packed KDA requires batch dimension one.")
197 return hidden_states, sequence_info, is_flat_input
198
199
200def _global_peer_rank(cp_mesh: DeviceMesh, local_rank: int) -> int:
201 """Map one CP-local rank index to its global distributed rank."""
202 return int(cp_mesh.rank_list[local_rank])
203
204
205def _local_cp_tensor(tensor: torch.Tensor) -> torch.Tensor:
206 """Return the local sequence shard of a CP input tensor."""
207 if isinstance(tensor, DTensor):
208 return tensor.to_local()
209 return tensor
210
211
212def _all_to_all_previous_rank_halo(
213 tail: torch.Tensor,
214 cp_mesh: DeviceMesh,
215 cp_rank: int,
216 cp_size: int,
217) -> torch.Tensor:
218 """Send a causal-convolution halo only to the next CP rank."""
219 if cp_size == 1:
220 return torch.zeros_like(tail)
221
222 cp_group = cp_mesh.get_group()
223 group_ranks = tuple(int(rank) for rank in dist.get_process_group_ranks(cp_group))
224 rank_list = tuple(int(rank) for rank in cp_mesh.rank_list)
225 rank_to_group_index = {rank: index for index, rank in enumerate(group_ranks)}
226 halo_width = tail.shape[1]
227
228 input_splits = [0] * cp_size
229 exchange_input = tail.permute(1, 0, 2).contiguous()
230 if cp_rank < cp_size - 1:
231 input_splits[rank_to_group_index[rank_list[cp_rank + 1]]] = halo_width
232 else:
233 exchange_input = exchange_input[:0]
234
235 output_splits = [0] * cp_size
236 if cp_rank > 0:
237 output_splits[rank_to_group_index[rank_list[cp_rank - 1]]] = halo_width
238
239 exchange_output = communication.differentiable_all_to_all_single(
240 exchange_input,
241 input_splits,
242 output_splits,
243 group=cp_group,
244 )
245 if cp_rank == 0:
246 return torch.zeros_like(tail) + exchange_output.sum().to(tail.dtype) * 0
247 return exchange_output.permute(1, 0, 2).contiguous()
248
249
250def _run_causal_short_conv(
251 tensor: torch.Tensor,
252 convolution: nn.Conv1d,
253 halo: Optional[torch.Tensor] = None,
254 sequence_info: Optional[PackedSequenceInfo] = None,
255 backend: str = "eager",
256) -> torch.Tensor:
257 """Apply one local ShortConv, optionally with a preceding-rank halo."""
258 if backend != "eager":
259 return fused_causal_conv1d(
260 tensor, convolution.weight, convolution.bias, sequence_info, halo,
261 backend=backend, dilation=convolution.dilation[0])
262 if sequence_info is not None:
263 return packed_causal_conv1d(tensor, convolution, sequence_info, halo)
264 if halo is None:
265 conv_input = tensor.transpose(1, 2)
266 padding = convolution.padding
267 else:
268 conv_input = torch.cat((halo, tensor), dim=1).transpose(1, 2)
269 padding = 0
270 output = F.conv1d( # pylint: disable=not-callable
271 input=conv_input,
272 weight=convolution.weight,
273 bias=convolution.bias,
274 stride=convolution.stride,
275 padding=padding,
276 dilation=convolution.dilation,
277 groups=convolution.groups,
278 )
279 if halo is None:
280 output = output[:, :, : tensor.shape[1]]
281 return F.silu(output).transpose(1, 2)
282
283
284def _causal_short_convs_with_cp_halo(
285 projected: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
286 convolutions: tuple[nn.Conv1d, nn.Conv1d, nn.Conv1d],
287 cp_mesh: DeviceMesh,
288 cp_rank: int,
289 cp_size: int,
290 sequence_info: Optional[PackedSequenceInfo] = None,
291 backend: str = "eager",
292) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
293 """Run three causal depthwise ShortConvs with one packed halo exchange."""
294 halo_widths = {
295 (conv.kernel_size[0] - 1) * conv.dilation[0] for conv in convolutions
296 }
297 if len(halo_widths) != 1:
298 raise ValueError("KDA P2P requires Q/K/V ShortConv halo widths to match.")
299 halo_width = halo_widths.pop()
300 if halo_width == 0 or cp_size == 1:
301 return tuple(
302 _run_causal_short_conv(tensor, convolution, sequence_info=sequence_info, backend=backend)
303 for tensor, convolution in zip(projected, convolutions)
304 )
305 if projected[0].shape[1] < halo_width:
306 raise ValueError(
307 "KDA P2P ShortConv requires local_seq_len >= halo width, got "
308 f"{projected[0].shape[1]} < {halo_width}."
309 )
310
311 channel_sizes = [tensor.shape[-1] for tensor in projected]
312 packed_tail = torch.cat(
313 [tensor[:, -halo_width:, :] for tensor in projected],
314 dim=-1,
315 ).contiguous()
316 packed_halo = _all_to_all_previous_rank_halo(
317 packed_tail,
318 cp_mesh,
319 cp_rank,
320 cp_size,
321 )
322 halos = torch.split(packed_halo, channel_sizes, dim=-1)
323 return tuple(
324 _run_causal_short_conv(tensor, convolution, halo, sequence_info=sequence_info, backend=backend)
325 for tensor, halo, convolution in zip(projected, halos, convolutions)
326 )
327
328
329def _slice_local_heads(
330 tensor: torch.Tensor,
331 cp_rank: int,
332 cp_size: int,
333) -> torch.Tensor:
334 """Return the current CP rank's contiguous head slice."""
335 if tensor.shape[0] % cp_size != 0:
336 raise ValueError(
337 f"KDA head dimension {tensor.shape[0]} must be divisible by "
338 f"cp_size {cp_size}."
339 )
340 local_heads = tensor.shape[0] // cp_size
341 return tensor.narrow(0, cp_rank * local_heads, local_heads)
342
343
344def _differentiable_all_to_all_shard(
345 tensor: torch.Tensor,
346 device_mesh: DeviceMesh,
347 *,
348 split_dim: int,
349 concat_dim: int,
350) -> torch.Tensor:
351 """Redistribute a local tensor from concat-dim shard to split-dim shard."""
352 split_count = device_mesh.size()
353 if split_count == 1:
354 return tensor
355
356 original_shape = tuple(tensor.shape)
357 if original_shape[split_dim] % split_count != 0:
358 raise ValueError(
359 f"KDA all-to-all split dimension {split_dim} with size "
360 f"{original_shape[split_dim]} must be divisible by cp_size "
361 f"{split_count}."
362 )
363 split_size = original_shape[split_dim] // split_count
364 final_shape = list(original_shape)
365 final_shape[split_dim] = split_size
366 final_shape[concat_dim] *= split_count
367
368 reshape_dims = list(original_shape)
369 reshape_dims[split_dim] = split_count
370 reshape_dims.insert(split_dim + 1, split_size)
371 permutation = list(range(len(reshape_dims)))
372 permutation.remove(split_dim)
373 permutation.insert(0, split_dim)
374 all_to_all_input = tensor.reshape(reshape_dims).permute(permutation).contiguous()
375 all_to_all_input = all_to_all_input.flatten(0, 1)
376
377 split_length = all_to_all_input.shape[0] // split_count
378 splits = [split_length] * split_count
379 output = communication.differentiable_all_to_all_single(
380 all_to_all_input,
381 splits,
382 splits,
383 group=device_mesh.get_group(),
384 )
385
386 output_reshape = list(output.shape)
387 output_reshape[0] = split_count
388 output_reshape.insert(1, output.shape[0] // split_count)
389 permutation = list(range(len(output_reshape)))
390 rank_dimension = permutation.pop(0)
391 permutation.insert(concat_dim, rank_dimension)
392 output = output.reshape(output_reshape).permute(permutation).contiguous()
393
394 output_shape = list(output.shape)
395 output_shape[concat_dim] *= output_shape[concat_dim + 1]
396 output_shape.pop(concat_dim + 1)
397 return output.reshape(output_shape).view(final_shape)
398
399
400class _RecvKDAInitialState(torch.autograd.Function):
401 """Receive a KDA initial state and return its gradient to the sender."""
402
403 @staticmethod
404 def forward( # pylint: disable=arguments-differ
405 ctx: Any,
406 anchor: torch.Tensor,
407 cp_group: Any,
408 prev_rank: int,
409 state_shape: tuple[int, ...],
410 ) -> torch.Tensor:
411 """Receive the initial recurrent state from the previous CP rank.
412
413 Args:
414 ctx: Autograd context holding tensors and communication metadata.
415 anchor: Tensor supplying the receive device and autograd dependency.
416 cp_group: Precreated chronological CP process group.
417 prev_rank: Global rank of the preceding sequence partition.
418 state_shape: Shape of the FP32 recurrent state.
419 """
420 state = torch.empty(state_shape, device=anchor.device, dtype=torch.float32)
421 dist.recv(state, src=prev_rank, group=cp_group)
422 ctx.cp_group = cp_group
423 ctx.prev_rank = prev_rank
424 return state
425
426 @staticmethod
427 def backward(
428 ctx: Any,
429 grad_state: Optional[torch.Tensor],
430 ) -> tuple[None, None, None, None]:
431 """Send the accumulated initial-state gradient to the previous rank.
432
433 Args:
434 ctx: Autograd context holding tensors and communication metadata.
435 grad_state: Gradient of the received initial state.
436 """
437 if grad_state is None:
438 raise RuntimeError("KDA P2P backward is missing the initial-state gradient.")
439 dist.send(grad_state.contiguous(), dst=ctx.prev_rank, group=ctx.cp_group)
440 return None, None, None, None
441
442
443class _SendKDAFinalState(torch.autograd.Function):
444 """Send a KDA final state and receive its gradient during backward."""
445
446 @staticmethod
447 def forward( # pylint: disable=arguments-differ
448 ctx: Any,
449 final_state: torch.Tensor,
450 cp_group: Any,
451 next_rank: int,
452 ) -> torch.Tensor:
453 """Send the final recurrent state to the next CP rank.
454
455 Args:
456 ctx: Autograd context holding tensors and communication metadata.
457 final_state: Outgoing recurrent state.
458 cp_group: Precreated chronological CP process group.
459 next_rank: Global rank of the following sequence partition.
460 """
461 dist.send(final_state.contiguous(), dst=next_rank, group=cp_group)
462 ctx.cp_group = cp_group
463 ctx.next_rank = next_rank
464 ctx.state_shape = tuple(final_state.shape)
465 ctx.state_dtype = final_state.dtype
466 return final_state.new_zeros(())
467
468 @staticmethod
469 def backward(
470 ctx: Any,
471 grad_token: torch.Tensor,
472 ) -> tuple[torch.Tensor, None, None]:
473 """Receive the final-state gradient from the next rank.
474
475 Args:
476 ctx: Autograd context holding tensors and communication metadata.
477 grad_token: Gradient token supplying the receive device.
478 """
479 grad_state = torch.empty(
480 ctx.state_shape,
481 device=grad_token.device,
482 dtype=ctx.state_dtype,
483 )
484 dist.recv(grad_state, src=ctx.next_rank, group=ctx.cp_group)
485 return grad_state, None, None
486
487
488def _eager_kda_packed_p2p(
489 query: torch.Tensor,
490 key: torch.Tensor,
491 value: torch.Tensor,
492 gate: torch.Tensor,
493 beta: torch.Tensor,
494 *,
495 a_log: torch.Tensor,
496 dt_bias: torch.Tensor,
497 scale: Optional[float],
498 lower_bound: float,
499 chunk_size: int,
500 cp_group: Any,
501 prev_rank: int,
502 next_rank: int,
503 cp_rank: int,
504 cp_size: int,
505 sequence_info: Optional[PackedSequenceInfo] = None,
506) -> torch.Tensor:
507 """Run native per-document recurrence with differentiable neighbor states."""
508 # Portable semantic reference: native fragment recurrence, fixed sends.
509 incoming = None
510 if cp_rank > 0:
511 shape = (1, value.shape[2], query.shape[-1], value.shape[-1])
512 incoming = _RecvKDAInitialState.apply(query, cp_group, prev_rank, shape)
513 output, finals = chunk_kda(
514 query, key, value, gate, beta, a_log=a_log, dt_bias=dt_bias, scale=scale,
515 initial_state=sequence_info.first_state(incoming), output_final_state=True,
516 use_qk_l2norm_in_kernel=True, use_gate_in_kernel=True, use_beta_sigmoid_in_kernel=True,
517 lower_bound=lower_bound, chunk_size=chunk_size, backend="eager",
518 cu_seqlens=sequence_info.local_offsets,
519 )
520 if incoming is not None:
521 output = output + incoming.sum().to(output.dtype) * 0
522 if cp_rank < cp_size - 1:
523 token = _SendKDAFinalState.apply(finals[-1:].contiguous(), cp_group, next_rank)
524 output = output + token.to(output.dtype) * 0
525 return output
526
527
528def _eager_kda_state_p2p(
529 query: torch.Tensor,
530 key: torch.Tensor,
531 value: torch.Tensor,
532 gate: torch.Tensor,
533 beta: torch.Tensor,
534 *,
535 a_log: torch.Tensor,
536 dt_bias: torch.Tensor,
537 scale: Optional[float],
538 lower_bound: float,
539 chunk_size: int,
540 cp_group: Any,
541 prev_rank: int,
542 next_rank: int,
543 cp_rank: int,
544 cp_size: int,
545 sequence_info: Optional[PackedSequenceInfo] = None,
546) -> torch.Tensor:
547 """Run the differentiable Torch reference for affine state-P2P KDA."""
548 if sequence_info is not None:
549 return _eager_kda_packed_p2p(
550 query, key, value, gate, beta, a_log=a_log, dt_bias=dt_bias, scale=scale,
551 lower_bound=lower_bound, chunk_size=chunk_size, cp_group=cp_group,
552 prev_rank=prev_rank, next_rank=next_rank, cp_rank=cp_rank, cp_size=cp_size,
553 sequence_info=sequence_info,
554 )
555 if cp_size == 1:
556 return _run_local_kda(
557 query,
558 key,
559 value,
560 gate,
561 beta,
562 a_log=a_log,
563 dt_bias=dt_bias,
564 scale=scale,
565 lower_bound=lower_bound,
566 chunk_size=chunk_size,
567 safe_gate=True,
568 backend="eager",
569 )
570
571 state_ext = None
572 transition = None
573 if cp_rank < cp_size - 1:
574 state_ext, transition = torch_kda_state_summary(
575 query,
576 key,
577 value,
578 gate,
579 beta,
580 a_log=a_log,
581 dt_bias=dt_bias,
582 scale=scale,
583 use_qk_l2norm_in_kernel=True,
584 use_gate_in_kernel=True,
585 use_beta_sigmoid_in_kernel=True,
586 lower_bound=lower_bound,
587 chunk_size=chunk_size,
588 )
589
590 initial_state = None
591 if cp_rank > 0:
592 state_shape = (
593 query.shape[0],
594 value.shape[2],
595 query.shape[-1],
596 value.shape[-1],
597 )
598 initial_state = _RecvKDAInitialState.apply(
599 query,
600 cp_group,
601 prev_rank,
602 state_shape,
603 )
604
605 send_token = None
606 if cp_rank < cp_size - 1:
607 final_state = torch_apply_kda_state_summary(state_ext, transition, initial_state)
608 send_token = _SendKDAFinalState.apply(final_state, cp_group, next_rank)
609
610 output, _ = chunk_kda(
611 query,
612 key,
613 value,
614 gate,
615 beta,
616 a_log=a_log,
617 dt_bias=dt_bias,
618 scale=scale,
619 initial_state=initial_state,
620 output_final_state=False,
621 use_qk_l2norm_in_kernel=True,
622 use_gate_in_kernel=True,
623 use_beta_sigmoid_in_kernel=True,
624 lower_bound=lower_bound,
625 chunk_size=chunk_size,
626 backend="eager",
627 )
628 if send_token is not None:
629 output = output + send_token.to(output.dtype) * 0
630 return output
631
632
633def _validate_projected_inputs(
634 ulysses_degree: int,
635 query: torch.Tensor,
636 key: torch.Tensor,
637 value: torch.Tensor,
638 gate: torch.Tensor,
639 beta: torch.Tensor,
640 a_log: torch.Tensor,
641 dt_bias: torch.Tensor,
642) -> None:
643 """Validate the projected-tensor contract and Ulysses divisibility."""
644 if (query.dim(), key.dim(), value.dim(), gate.dim()) != (4, 4, 4, 4):
645 raise ValueError("query, key, value, and gate must be rank-4 tensors.")
646 if beta.dim() != 3:
647 raise ValueError("beta must be a rank-3 tensor.")
648 if query.shape != key.shape:
649 raise ValueError("query and key must have identical shapes.")
650
651 batch_size, local_seq_len, num_k_heads, k_head_dim = query.shape
652 expected_prefix = (batch_size, local_seq_len)
653 if value.shape[:2] != expected_prefix:
654 raise ValueError("value batch and sequence dimensions must match query.")
655 num_v_heads = value.shape[2]
656 if gate.shape != (batch_size, local_seq_len, num_v_heads, k_head_dim):
657 raise ValueError(
658 "gate must have shape [B, S_local, num_v_heads, k_head_dim]."
659 )
660 if beta.shape != (batch_size, local_seq_len, num_v_heads):
661 raise ValueError("beta must have shape [B, S_local, num_v_heads].")
662 if num_v_heads % num_k_heads != 0:
663 raise ValueError("num_v_heads must be divisible by num_k_heads.")
664 if num_k_heads % ulysses_degree != 0:
665 raise ValueError(
666 f"KDA num_k_heads ({num_k_heads}) must be divisible by "
667 f"cp_size ({ulysses_degree}) for Ulysses CP."
668 )
669 if num_v_heads % ulysses_degree != 0:
670 raise ValueError(
671 f"KDA num_v_heads ({num_v_heads}) must be divisible by "
672 f"cp_size ({ulysses_degree}) for Ulysses CP."
673 )
674 if a_log.numel() != num_v_heads:
675 raise ValueError(
676 f"a_log must contain {num_v_heads} elements, got {a_log.numel()}."
677 )
678 if dt_bias.numel() != num_v_heads * k_head_dim:
679 raise ValueError(
680 f"dt_bias must contain {num_v_heads * k_head_dim} elements, "
681 f"got {dt_bias.numel()}."
682 )
683
684
685class KimiDeltaAttentionUlyssesCP(nn.Module):
686 """Run KDA on sequence-sharded projected tensors with Ulysses all-to-all.
687
688 The input and output layout is token-first with a local sequence shard:
689
690 ``[B, S/P, H, D] -> [B, S, H/P, D] -> [B, S/P, H, D]``.
691
692 This first-stage interface intentionally starts after projections and
693 ShortConv and ends before the output norm/gate/projection. It is therefore
694 independent of one particular Kimi K3 model class. Training uses a zero
695 initial recurrent state and does not materialize a final state.
696 """
697
698 def __init__(
699 self,
700 device_mesh: DeviceMesh,
701 *,
702 chunk_size: int = 64,
703 lower_bound: float = -5.0,
704 safe_gate: bool = True,
705 backend: str = "eager",
706 ) -> None:
707 """Initialize the KDA Ulysses executor for one 1-D CP mesh."""
708 super().__init__()
709 if chunk_size <= 0:
710 raise ValueError(f"chunk_size must be positive, got {chunk_size}.")
711 if lower_bound >= 0:
712 raise ValueError(f"lower_bound must be negative, got {lower_bound}.")
713 backend = backend.lower()
714 if backend not in _KDA_BACKENDS:
715 raise ValueError(
716 f"unsupported KDA backend {backend!r}; "
717 f"expected one of {sorted(_KDA_BACKENDS)}."
718 )
719 self.cp_mesh = _ensure_1d(device_mesh)
720 self.cp_size = self.cp_mesh.size()
721 self.cp_rank = self.cp_mesh.get_local_rank()
722 self.chunk_size = chunk_size
723 self.lower_bound = lower_bound
724 self.safe_gate = safe_gate
725 self.backend = backend
726 self.seq_dim = 1
727 self.head_dim = 2
728
729 def _validate_inputs(
730 self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
731 gate: torch.Tensor, beta: torch.Tensor, a_log: torch.Tensor, dt_bias: torch.Tensor,
732 ) -> None:
733 """Validate the projected contract for this Ulysses degree."""
734 _validate_projected_inputs(self.cp_size, query, key, value, gate, beta, a_log, dt_bias)
735
736 def _seq_to_head(self, tensor: torch.Tensor) -> torch.Tensor:
737 """Redistribute a local sequence shard into a local head shard."""
738 return _differentiable_all_to_all_shard(
739 tensor,
740 self.cp_mesh,
741 split_dim=self.head_dim,
742 concat_dim=self.seq_dim,
743 )
744
745 def _head_to_seq(self, tensor: torch.Tensor) -> torch.Tensor:
746 """Redistribute a local head shard back into a local sequence shard."""
747 return _differentiable_all_to_all_shard(
748 tensor,
749 self.cp_mesh,
750 split_dim=self.seq_dim,
751 concat_dim=self.head_dim,
752 )
753
754 def forward(
755 self,
756 query: torch.Tensor,
757 key: torch.Tensor,
758 value: torch.Tensor,
759 gate: torch.Tensor,
760 beta: torch.Tensor,
761 *,
762 a_log: torch.Tensor,
763 dt_bias: torch.Tensor,
764 scale: Optional[float] = None,
765 sequence_info: Optional[PackedSequenceInfo] = None,
766 ) -> torch.Tensor:
767 """Run chunkwise KDA using full sequence and local heads.
768
769 Args:
770 query: Local query tensor [B,T,H,K].
771 key: Matching local key tensor.
772 value: Local value tensor [B,T,HV,V].
773 gate: Per-token gate logits [B,T,HV,K].
774 beta: Per-token beta logits [B,T,HV].
775 a_log: Decay parameter with one value per value head.
776 dt_bias: Learned bias with one value per gate channel.
777 scale: Optional attention scaling factor.
778 """
779 is_flat_input = query.ndim == 3
780 if is_flat_input:
781 output = self.forward(*(tensor.unsqueeze(0) for tensor in (query, key, value, gate, beta)),
782 a_log=a_log, dt_bias=dt_bias, scale=scale, sequence_info=sequence_info)
783 return output.squeeze(0)
784 self._validate_inputs(query, key, value, gate, beta, a_log, dt_bias)
785 exchanged_info = None if sequence_info is None else sequence_info.ulysses(self.cp_size, self.cp_rank)
786 num_v_heads = value.shape[2]
787 k_head_dim = query.shape[-1]
788
789 query = self._seq_to_head(query)
790 key = self._seq_to_head(key)
791 value = self._seq_to_head(value)
792 gate = self._seq_to_head(gate)
793 beta = self._seq_to_head(beta)
794
795 local_a_log = _slice_local_heads(
796 a_log.reshape(num_v_heads),
797 self.cp_rank,
798 self.cp_size,
799 )
800 local_dt_bias = _slice_local_heads(
801 dt_bias.reshape(num_v_heads, k_head_dim),
802 self.cp_rank,
803 self.cp_size,
804 )
805 output = _run_local_kda(
806 query,
807 key,
808 value,
809 gate,
810 beta,
811 a_log=local_a_log,
812 dt_bias=local_dt_bias,
813 scale=scale,
814 lower_bound=self.lower_bound,
815 chunk_size=self.chunk_size,
816 safe_gate=self.safe_gate,
817 backend=self.backend,
818 sequence_info=exchanged_info,
819 )
820 return self._head_to_seq(output)
821
822
823class _KDAStateCP(nn.Module):
824 """Run local KDA with the configured state exchange."""
825
826 def __init__(
827 self,
828 device_mesh: DeviceMesh,
829 *,
830 chunk_size: int = 64,
831 lower_bound: float = -5.0,
832 safe_gate: bool = True,
833 backend: str = "eager",
834 state_cp_method: str = "p2p",
835 allgather_group_size: int = 1,
836 ) -> None:
837 """Initialize local KDA and its state exchange on one chronological axis."""
838 super().__init__()
839 if chunk_size <= 0:
840 raise ValueError(f"chunk_size must be positive, got {chunk_size}.")
841 backend = backend.lower()
842 if backend not in _KDA_BACKENDS:
843 raise ValueError(
844 f"unsupported KDA backend {backend!r}; "
845 f"expected one of {sorted(_KDA_BACKENDS)}."
846 )
847 self.cp_mesh = _ensure_1d(device_mesh)
848 self.cp_size = self.cp_mesh.size()
849 self.cp_rank = self.cp_mesh.get_local_rank()
850 self.cp_group = self.cp_mesh.get_group()
851 if backend == "ascendc" and (chunk_size != 64 or not safe_gate):
852 raise ValueError("Ascend C KDA requires safe_gate=True and chunk_size=64.")
853 if backend == "eager" and state_cp_method != "p2p":
854 raise ValueError("Eager KDA state exchange supports P2P only.")
855 self.state_cp = build_kda_state_cp(self.cp_mesh, state_cp_method, allgather_group_size)
856 self.prev_rank = _global_peer_rank(
857 self.cp_mesh,
858 max(self.cp_rank - 1, 0),
859 )
860 self.next_rank = _global_peer_rank(
861 self.cp_mesh,
862 min(self.cp_rank + 1, self.cp_size - 1),
863 )
864 self.chunk_size = chunk_size
865 self.lower_bound = lower_bound
866 self.safe_gate = safe_gate
867 self.backend = backend
868
869 def forward(
870 self,
871 query: torch.Tensor,
872 key: torch.Tensor,
873 value: torch.Tensor,
874 gate: torch.Tensor,
875 beta: torch.Tensor,
876 *,
877 a_log: torch.Tensor,
878 dt_bias: torch.Tensor,
879 scale: Optional[float] = None,
880 sequence_info: Optional[PackedSequenceInfo] = None,
881 ) -> torch.Tensor:
882 """Run one local KDA segment and propagate its recurrent state.
883
884 Args:
885 query: Local query tensor [B,T,H,K].
886 key: Matching local key tensor.
887 value: Local value tensor [B,T,HV,V].
888 gate: Per-token gate logits [B,T,HV,K].
889 beta: Per-token beta logits [B,T,HV].
890 a_log: Decay parameter with one value per value head.
891 dt_bias: Learned bias with one value per gate channel.
892 scale: Optional attention scaling factor.
893 """
894 if self.cp_size == 1:
895 return _run_local_kda(
896 query,
897 key,
898 value,
899 gate,
900 beta,
901 a_log=a_log,
902 dt_bias=dt_bias,
903 scale=scale,
904 lower_bound=self.lower_bound,
905 chunk_size=self.chunk_size,
906 safe_gate=self.safe_gate,
907 backend=self.backend,
908 sequence_info=sequence_info,
909 )
910 if self.backend == "eager":
911 return _eager_kda_state_p2p(
912 query,
913 key,
914 value,
915 gate,
916 beta,
917 a_log=a_log,
918 dt_bias=dt_bias,
919 scale=scale,
920 lower_bound=self.lower_bound,
921 chunk_size=self.chunk_size,
922 cp_group=self.cp_group,
923 prev_rank=self.prev_rank,
924 next_rank=self.next_rank,
925 cp_rank=self.cp_rank,
926 cp_size=self.cp_size,
927 sequence_info=sequence_info,
928 )
929
930 if self.backend == "ascendc":
931 # The native package remains optional; loading it belongs to the selected backend.
932 from hyper_parallel.components.functional.kimi_delta_attention_ascendc import ( # pylint: disable=import-outside-toplevel
933 ascendc_state_kda,
934 )
935 return ascendc_state_kda(
936 query, key, value, gate, beta, a_log=a_log, dt_bias=dt_bias, scale=scale,
937 lower_bound=self.lower_bound, state_cp=self.state_cp, cp_mesh=self.cp_mesh,
938 sequence_info=sequence_info)
939
940 from hyper_parallel.components.functional.kimi_delta_attention import ( # pylint: disable=import-outside-toplevel
941 fused_chunk_kda_p2p,
942 )
943 return fused_chunk_kda_p2p(
944 query,
945 key,
946 value,
947 gate,
948 beta,
949 a_log=a_log.float(),
950 dt_bias=dt_bias.float(),
951 cp_group=self.cp_group,
952 prev_rank=self.prev_rank,
953 next_rank=self.next_rank,
954 cp_rank=self.cp_rank,
955 cp_size=self.cp_size,
956 scale=scale,
957 lower_bound=self.lower_bound,
958 chunk_size=self.chunk_size,
959 safe_gate=self.safe_gate,
960 state_cp=self.state_cp,
961 sequence_info=sequence_info,
962 )
963
964
965class KimiDeltaAttentionP2PCP(_KDAStateCP):
966 """Preserve the public projected-P2P type while sharing state execution.
967
968 Inherited defaults select P2P. Keeping a named subclass preserves the old
969 import and type name without duplicating the common implementation.
970 New hybrid configurations use KimiDeltaAttentionCoreCP.
971 """
972
973
974def _validate_layer_short_conv(module: nn.Module, mode: str) -> None:
975 """Check the shared depthwise-convolution contract before building execution."""
976 if _uses_short_conv(module):
977 for name in ("q_conv1d", "k_conv1d", "v_conv1d"):
978 if not hasattr(module, name):
979 raise TypeError(f"Kimi K3 layer is missing {name}.")
980 convolution = getattr(module, name)
981 if convolution.stride != (1,) or convolution.dilation != (1,):
982 raise ValueError(
983 f"Kimi K3 {mode} CP supports ShortConv stride=1 and "
984 "dilation=1 only."
985 )
986 if convolution.groups != convolution.in_channels or convolution.out_channels != convolution.in_channels:
987 raise ValueError(f"Kimi K3 {mode} CP expects depthwise ShortConv with equal input/output channels.")
988 if convolution.padding != (convolution.kernel_size[0] - 1,) or convolution.padding_mode != "zeros":
989 raise ValueError(f"Kimi K3 {mode} CP expects causal zero-padded ShortConv.")
990 if getattr(convolution, "activation", None) not in ("silu", "swish"):
991 raise ValueError(
992 f"Kimi K3 {mode} CP expects ShortConv SiLU activation."
993 )
994
995
996class KimiDeltaAttentionLayerUlyssesCP(KimiDeltaAttentionUlyssesCP):
997 """Wrap a training-time Kimi K3 attention layer with Ulysses CP.
998
999 The wrapped module keeps ownership of all model parameters. Its local
1000 sequence shard is projected first, then Q/K/V, per-dimension gate logits,
1001 and beta logits are packed into one sequence-to-head all-to-all. Each CP
1002 rank runs the wrapped layer's ShortConv weights and fused KDA core for its
1003 local head shard and the full sequence. The reverse all-to-all restores a
1004 local sequence shard before the original output gate, norm, and projection.
1005
1006 Packed samples use global document offsets. ``backend`` selects both the
1007 attention core and local convolution for dense and packed inputs.
1008 Inference cache and padded sequences are unsupported.
1009 """
1010
1011 def __init__(
1012 self,
1013 module: nn.Module,
1014 device_mesh: DeviceMesh,
1015 *,
1016 chunk_size: int = 64,
1017 backend: str = "eager",
1018 ) -> None:
1019 """Initialize the full-layer adapter without replacing its parameters.
1020
1021 Args:
1022 module: Original Kimi attention layer that owns the parameters.
1023 device_mesh: One-dimensional Ulysses mesh.
1024 chunk_size: Chunk length for the attention core.
1025 backend: Attention and convolution backend: ``eager`` or ``triton``.
1026
1027 Raises:
1028 ValueError: If a backend or the wrapped layer contract is unsupported.
1029 """
1030 lower_bound = _gate_lower_bound(module)
1031 if lower_bound is None:
1032 raise ValueError(
1033 "Kimi K3 Ulysses CP requires the lower-bounded gate parameterization."
1034 )
1035 super().__init__(
1036 device_mesh,
1037 chunk_size=chunk_size,
1038 lower_bound=float(lower_bound),
1039 safe_gate=_uses_safe_gate(module),
1040 backend=backend,
1041 )
1042 self.module = module
1043 self._validate_module()
1044
1045 def _validate_module(self) -> None:
1046 """Validate the first-stage KimiDeltaAttention module contract."""
1047 required_attributes = (
1048 "q_proj",
1049 "k_proj",
1050 "v_proj",
1051 "b_proj",
1052 "o_norm",
1053 "o_proj",
1054 "A_log",
1055 "dt_bias",
1056 "num_heads",
1057 "head_k_dim",
1058 )
1059 missing = [name for name in required_attributes if not hasattr(self.module, name)]
1060 if missing:
1061 raise TypeError(
1062 "Kimi K3 layer is missing required attributes: "
1063 + ", ".join(missing)
1064 )
1065 if self.module.num_heads % self.cp_size != 0:
1066 raise ValueError(
1067 f"KDA num_heads ({self.module.num_heads}) must be divisible by "
1068 f"cp_size ({self.cp_size})."
1069 )
1070 num_v_heads = _num_value_heads(self.module)
1071 if num_v_heads % self.cp_size != 0:
1072 raise ValueError(
1073 f"KDA num_v_heads ({num_v_heads}) must be divisible "
1074 f"by cp_size ({self.cp_size})."
1075 )
1076 if num_v_heads % self.module.num_heads != 0:
1077 raise ValueError("KDA num_v_heads must be divisible by num_heads.")
1078 if not (hasattr(self.module, "f_proj") or _is_official_kimi_layer(self.module)):
1079 raise TypeError("Kimi K3 layer is missing its forget-gate projection.")
1080 if not (
1081 hasattr(self.module, "g_proj")
1082 or all(hasattr(self.module, name) for name in ("g_a_proj", "g_b_proj"))
1083 ):
1084 raise TypeError("Kimi K3 layer is missing its output-gate projection.")
1085 if getattr(self.module, "allow_neg_eigval", False):
1086 raise NotImplementedError(
1087 "Kimi K3 Ulysses CP does not yet support allow_neg_eigval=True."
1088 )
1089
1090 _validate_layer_short_conv(self.module, "Ulysses")
1091
1092 def _pack_projected_inputs(
1093 self,
1094 query: torch.Tensor,
1095 key: torch.Tensor,
1096 value: torch.Tensor,
1097 gate: torch.Tensor,
1098 beta: torch.Tensor,
1099 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
1100 """Pack all projected paths into one sequence-to-head all-to-all."""
1101 if self.cp_size == 1:
1102 return query, key, value, gate, beta
1103
1104 base = self.module
1105 split_sizes = (
1106 _key_projection_dim(base) // self.cp_size,
1107 _key_projection_dim(base) // self.cp_size,
1108 _value_projection_dim(base) // self.cp_size,
1109 _num_value_heads(base) * base.head_k_dim // self.cp_size,
1110 _num_value_heads(base) // self.cp_size,
1111 )
1112 projected = (query, key, value, gate, beta)
1113 chunks = [torch.split(tensor, size, dim=-1) for tensor, size in zip(projected, split_sizes)]
1114 rank_major = [
1115 torch.cat(parts, dim=-1)
1116 for parts in zip(*chunks)
1117 ]
1118 packed = self._seq_to_head(torch.cat(rank_major, dim=-1).contiguous())
1119 return torch.split(packed, split_sizes, dim=-1)
1120
1121 def _local_short_conv(
1122 self,
1123 projected: torch.Tensor,
1124 convolution: nn.Conv1d,
1125 sequence_info: Optional[PackedSequenceInfo] = None,
1126 ) -> torch.Tensor:
1127 """Run one wrapped ShortConv on this rank's contiguous channel slice."""
1128 local_channels = projected.shape[-1]
1129 channel_start = self.cp_rank * local_channels
1130 weight = convolution.weight.narrow(0, channel_start, local_channels)
1131 bias = convolution.bias
1132 if bias is not None:
1133 bias = bias.narrow(0, channel_start, local_channels)
1134 if sequence_info is not None:
1135 return packed_causal_conv1d(
1136 projected, convolution, sequence_info, weight=weight, bias=bias,
1137 backend=self.backend,
1138 )
1139 if self.backend != "eager":
1140 return fused_causal_conv1d(
1141 projected, weight, bias, backend=self.backend,
1142 dilation=convolution.dilation[0],
1143 )
1144 output = F.conv1d( # pylint: disable=not-callable
1145 projected.transpose(1, 2),
1146 weight,
1147 bias,
1148 stride=convolution.stride,
1149 padding=convolution.padding,
1150 dilation=convolution.dilation,
1151 groups=local_channels,
1152 )
1153 output = output[:, :, : projected.shape[1]].transpose(1, 2)
1154 return F.silu(output)
1155
1156 def forward( # pylint: disable=arguments-renamed
1157 self,
1158 hidden_states: torch.Tensor,
1159 attention_mask: Optional[torch.Tensor] = None,
1160 past_key_values: Optional[Any] = None,
1161 use_cache: bool = False,
1162 output_attentions: bool = False,
1163 **kwargs: Any,
1164 ) -> tuple[torch.Tensor, None, None]:
1165 """Run the real Kimi layer using KDA and Ulysses CP.
1166
1167 Args:
1168 hidden_states: Local sequence shard [B,T,hidden_size].
1169 attention_mask: Optional dense mask; padding is unsupported.
1170 past_key_values: Recurrent inference cache, currently unsupported.
1171 use_cache: Must be False for this training adapter.
1172 output_attentions: Must be False; KDA does not materialize attention weights.
1173 kwargs: Optional global sample offsets or prebuilt packed_sequence_info.
1174 """
1175 if not self.module.training:
1176 raise NotImplementedError("Kimi K3 Ulysses CP currently supports training only.")
1177 if past_key_values is not None or use_cache:
1178 raise NotImplementedError("Kimi K3 Ulysses CP does not support recurrent cache.")
1179 if output_attentions:
1180 raise NotImplementedError("KDA does not materialize attention weights.")
1181 if attention_mask is not None and not bool(attention_mask.bool().all().item()):
1182 raise NotImplementedError("Padded KDA CP batches are not implemented.")
1183
1184 hidden_states, sequence_info, is_flat_input = _prepare_layer_input(
1185 hidden_states, kwargs, self.cp_rank, self.cp_size)
1186 exchanged_info = None if sequence_info is None else sequence_info.ulysses(self.cp_size, self.cp_rank)
1187 base = self.module
1188 batch_size, local_seq_len, _ = hidden_states.shape
1189 query = base.q_proj(hidden_states)
1190 key = base.k_proj(hidden_states)
1191 value = base.v_proj(hidden_states)
1192 gate = _forget_gate_projection(base, hidden_states)
1193 beta = base.b_proj(hidden_states)
1194 output_gate = _output_gate_projection(base, hidden_states).reshape(
1195 batch_size,
1196 local_seq_len,
1197 _num_value_heads(base),
1198 _value_head_dim(base),
1199 )
1200
1201 query, key, value, gate, beta = self._pack_projected_inputs(query, key, value, gate, beta)
1202 if _uses_short_conv(base):
1203 query = self._local_short_conv(query, base.q_conv1d, sequence_info=exchanged_info)
1204 key = self._local_short_conv(key, base.k_conv1d, sequence_info=exchanged_info)
1205 value = self._local_short_conv(value, base.v_conv1d, sequence_info=exchanged_info)
1206 else:
1207 query, key, value = (F.silu(tensor) for tensor in (query, key, value))
1208
1209 full_seq_len = query.shape[1]
1210 local_k_heads = base.num_heads // self.cp_size
1211 local_v_heads = _num_value_heads(base) // self.cp_size
1212 query = query.reshape(
1213 batch_size, full_seq_len, local_k_heads, base.head_k_dim
1214 )
1215 key = key.reshape(batch_size, full_seq_len, local_k_heads, base.head_k_dim)
1216 value = value.reshape(
1217 batch_size, full_seq_len, local_v_heads, _value_head_dim(base)
1218 )
1219 gate = gate.reshape(
1220 batch_size, full_seq_len, local_v_heads, base.head_k_dim
1221 )
1222 beta = beta.reshape(batch_size, full_seq_len, local_v_heads)
1223
1224 local_a_log = _slice_local_heads(
1225 base.A_log.reshape(_num_value_heads(base)),
1226 self.cp_rank,
1227 self.cp_size,
1228 )
1229 local_dt_bias = _slice_local_heads(
1230 base.dt_bias.reshape(_num_value_heads(base), base.head_k_dim),
1231 self.cp_rank,
1232 self.cp_size,
1233 )
1234 output = _run_local_kda(
1235 query,
1236 key,
1237 value,
1238 gate,
1239 beta,
1240 a_log=local_a_log,
1241 dt_bias=local_dt_bias,
1242 scale=None,
1243 lower_bound=self.lower_bound,
1244 chunk_size=self.chunk_size,
1245 safe_gate=self.safe_gate,
1246 backend=self.backend,
1247 sequence_info=exchanged_info,
1248 )
1249 output = self._head_to_seq(output)
1250 output = base.o_norm(output, output_gate)
1251 output = output.reshape(
1252 batch_size, local_seq_len, _value_projection_dim(base)
1253 )
1254 output = base.o_proj(output)
1255 return _format_layer_output(base, output.squeeze(0) if is_flat_input else output)
1256
1257
1258def _validate_cp_degrees(cp_size: int, ulysses_degree: int, allgather_group_size: int) -> None:
1259 """Validate both split extents before head division or process-group creation."""
1260 if (not isinstance(ulysses_degree, int) or isinstance(ulysses_degree, bool)
1261 or ulysses_degree < 1 or cp_size % ulysses_degree):
1262 raise ValueError("ulysses_degree must be a positive integer dividing the CP size.")
1263 if (not isinstance(allgather_group_size, int) or isinstance(allgather_group_size, bool)
1264 or allgather_group_size < 1 or (cp_size // ulysses_degree) % allgather_group_size):
1265 raise ValueError("allgather_group_size must be a positive integer dividing the state CP size.")
1266
1267
1268class KimiDeltaAttentionLayerCP(nn.Module):
1269 """Project local tokens, exchange convolution halos, and invoke the configured CP core."""
1270
1271 def __init__(
1272 self,
1273 module: nn.Module,
1274 device_mesh: DeviceMesh,
1275 *,
1276 chunk_size: int = 64,
1277 backend: str = "eager",
1278 ulysses_degree: int = 1,
1279 state_cp_method: str = "p2p",
1280 allgather_group_size: int = 1,
1281 ) -> None:
1282 """Keep layer ownership and full-CP halo metadata separate from the core."""
1283 super().__init__()
1284 lower_bound = _gate_lower_bound(module)
1285 if lower_bound is None:
1286 raise ValueError("Kimi K3 CP requires the lower-bounded gate parameterization.")
1287 self.cp_mesh = _ensure_1d(device_mesh)
1288 self.cp_rank, self.cp_size = self.cp_mesh.get_local_rank(), self.cp_mesh.size()
1289 self.module, self.backend = module, backend.lower()
1290 self._validate_module()
1291 _validate_cp_degrees(self.cp_size, ulysses_degree, allgather_group_size)
1292 if module.num_heads % ulysses_degree or _num_value_heads(module) % ulysses_degree:
1293 raise ValueError("KDA query and value heads must both be divisible by ulysses_degree.")
1294 self.core = KimiDeltaAttentionCoreCP(
1295 device_mesh, ulysses_degree=ulysses_degree, chunk_size=chunk_size,
1296 lower_bound=float(lower_bound), safe_gate=_uses_safe_gate(module), backend=backend,
1297 state_cp_method=state_cp_method, allgather_group_size=allgather_group_size)
1298
1299 def _validate_module(self) -> None:
1300 """Validate the KimiDeltaAttention contract used by state CP."""
1301 required_attributes = (
1302 "q_proj",
1303 "k_proj",
1304 "v_proj",
1305 "b_proj",
1306 "o_norm",
1307 "o_proj",
1308 "A_log",
1309 "dt_bias",
1310 "num_heads",
1311 "head_k_dim",
1312 )
1313 missing = [name for name in required_attributes if not hasattr(self.module, name)]
1314 if missing:
1315 raise TypeError(
1316 "Kimi K3 layer is missing required attributes: "
1317 + ", ".join(missing)
1318 )
1319 num_v_heads = _num_value_heads(self.module)
1320 if num_v_heads % self.module.num_heads:
1321 raise ValueError("KDA num_v_heads must be divisible by num_heads.")
1322 if self.backend in ("triton", "ascendc") and (
1323 self.module.head_k_dim != 128 or _value_head_dim(self.module) != 128
1324 ):
1325 raise NotImplementedError(
1326 "Kimi K3 CP currently requires key/value head dimensions 128."
1327 )
1328 if getattr(self.module, "allow_neg_eigval", False):
1329 raise NotImplementedError(
1330 "Kimi K3 CP does not support allow_neg_eigval=True."
1331 )
1332 if not (hasattr(self.module, "f_proj") or _is_official_kimi_layer(self.module)):
1333 raise TypeError("Kimi K3 layer is missing its forget-gate projection.")
1334 if not (
1335 hasattr(self.module, "g_proj")
1336 or all(hasattr(self.module, name) for name in ("g_a_proj", "g_b_proj"))
1337 ):
1338 raise TypeError("Kimi K3 layer is missing its output-gate projection.")
1339 _validate_layer_short_conv(self.module, "state")
1340
1341 def forward( # pylint: disable=arguments-renamed
1342 self,
1343 hidden_states: torch.Tensor,
1344 attention_mask: Optional[torch.Tensor] = None,
1345 past_key_values: Optional[Any] = None,
1346 use_cache: bool = False,
1347 output_attentions: bool = False,
1348 **kwargs: Any,
1349 ) -> tuple[torch.Tensor, None, None]:
1350 """Run projections, full-CP halos, the configured CP core, and output transforms.
1351
1352 Args:
1353 hidden_states: Local sequence shard [B,T,hidden_size].
1354 attention_mask: Optional dense mask; padding is unsupported.
1355 past_key_values: Recurrent inference cache, currently unsupported.
1356 use_cache: Must be False for this training adapter.
1357 output_attentions: Must be False; KDA does not materialize attention weights.
1358 kwargs: Optional global sample offsets or prebuilt packed_sequence_info.
1359 """
1360 if not self.module.training:
1361 raise NotImplementedError("Kimi K3 CP currently supports training only.")
1362 if past_key_values is not None or use_cache:
1363 raise NotImplementedError("Kimi K3 CP does not support recurrent cache.")
1364 if output_attentions:
1365 raise NotImplementedError("KDA does not materialize attention weights.")
1366 if attention_mask is not None and not bool(attention_mask.bool().all().item()):
1367 raise NotImplementedError("Padded KDA CP batches are not implemented.")
1368
1369 hidden_states, sequence_info, is_flat_input = _prepare_layer_input(
1370 hidden_states, kwargs, self.cp_rank, self.cp_size)
1371 base = self.module
1372 batch_size, local_seq_len, _ = hidden_states.shape
1373 query = base.q_proj(hidden_states)
1374 key = base.k_proj(hidden_states)
1375 value = base.v_proj(hidden_states)
1376 if _uses_short_conv(base):
1377 query, key, value = _causal_short_convs_with_cp_halo( # pylint: disable=unbalanced-tuple-unpacking
1378 (query, key, value),
1379 (base.q_conv1d, base.k_conv1d, base.v_conv1d),
1380 self.cp_mesh,
1381 self.cp_rank,
1382 self.cp_size,
1383 sequence_info=sequence_info,
1384 backend="ascendc" if self.backend == "ascendc" else "eager",
1385 )
1386 else:
1387 query, key, value = (F.silu(tensor) for tensor in (query, key, value))
1388
1389 query = query.reshape(
1390 batch_size,
1391 local_seq_len,
1392 base.num_heads,
1393 base.head_k_dim,
1394 )
1395 key = key.reshape(
1396 batch_size,
1397 local_seq_len,
1398 base.num_heads,
1399 base.head_k_dim,
1400 )
1401 value = value.reshape(
1402 batch_size,
1403 local_seq_len,
1404 _num_value_heads(base),
1405 _value_head_dim(base),
1406 )
1407 gate = _forget_gate_projection(base, hidden_states).reshape(
1408 batch_size,
1409 local_seq_len,
1410 _num_value_heads(base),
1411 base.head_k_dim,
1412 )
1413 beta = base.b_proj(hidden_states).reshape(
1414 batch_size,
1415 local_seq_len,
1416 _num_value_heads(base),
1417 )
1418 output_gate = _output_gate_projection(base, hidden_states).reshape(
1419 batch_size,
1420 local_seq_len,
1421 _num_value_heads(base),
1422 _value_head_dim(base),
1423 )
1424
1425 output = self.core(
1426 query,
1427 key,
1428 value,
1429 gate,
1430 beta,
1431 a_log=base.A_log,
1432 dt_bias=base.dt_bias,
1433 sequence_info=sequence_info,
1434 )
1435 output = base.o_norm(output, output_gate)
1436 output = output.reshape(
1437 batch_size, local_seq_len, _value_projection_dim(base)
1438 )
1439 output = base.o_proj(output)
1440 return _format_layer_output(base, output.squeeze(0) if is_flat_input else output)
1441
1442
1443class KimiDeltaAttentionCoreCP(nn.Module):
1444 """Compose optional Ulysses redistribution with local KDA and state exchange.
1445
1446 Inputs are projected Q/K/V/gate/beta after ShortConv. The layer adapter
1447 handles projections and full-sequence halo neighbors before calling here.
1448 """
1449
1450 def __init__(self, device_mesh: DeviceMesh, *, ulysses_degree: int = 1,
1451 chunk_size: int = 64, lower_bound: float = -5.0, safe_gate: bool = True,
1452 backend: str = "triton", state_cp_method: str = "p2p",
1453 allgather_group_size: int = 1) -> None:
1454 """Build all communication groups once, before forward or checkpoint replay."""
1455 super().__init__()
1456 device_mesh = _ensure_1d(device_mesh)
1457 if state_cp_method not in ("p2p", "allgather", "grouped_allgather_p2p", "recursive_doubling"):
1458 raise ValueError("KDA requires a supported state_cp_method.")
1459 if backend.lower() == "eager" and state_cp_method != "p2p":
1460 raise ValueError("Eager KDA state exchange supports P2P only.")
1461 _validate_cp_degrees(device_mesh.size(), ulysses_degree, allgather_group_size)
1462 if state_cp_method != "grouped_allgather_p2p" and allgather_group_size != 1:
1463 raise ValueError("allgather_group_size is only valid for grouped_allgather_p2p.")
1464 if lower_bound >= 0:
1465 raise ValueError("lower_bound must be negative.")
1466 self.cp_size = device_mesh.size()
1467 self.cp_rank = device_mesh.get_local_rank()
1468 self.ulysses_degree = ulysses_degree
1469 self.ulysses_mesh = None
1470 self.ulysses_rank = 0
1471 state_mesh = device_mesh
1472 if ulysses_degree > 1:
1473 hybrid = split_kda_mesh(device_mesh, ulysses_degree, ("kda_state", "kda_ulysses"))
1474 self.ulysses_mesh = hybrid["kda_ulysses"]
1475 self.ulysses_rank = self.ulysses_mesh.get_local_rank()
1476 state_mesh = hybrid["kda_state"]
1477 self.state = _KDAStateCP(
1478 state_mesh, chunk_size=chunk_size, lower_bound=lower_bound, safe_gate=safe_gate,
1479 backend=backend, state_cp_method=state_cp_method, allgather_group_size=allgather_group_size)
1480
1481 def forward(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
1482 gate: torch.Tensor, beta: torch.Tensor, *, a_log: torch.Tensor,
1483 dt_bias: torch.Tensor, scale: float | None = None,
1484 sequence_info: Optional[PackedSequenceInfo] = None) -> torch.Tensor:
1485 """Run state CP with optional token/head redistribution.
1486
1487 Args:
1488 query: Projected query tensor [B,T,H,K].
1489 key: Matching local key tensor.
1490 value: Projected value tensor [B,T,HV,V].
1491 gate: Per-token gate logits [B,T,HV,K].
1492 beta: Per-token beta logits [B,T,HV].
1493 a_log: Learned decay parameter with one entry per value head.
1494 dt_bias: Learned gate bias with HV times K entries.
1495 scale: Attention scaling factor; None selects the local default.
1496
1497 Returns:
1498 Output restored to this rank's original token/head layout.
1499 """
1500 is_flat_input = query.ndim == 3
1501 if is_flat_input:
1502 output = self.forward(*(tensor.unsqueeze(0) for tensor in (query, key, value, gate, beta)),
1503 a_log=a_log, dt_bias=dt_bias, scale=scale, sequence_info=sequence_info)
1504 return output.squeeze(0)
1505 if sequence_info is not None:
1506 if query.shape[0] != 1:
1507 raise ValueError("Packed KDA core requires batch dimension one.")
1508 resolve_packed_sequence({"packed_sequence_info": sequence_info}, query.shape[1],
1509 cp_rank=self.cp_rank, cp_size=self.cp_size)
1510 if self.ulysses_mesh is None:
1511 return self.state(query, key, value, gate, beta, a_log=a_log, dt_bias=dt_bias, scale=scale,
1512 sequence_info=sequence_info)
1513 _validate_projected_inputs(self.ulysses_degree, query, key, value, gate, beta, a_log, dt_bias)
1514 heads, key_dim = value.shape[2], query.shape[-1]
1515 inputs = self._exchange_inputs((query, key, value, gate, beta))
1516 local_a = _slice_local_heads(a_log.reshape(heads), self.ulysses_rank, self.ulysses_degree)
1517 local_bias = _slice_local_heads(dt_bias.reshape(heads, key_dim), self.ulysses_rank, self.ulysses_degree)
1518 exchanged_info = None if sequence_info is None else sequence_info.ulysses(
1519 self.ulysses_degree, self.ulysses_rank)
1520 output = self.state(*inputs, a_log=local_a, dt_bias=local_bias, scale=scale, sequence_info=exchanged_info)
1521 return _differentiable_all_to_all_shard(output, self.ulysses_mesh, split_dim=1, concat_dim=2)
1522
1523 def _exchange_inputs(self, inputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]:
1524 """Redistribute projected inputs using the established synchronous schedule."""
1525 return tuple(_differentiable_all_to_all_shard(tensor, self.ulysses_mesh, split_dim=2, concat_dim=1)
1526 for tensor in inputs)
1527
1528
1529class KimiDeltaAttentionLayerP2PCP(KimiDeltaAttentionLayerCP):
1530 """Preserve the public layer-P2P type while sharing layer execution.
1531
1532 Inherited defaults select P2P with no Ulysses. The subclass retains the old
1533 import and type name; new hybrid configurations use KimiDeltaAttentionLayerCP.
1534 """