1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""KDA state exchange and mesh construction for gathered or recursive scans."""
16from __future__ import annotations
17
18from typing import Any
19
20import torch
21import torch.distributed as dist
22
23from hyper_parallel.core.dtensor.device_mesh import DeviceMesh
24from hyper_parallel.distributed.context_parallel.kimi_delta_attention_rd import KDAStateRecursiveDoubling
25
26
27def _validate_mesh_order(mesh: DeviceMesh) -> None:
28 """Prevent collective slots from being mistaken for chronological positions."""
29 group_ranks = tuple(dist.get_process_group_ranks(mesh.get_group()))
30 if tuple(mesh.rank_list) != group_ranks:
31 raise ValueError("KDA mesh sequence order must match its communication group order.")
32
33
34def split_kda_mesh(mesh: DeviceMesh, degree: int, names: tuple[str, str]) -> DeviceMesh:
35 """Split a CP axis without losing the root rank map or sibling DP groups.
36
37 Args:
38 mesh: One-dimensional chronological CP mesh.
39 degree: Size of the inner split axis.
40 names: Names of the outer and inner axes.
41
42 Returns:
43 Two-dimensional submesh attached to the original root.
44 """
45 if mesh.ndim != 1 or not isinstance(degree, int) or isinstance(degree, bool):
46 raise ValueError("KDA mesh splitting requires a 1-D mesh and integer degree.")
47 if degree < 1 or mesh.size() % degree:
48 raise ValueError("KDA split degree must be positive and divide the CP size.")
49 _validate_mesh_order(mesh)
50 # Group creation uses WORLD; every rank must derive the same sibling groups.
51 root = mesh._get_root_mesh() # pylint: disable=protected-access
52 if root.size() != dist.get_world_size():
53 raise ValueError("Hybrid KDA requires a CP submesh retaining its WORLD root mesh.")
54 # Like the existing hybrid CP helper, preserve all sibling groups through the root layout.
55 return mesh._unflatten(0, (mesh.size() // degree, degree), names) # pylint: disable=protected-access
56
57
58def build_kda_state_cp(mesh: DeviceMesh, protocol: str, allgather_group_size: int) -> object | None:
59 """Construct a state-CP executor holding only communication metadata.
60
61 Args:
62 mesh: One-dimensional mesh ordered along the state sequence.
63 protocol: p2p, allgather, grouped_allgather_p2p, or recursive_doubling.
64 allgather_group_size: Consecutive ranks per local gather, only for the grouped protocol.
65
66 Returns:
67 State executor with an invocation-owned cache, or None for the P2P/local path.
68 """
69 if protocol not in ("p2p", "allgather", "grouped_allgather_p2p", "recursive_doubling"):
70 raise ValueError(f"Unknown KDA state_cp_method: {protocol!r}.")
71 if not isinstance(allgather_group_size, int) or isinstance(allgather_group_size, bool) or allgather_group_size < 1:
72 raise ValueError("KDA allgather_group_size must be a positive integer.")
73 if protocol != "grouped_allgather_p2p" and allgather_group_size != 1:
74 raise ValueError("KDA allgather_group_size is only valid for grouped_allgather_p2p.")
75 if protocol == "p2p":
76 return None
77 ranks = tuple(mesh.rank_list)
78 rank, size = mesh.get_local_rank(), mesh.size()
79 if protocol == "recursive_doubling":
80 return KDAStateRecursiveDoubling(mesh.get_group(), rank, ranks) if size > 1 else None
81 if protocol == "allgather":
82 _validate_mesh_order(mesh)
83 return KDAStateAllGather(mesh.get_group(), rank, size)
84 if size % allgather_group_size:
85 raise ValueError("KDA allgather_group_size must divide the state CP size.")
86 if allgather_group_size == 1:
87 return None
88 if allgather_group_size == size:
89 _validate_mesh_order(mesh)
90 return KDAStateAllGather(mesh.get_group(), rank, size)
91 split = split_kda_mesh(mesh, allgather_group_size, ("kda_outer", "kda_group"))
92 return KDAStateAllGatherP2P(
93 mesh.get_group(), split["kda_group"].get_group(), rank, ranks, allgather_group_size)
94
95
96def _validate_summary(state: torch.Tensor, matrix: torch.Tensor) -> None:
97 """Validate the dense affine contract before entering communication."""
98 if (state.ndim != 4 or state.shape[-2:] != (128, 128) or matrix.shape != state.shape
99 or state.shape[0] == 0 or state.shape[1] == 0):
100 raise ValueError("KDA state CP requires nonempty matching [B,H,128,128] summaries.")
101 if state.dtype != torch.float32 or matrix.dtype != torch.float32 or state.device != matrix.device:
102 raise ValueError("KDA state CP requires FP32 summaries on the same device.")
103 if state.device.type not in ("cpu", "npu"):
104 raise ValueError("KDA state CP supports CPU protocol checks and Ascend NPU execution only.")
105
106
107def merge_affine_summaries(
108 states: torch.Tensor, matrices: torch.Tensor, rank: int, forward: bool,
109) -> torch.Tensor:
110 """Merge the exclusive prefix or transposed suffix in chronological order.
111
112 Args:
113 states: Affine offsets shaped [P,B,H,128,128].
114 matrices: Matching transfer matrices, possibly views of packed storage.
115 rank: Chronological rank whose input state or final-state gradient is requested.
116 forward: Select forward prefix instead of backward suffix.
117
118 Returns:
119 Contiguous FP32 recurrent state or state gradient shaped [B,H,128,128].
120 """
121 if states.ndim != 5 or states.shape != matrices.shape or states.shape[0] == 0:
122 raise ValueError("KDA merge requires matching rank-stacked summaries.")
123 size, batch, heads = states.shape[:3]
124 _validate_summary(states[0], matrices[0])
125 if not 0 <= rank < size:
126 raise ValueError("KDA merge rank is outside the summary group.")
127 if any(t.stride(1) != heads * t.stride(2) or t.stride(-1) != 1 for t in (states, matrices)):
128 raise ValueError("KDA merge requires consecutive batch/head slabs and columns.")
129 if states.device.type != "npu":
130 raise ValueError("KDA fused state merge requires Ascend NPU tensors.")
131 count = rank if forward else size - rank - 1
132 output = states.new_empty((batch, heads, 128, 128))
133 if count == 0:
134 return output.zero_()
135 # Triton is optional for imports; production merging requires Ascend.
136 # pylint: disable=import-outside-toplevel
137 from hyper_parallel.components.functional._triton.kimi_delta_attention.cp_merge import (
138 kda_cached_affine_merge_kernel,
139 )
140 # pylint: enable=import-outside-toplevel
141
142 kda_cached_affine_merge_kernel[(2, batch * heads)](
143 output, states, matrices, count, rank,
144 state_rank_stride=states.stride(0), state_head_stride=states.stride(2), state_row_stride=states.stride(3),
145 matrix_rank_stride=matrices.stride(0), matrix_head_stride=matrices.stride(2),
146 matrix_row_stride=matrices.stride(3),
147 forward=forward, block_v=64, num_warps=2, num_stages=2)
148 return output
149
150
151def _gather_maps(state: torch.Tensor, matrix: torch.Tensor, group: Any, size: int) -> torch.Tensor:
152 """Pack validated local summaries for one invocation; the caller owns the gathered storage."""
153 packed = torch.cat((state, matrix), dim=-1)
154 output = packed.new_empty((size * packed.shape[0], *packed.shape[1:]))
155 dist.all_gather_into_tensor(output, packed, group=group)
156 return output.view(size, *packed.shape)
157
158
159class KDAStateAllGather:
160 """Gather S/M forward and G/M backward; only local M survives forward."""
161
162 def __init__(self, group: Any, rank: int, size: int) -> None:
163 """Bind precreated communication metadata, without tensor workspaces."""
164 if not isinstance(size, int) or isinstance(size, bool) or size < 1 or not 0 <= rank < size:
165 raise ValueError("AllGather requires a positive size and valid local rank.")
166 self.group, self.rank, self.size = group, rank, size
167
168 def gather_packed(self, packed: torch.Tensor) -> torch.Tensor:
169 """Collect native [1,H,128,256] summaries without a second packing copy.
170
171 The autograd caller owns the returned storage and its reverse protocol;
172 this metadata-only executor retains no per-invocation tensors.
173 """
174 output = packed.new_empty((self.size, *packed.shape[1:]))
175 dist.all_gather_into_tensor(output, packed, group=self.group)
176 return output
177
178 def forward(self, state: torch.Tensor, matrix: torch.Tensor) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]:
179 """Return the incoming state and this invocation's original local M.
180
181 Args:
182 state: Local FP32 offset S [B,H,128,128].
183 matrix: Local forward transfer M of the same shape.
184
185 Returns:
186 This rank's incoming recurrent state and a one-tensor local M cache.
187 """
188 _validate_summary(state, matrix)
189 maps = _gather_maps(state, matrix, self.group, self.size)
190 return merge_affine_summaries(maps[..., :128], maps[..., 128:], self.rank, True), (matrix,)
191
192 def backward(self, gradient: torch.Tensor, cache: tuple[torch.Tensor, ...]) -> torch.Tensor:
193 """Collect new G with the saved local M and recover the outgoing gradient.
194
195 Args:
196 gradient: Local FP32 offset G [B,H,128,128].
197 cache: One-tensor tuple containing this invocation's local forward M.
198
199 Returns:
200 This rank's outgoing recurrent-state gradient.
201 """
202 matrix, = cache
203 _validate_summary(gradient, matrix)
204 maps = _gather_maps(gradient, matrix, self.group, self.size)
205 return merge_affine_summaries(maps[..., :128], maps[..., 128:], self.rank, False)
206
207
208def _validate_scan_range(indices: range, size: int, forward: bool, prefix_count: int | None) -> None:
209 """Check the requested chronological interval and optional saved prefix."""
210 count = len(indices)
211 if prefix_count is not None and not 0 <= prefix_count <= count:
212 raise ValueError("The saved prefix must lie within the scan interval.")
213 if count:
214 endpoints_valid = all(0 <= index < size for index in (indices[0], indices[-1]))
215 if indices.step != (1 if forward else -1) or not endpoints_valid:
216 raise ValueError("KDA scans require an ordered interval inside the group.")
217
218
219def _scan_maps(maps: torch.Tensor, incoming: torch.Tensor,
220 indices: range, forward: bool, prefix_count: int | None = None,
221 ) -> tuple[torch.Tensor, torch.Tensor | None]:
222 """Scan validated group maps once, optionally returning an intermediate prefix."""
223 count = len(indices)
224 _validate_scan_range(indices, maps.shape[0], forward, prefix_count)
225 if not count:
226 return incoming, incoming if prefix_count == 0 else None
227 if incoming.device.type != "npu" or not incoming.is_contiguous():
228 raise ValueError("KDA fused scans require a contiguous Ascend input state.")
229 output = torch.empty_like(incoming)
230 prefix = None
231 if prefix_count is not None:
232 prefix = incoming if prefix_count == 0 else output if prefix_count == count else torch.empty_like(incoming)
233 save_prefix = prefix is not None and prefix_count not in (0, count)
234 # Keep the optional Triton dependency out of CPU-only model imports.
235 # pylint: disable=import-outside-toplevel
236 from hyper_parallel.components.functional._triton.kimi_delta_attention.cp_merge import kda_affine_scan_kernel
237 # pylint: enable=import-outside-toplevel
238
239 batch, heads = incoming.shape[:2]
240 kda_affine_scan_kernel[(2, batch * heads)](
241 output, prefix, maps, incoming, count, indices.start, prefix_count or 0,
242 rank_stride=maps.stride(0), head_stride=maps.stride(2), row_stride=maps.stride(3),
243 forward=forward, save_prefix=save_prefix, block_v=64, num_warps=2, num_stages=2)
244 return output, prefix
245
246
247class KDAStateAllGatherP2P:
248 """Use group-local gathers and an owner chain, retaining only each local M."""
249
250 def __init__(self, group: Any, intra_group: Any, rank: int,
251 ranks: tuple[int, ...], width: int) -> None:
252 """Bind ranks already validated against the process-group order by the factory."""
253 size = len(ranks)
254 if not isinstance(width, int) or isinstance(width, bool) or width < 2 or size % width:
255 raise ValueError("Grouped AllGather width must be >= 2 and divide the CP size.")
256 if not 0 <= rank < size or len(set(ranks)) != size:
257 raise ValueError("Grouped AllGather requires unique global ranks and a valid local rank.")
258 self.group, self.intra, self.rank, self.width = group, intra_group, rank, width
259 self.ranks = tuple(ranks)
260 self.block, self.local = divmod(rank, width)
261 self.blocks = size // width
262 self.owner = self.block * width + width - 1
263
264 def _receive(self, state: torch.Tensor, forward: bool) -> tuple[torch.Tensor, Any]:
265 """Post the owner receive before gathering local summaries."""
266 incoming = torch.zeros_like(state)
267 recv_handle = None
268 peer = self.owner - self.width if forward else self.owner + self.width
269 if self.rank == self.owner and 0 <= peer < len(self.ranks):
270 recv_handle = dist.irecv(incoming, src=self.ranks[peer], group=self.group)
271 return incoming, recv_handle
272
273 def forward(self, state: torch.Tensor, matrix: torch.Tensor) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]:
274 """Relay ordered group scans and restore this rank's incoming state.
275
276 Args:
277 state: Local FP32 offset S [B,H,128,128].
278 matrix: Local forward transfer M of the same shape.
279
280 Returns:
281 This rank's incoming recurrent state and a one-tensor local M cache.
282 """
283 _validate_summary(state, matrix)
284 incoming, recv_handle = self._receive(state, True)
285 maps = _gather_maps(state, matrix, self.intra, self.width)
286 send_handle, send_state = None, None
287 result = None
288 if self.rank == self.owner:
289 if recv_handle is not None:
290 recv_handle.wait()
291 if self.block + 1 < self.blocks:
292 send_state, result = _scan_maps(maps, incoming, range(self.width), True,
293 prefix_count=self.local)
294 send_handle = dist.isend(send_state, dst=self.ranks[self.owner + self.width], group=self.group)
295 if self.block > 0:
296 dist.broadcast(incoming, src=self.ranks[self.owner], group=self.intra)
297 if result is None:
298 result, _ = _scan_maps(maps, incoming, range(self.local), True)
299 if send_handle is not None:
300 send_handle.wait()
301 return result, (matrix,)
302
303 def backward(self, gradient: torch.Tensor, cache: tuple[torch.Tensor, ...]) -> torch.Tensor:
304 """Regather G/M and reverse the owner chain with ordered state feedback.
305
306 Args:
307 gradient: Local FP32 offset G [B,H,128,128].
308 cache: One-tensor tuple containing this invocation's local forward M.
309
310 Returns:
311 This rank's outgoing recurrent-state gradient.
312 """
313 matrix, = cache
314 _validate_summary(gradient, matrix)
315 incoming, recv_handle = self._receive(gradient, False)
316 maps = _gather_maps(gradient, matrix, self.intra, self.width)
317 send_handle, send_state = None, None
318 if self.rank == self.owner:
319 if recv_handle is not None:
320 recv_handle.wait()
321 if self.block > 0:
322 send_state, _ = _scan_maps(maps, incoming, range(self.width - 1, -1, -1), False)
323 send_handle = dist.isend(send_state, dst=self.ranks[self.owner - self.width], group=self.group)
324 if self.block + 1 < self.blocks:
325 dist.broadcast(incoming, src=self.ranks[self.owner], group=self.intra)
326 result, _ = _scan_maps(maps, incoming, range(self.width - 1, self.local, -1), False)
327 if send_handle is not None:
328 send_handle.wait()
329 return result