Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed / context_parallel / kimi_delta_attention_state.py: 37%

173 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""KDA state exchange and mesh construction for gathered or recursive scans.""" 

16from __future__ import annotations 

17 

18from typing import Any 

19 

20import torch 

21import torch.distributed as dist 

22 

23from hyper_parallel.core.dtensor.device_mesh import DeviceMesh 

24from hyper_parallel.distributed.context_parallel.kimi_delta_attention_rd import KDAStateRecursiveDoubling 

25 

26 

27def _validate_mesh_order(mesh: DeviceMesh) -> None: 

28 """Prevent collective slots from being mistaken for chronological positions.""" 

29 group_ranks = tuple(dist.get_process_group_ranks(mesh.get_group())) 

30 if tuple(mesh.rank_list) != group_ranks: 

31 raise ValueError("KDA mesh sequence order must match its communication group order.") 

32 

33 

34def split_kda_mesh(mesh: DeviceMesh, degree: int, names: tuple[str, str]) -> DeviceMesh: 

35 """Split a CP axis without losing the root rank map or sibling DP groups. 

36 

37 Args: 

38 mesh: One-dimensional chronological CP mesh. 

39 degree: Size of the inner split axis. 

40 names: Names of the outer and inner axes. 

41 

42 Returns: 

43 Two-dimensional submesh attached to the original root. 

44 """ 

45 if mesh.ndim != 1 or not isinstance(degree, int) or isinstance(degree, bool): 

46 raise ValueError("KDA mesh splitting requires a 1-D mesh and integer degree.") 

47 if degree < 1 or mesh.size() % degree: 

48 raise ValueError("KDA split degree must be positive and divide the CP size.") 

49 _validate_mesh_order(mesh) 

50 # Group creation uses WORLD; every rank must derive the same sibling groups. 

51 root = mesh._get_root_mesh() # pylint: disable=protected-access 

52 if root.size() != dist.get_world_size(): 

53 raise ValueError("Hybrid KDA requires a CP submesh retaining its WORLD root mesh.") 

54 # Like the existing hybrid CP helper, preserve all sibling groups through the root layout. 

55 return mesh._unflatten(0, (mesh.size() // degree, degree), names) # pylint: disable=protected-access 

56 

57 

58def build_kda_state_cp(mesh: DeviceMesh, protocol: str, allgather_group_size: int) -> object | None: 

59 """Construct a state-CP executor holding only communication metadata. 

60 

61 Args: 

62 mesh: One-dimensional mesh ordered along the state sequence. 

63 protocol: p2p, allgather, grouped_allgather_p2p, or recursive_doubling. 

64 allgather_group_size: Consecutive ranks per local gather, only for the grouped protocol. 

65 

66 Returns: 

67 State executor with an invocation-owned cache, or None for the P2P/local path. 

68 """ 

69 if protocol not in ("p2p", "allgather", "grouped_allgather_p2p", "recursive_doubling"): 

70 raise ValueError(f"Unknown KDA state_cp_method: {protocol!r}.") 

71 if not isinstance(allgather_group_size, int) or isinstance(allgather_group_size, bool) or allgather_group_size < 1: 

72 raise ValueError("KDA allgather_group_size must be a positive integer.") 

73 if protocol != "grouped_allgather_p2p" and allgather_group_size != 1: 

74 raise ValueError("KDA allgather_group_size is only valid for grouped_allgather_p2p.") 

75 if protocol == "p2p": 

76 return None 

77 ranks = tuple(mesh.rank_list) 

78 rank, size = mesh.get_local_rank(), mesh.size() 

79 if protocol == "recursive_doubling": 

80 return KDAStateRecursiveDoubling(mesh.get_group(), rank, ranks) if size > 1 else None 

81 if protocol == "allgather": 

82 _validate_mesh_order(mesh) 

83 return KDAStateAllGather(mesh.get_group(), rank, size) 

84 if size % allgather_group_size: 

85 raise ValueError("KDA allgather_group_size must divide the state CP size.") 

86 if allgather_group_size == 1: 

87 return None 

88 if allgather_group_size == size: 

89 _validate_mesh_order(mesh) 

90 return KDAStateAllGather(mesh.get_group(), rank, size) 

91 split = split_kda_mesh(mesh, allgather_group_size, ("kda_outer", "kda_group")) 

92 return KDAStateAllGatherP2P( 

93 mesh.get_group(), split["kda_group"].get_group(), rank, ranks, allgather_group_size) 

94 

95 

96def _validate_summary(state: torch.Tensor, matrix: torch.Tensor) -> None: 

97 """Validate the dense affine contract before entering communication.""" 

98 if (state.ndim != 4 or state.shape[-2:] != (128, 128) or matrix.shape != state.shape 

99 or state.shape[0] == 0 or state.shape[1] == 0): 

100 raise ValueError("KDA state CP requires nonempty matching [B,H,128,128] summaries.") 

101 if state.dtype != torch.float32 or matrix.dtype != torch.float32 or state.device != matrix.device: 

102 raise ValueError("KDA state CP requires FP32 summaries on the same device.") 

103 if state.device.type not in ("cpu", "npu"): 

104 raise ValueError("KDA state CP supports CPU protocol checks and Ascend NPU execution only.") 

105 

106 

107def merge_affine_summaries( 

108 states: torch.Tensor, matrices: torch.Tensor, rank: int, forward: bool, 

109) -> torch.Tensor: 

110 """Merge the exclusive prefix or transposed suffix in chronological order. 

111 

112 Args: 

113 states: Affine offsets shaped [P,B,H,128,128]. 

114 matrices: Matching transfer matrices, possibly views of packed storage. 

115 rank: Chronological rank whose input state or final-state gradient is requested. 

116 forward: Select forward prefix instead of backward suffix. 

117 

118 Returns: 

119 Contiguous FP32 recurrent state or state gradient shaped [B,H,128,128]. 

120 """ 

121 if states.ndim != 5 or states.shape != matrices.shape or states.shape[0] == 0: 

122 raise ValueError("KDA merge requires matching rank-stacked summaries.") 

123 size, batch, heads = states.shape[:3] 

124 _validate_summary(states[0], matrices[0]) 

125 if not 0 <= rank < size: 

126 raise ValueError("KDA merge rank is outside the summary group.") 

127 if any(t.stride(1) != heads * t.stride(2) or t.stride(-1) != 1 for t in (states, matrices)): 

128 raise ValueError("KDA merge requires consecutive batch/head slabs and columns.") 

129 if states.device.type != "npu": 

130 raise ValueError("KDA fused state merge requires Ascend NPU tensors.") 

131 count = rank if forward else size - rank - 1 

132 output = states.new_empty((batch, heads, 128, 128)) 

133 if count == 0: 

134 return output.zero_() 

135 # Triton is optional for imports; production merging requires Ascend. 

136 # pylint: disable=import-outside-toplevel 

137 from hyper_parallel.components.functional._triton.kimi_delta_attention.cp_merge import ( 

138 kda_cached_affine_merge_kernel, 

139 ) 

140 # pylint: enable=import-outside-toplevel 

141 

142 kda_cached_affine_merge_kernel[(2, batch * heads)]( 

143 output, states, matrices, count, rank, 

144 state_rank_stride=states.stride(0), state_head_stride=states.stride(2), state_row_stride=states.stride(3), 

145 matrix_rank_stride=matrices.stride(0), matrix_head_stride=matrices.stride(2), 

146 matrix_row_stride=matrices.stride(3), 

147 forward=forward, block_v=64, num_warps=2, num_stages=2) 

148 return output 

149 

150 

151def _gather_maps(state: torch.Tensor, matrix: torch.Tensor, group: Any, size: int) -> torch.Tensor: 

152 """Pack validated local summaries for one invocation; the caller owns the gathered storage.""" 

153 packed = torch.cat((state, matrix), dim=-1) 

154 output = packed.new_empty((size * packed.shape[0], *packed.shape[1:])) 

155 dist.all_gather_into_tensor(output, packed, group=group) 

156 return output.view(size, *packed.shape) 

157 

158 

159class KDAStateAllGather: 

160 """Gather S/M forward and G/M backward; only local M survives forward.""" 

161 

162 def __init__(self, group: Any, rank: int, size: int) -> None: 

163 """Bind precreated communication metadata, without tensor workspaces.""" 

164 if not isinstance(size, int) or isinstance(size, bool) or size < 1 or not 0 <= rank < size: 

165 raise ValueError("AllGather requires a positive size and valid local rank.") 

166 self.group, self.rank, self.size = group, rank, size 

167 

168 def gather_packed(self, packed: torch.Tensor) -> torch.Tensor: 

169 """Collect native [1,H,128,256] summaries without a second packing copy. 

170 

171 The autograd caller owns the returned storage and its reverse protocol; 

172 this metadata-only executor retains no per-invocation tensors. 

173 """ 

174 output = packed.new_empty((self.size, *packed.shape[1:])) 

175 dist.all_gather_into_tensor(output, packed, group=self.group) 

176 return output 

177 

178 def forward(self, state: torch.Tensor, matrix: torch.Tensor) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: 

179 """Return the incoming state and this invocation's original local M. 

180 

181 Args: 

182 state: Local FP32 offset S [B,H,128,128]. 

183 matrix: Local forward transfer M of the same shape. 

184 

185 Returns: 

186 This rank's incoming recurrent state and a one-tensor local M cache. 

187 """ 

188 _validate_summary(state, matrix) 

189 maps = _gather_maps(state, matrix, self.group, self.size) 

190 return merge_affine_summaries(maps[..., :128], maps[..., 128:], self.rank, True), (matrix,) 

191 

192 def backward(self, gradient: torch.Tensor, cache: tuple[torch.Tensor, ...]) -> torch.Tensor: 

193 """Collect new G with the saved local M and recover the outgoing gradient. 

194 

195 Args: 

196 gradient: Local FP32 offset G [B,H,128,128]. 

197 cache: One-tensor tuple containing this invocation's local forward M. 

198 

199 Returns: 

200 This rank's outgoing recurrent-state gradient. 

201 """ 

202 matrix, = cache 

203 _validate_summary(gradient, matrix) 

204 maps = _gather_maps(gradient, matrix, self.group, self.size) 

205 return merge_affine_summaries(maps[..., :128], maps[..., 128:], self.rank, False) 

206 

207 

208def _validate_scan_range(indices: range, size: int, forward: bool, prefix_count: int | None) -> None: 

209 """Check the requested chronological interval and optional saved prefix.""" 

210 count = len(indices) 

211 if prefix_count is not None and not 0 <= prefix_count <= count: 

212 raise ValueError("The saved prefix must lie within the scan interval.") 

213 if count: 

214 endpoints_valid = all(0 <= index < size for index in (indices[0], indices[-1])) 

215 if indices.step != (1 if forward else -1) or not endpoints_valid: 

216 raise ValueError("KDA scans require an ordered interval inside the group.") 

217 

218 

219def _scan_maps(maps: torch.Tensor, incoming: torch.Tensor, 

220 indices: range, forward: bool, prefix_count: int | None = None, 

221 ) -> tuple[torch.Tensor, torch.Tensor | None]: 

222 """Scan validated group maps once, optionally returning an intermediate prefix.""" 

223 count = len(indices) 

224 _validate_scan_range(indices, maps.shape[0], forward, prefix_count) 

225 if not count: 

226 return incoming, incoming if prefix_count == 0 else None 

227 if incoming.device.type != "npu" or not incoming.is_contiguous(): 

228 raise ValueError("KDA fused scans require a contiguous Ascend input state.") 

229 output = torch.empty_like(incoming) 

230 prefix = None 

231 if prefix_count is not None: 

232 prefix = incoming if prefix_count == 0 else output if prefix_count == count else torch.empty_like(incoming) 

233 save_prefix = prefix is not None and prefix_count not in (0, count) 

234 # Keep the optional Triton dependency out of CPU-only model imports. 

235 # pylint: disable=import-outside-toplevel 

236 from hyper_parallel.components.functional._triton.kimi_delta_attention.cp_merge import kda_affine_scan_kernel 

237 # pylint: enable=import-outside-toplevel 

238 

239 batch, heads = incoming.shape[:2] 

240 kda_affine_scan_kernel[(2, batch * heads)]( 

241 output, prefix, maps, incoming, count, indices.start, prefix_count or 0, 

242 rank_stride=maps.stride(0), head_stride=maps.stride(2), row_stride=maps.stride(3), 

243 forward=forward, save_prefix=save_prefix, block_v=64, num_warps=2, num_stages=2) 

244 return output, prefix 

245 

246 

247class KDAStateAllGatherP2P: 

248 """Use group-local gathers and an owner chain, retaining only each local M.""" 

249 

250 def __init__(self, group: Any, intra_group: Any, rank: int, 

251 ranks: tuple[int, ...], width: int) -> None: 

252 """Bind ranks already validated against the process-group order by the factory.""" 

253 size = len(ranks) 

254 if not isinstance(width, int) or isinstance(width, bool) or width < 2 or size % width: 

255 raise ValueError("Grouped AllGather width must be >= 2 and divide the CP size.") 

256 if not 0 <= rank < size or len(set(ranks)) != size: 

257 raise ValueError("Grouped AllGather requires unique global ranks and a valid local rank.") 

258 self.group, self.intra, self.rank, self.width = group, intra_group, rank, width 

259 self.ranks = tuple(ranks) 

260 self.block, self.local = divmod(rank, width) 

261 self.blocks = size // width 

262 self.owner = self.block * width + width - 1 

263 

264 def _receive(self, state: torch.Tensor, forward: bool) -> tuple[torch.Tensor, Any]: 

265 """Post the owner receive before gathering local summaries.""" 

266 incoming = torch.zeros_like(state) 

267 recv_handle = None 

268 peer = self.owner - self.width if forward else self.owner + self.width 

269 if self.rank == self.owner and 0 <= peer < len(self.ranks): 

270 recv_handle = dist.irecv(incoming, src=self.ranks[peer], group=self.group) 

271 return incoming, recv_handle 

272 

273 def forward(self, state: torch.Tensor, matrix: torch.Tensor) -> tuple[torch.Tensor, tuple[torch.Tensor, ...]]: 

274 """Relay ordered group scans and restore this rank's incoming state. 

275 

276 Args: 

277 state: Local FP32 offset S [B,H,128,128]. 

278 matrix: Local forward transfer M of the same shape. 

279 

280 Returns: 

281 This rank's incoming recurrent state and a one-tensor local M cache. 

282 """ 

283 _validate_summary(state, matrix) 

284 incoming, recv_handle = self._receive(state, True) 

285 maps = _gather_maps(state, matrix, self.intra, self.width) 

286 send_handle, send_state = None, None 

287 result = None 

288 if self.rank == self.owner: 

289 if recv_handle is not None: 

290 recv_handle.wait() 

291 if self.block + 1 < self.blocks: 

292 send_state, result = _scan_maps(maps, incoming, range(self.width), True, 

293 prefix_count=self.local) 

294 send_handle = dist.isend(send_state, dst=self.ranks[self.owner + self.width], group=self.group) 

295 if self.block > 0: 

296 dist.broadcast(incoming, src=self.ranks[self.owner], group=self.intra) 

297 if result is None: 

298 result, _ = _scan_maps(maps, incoming, range(self.local), True) 

299 if send_handle is not None: 

300 send_handle.wait() 

301 return result, (matrix,) 

302 

303 def backward(self, gradient: torch.Tensor, cache: tuple[torch.Tensor, ...]) -> torch.Tensor: 

304 """Regather G/M and reverse the owner chain with ordered state feedback. 

305 

306 Args: 

307 gradient: Local FP32 offset G [B,H,128,128]. 

308 cache: One-tensor tuple containing this invocation's local forward M. 

309 

310 Returns: 

311 This rank's outgoing recurrent-state gradient. 

312 """ 

313 matrix, = cache 

314 _validate_summary(gradient, matrix) 

315 incoming, recv_handle = self._receive(gradient, False) 

316 maps = _gather_maps(gradient, matrix, self.intra, self.width) 

317 send_handle, send_state = None, None 

318 if self.rank == self.owner: 

319 if recv_handle is not None: 

320 recv_handle.wait() 

321 if self.block > 0: 

322 send_state, _ = _scan_maps(maps, incoming, range(self.width - 1, -1, -1), False) 

323 send_handle = dist.isend(send_state, dst=self.ranks[self.owner - self.width], group=self.group) 

324 if self.block + 1 < self.blocks: 

325 dist.broadcast(incoming, src=self.ranks[self.owner], group=self.intra) 

326 result, _ = _scan_maps(maps, incoming, range(self.width - 1, self.local, -1), False) 

327 if send_handle is not None: 

328 send_handle.wait() 

329 return result