Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed / context_parallel / kimi_delta_attention.py: 23%

543 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""KDA context-parallel executors: state CP, Ulysses and their combinations.""" 

16from __future__ import annotations 

17 

18__all__ = [ 

19 "KimiDeltaAttentionCoreCP", 

20 "KimiDeltaAttentionLayerCP", 

21 "KimiDeltaAttentionLayerP2PCP", 

22 "KimiDeltaAttentionLayerUlyssesCP", 

23 "KimiDeltaAttentionP2PCP", 

24 "KimiDeltaAttentionUlyssesCP", 

25] 

26 

27from typing import Any, Optional 

28 

29import torch 

30import torch.distributed as dist 

31from torch import nn 

32from torch.nn import functional as F 

33 

34from hyper_parallel.components.functional.linear_attention_convolution import ( 

35 fused_causal_conv1d, 

36) 

37from hyper_parallel.components.functional.linear_attention_packed import ( 

38 PackedSequenceInfo, packed_causal_conv1d, resolve_packed_sequence, 

39) 

40 

41from hyper_parallel.core.context_parallel.context_parallel import _ensure_1d 

42from hyper_parallel.core.dtensor.device_mesh import DeviceMesh 

43from hyper_parallel.core.dtensor.dtensor import DTensor 

44from hyper_parallel.components.modules.kimi_delta_attention import ( 

45 chunk_kda, 

46 torch_apply_kda_state_summary, 

47 torch_kda_state_summary, 

48) 

49from hyper_parallel.core.utils import communication 

50from hyper_parallel.distributed.context_parallel.kimi_delta_attention_state import ( 

51 build_kda_state_cp, 

52 split_kda_mesh, 

53) 

54 

55 

56_KDA_BACKENDS = frozenset({"eager", "triton", "ascendc"}) 

57 

58 

59def _run_local_kda( 

60 query: torch.Tensor, 

61 key: torch.Tensor, 

62 value: torch.Tensor, 

63 gate: torch.Tensor, 

64 beta: torch.Tensor, 

65 *, 

66 a_log: torch.Tensor, 

67 dt_bias: torch.Tensor, 

68 scale: Optional[float], 

69 lower_bound: float, 

70 chunk_size: int, 

71 safe_gate: bool, 

72 backend: str, 

73 sequence_info: Optional[PackedSequenceInfo] = None, 

74) -> torch.Tensor: 

75 """Run the explicitly selected local KDA implementation.""" 

76 if backend in ("triton", "ascendc") and not safe_gate: 

77 raise RuntimeError("The fused KDA backend requires the lower-bounded gate.") 

78 cu_seqlens = None 

79 if sequence_info is not None: 

80 # Native APIs consume host metadata; do not materialize an NPU plan 

81 # only to copy its offsets back to the host at every layer. 

82 cu_seqlens = (sequence_info.local_offsets if backend == "ascendc" else 

83 sequence_info.materialize(query.device, chunk_size).cu_seqlens) 

84 output, _ = chunk_kda( 

85 query, 

86 key, 

87 value, 

88 gate, 

89 beta, 

90 a_log=a_log, 

91 dt_bias=dt_bias, 

92 scale=scale, 

93 lower_bound=lower_bound, 

94 chunk_size=chunk_size, 

95 initial_state=None, 

96 output_final_state=False, 

97 use_qk_l2norm_in_kernel=True, 

98 use_gate_in_kernel=True, 

99 use_beta_sigmoid_in_kernel=True, 

100 backend=backend, 

101 cu_seqlens=cu_seqlens, 

102 ) 

103 return output 

104 

105 

106def _is_official_kimi_layer(module: nn.Module) -> bool: 

107 """Whether ``module`` follows the official Kimi K3 projection contract.""" 

108 return hasattr(module, "f_a_proj") and hasattr(module, "f_b_proj") 

109 

110 

111def _num_value_heads(module: nn.Module) -> int: 

112 """Return the number of KDA value heads for either supported layer API.""" 

113 if hasattr(module, "num_v_heads"): 

114 return int(module.num_v_heads) 

115 return int(module.num_heads) 

116 

117 

118def _value_head_dim(module: nn.Module) -> int: 

119 """Return the KDA value head width for either supported layer API.""" 

120 if hasattr(module, "head_v_dim"): 

121 return int(module.head_v_dim) 

122 return int(module.head_dim) 

123 

124 

125def _key_projection_dim(module: nn.Module) -> int: 

126 """Return the flattened Q/K projection width.""" 

127 if hasattr(module, "key_dim"): 

128 return int(module.key_dim) 

129 return int(module.num_heads * module.head_k_dim) 

130 

131 

132def _value_projection_dim(module: nn.Module) -> int: 

133 """Return the flattened value projection width.""" 

134 if hasattr(module, "value_dim"): 

135 return int(module.value_dim) 

136 return int(_num_value_heads(module) * _value_head_dim(module)) 

137 

138 

139def _gate_lower_bound(module: nn.Module) -> Optional[float]: 

140 """Return the lower bound used by the safe KDA gate parameterization.""" 

141 return getattr(module, "lower_bound", getattr(module, "gate_lower_bound", None)) 

142 

143 

144def _uses_safe_gate(module: nn.Module) -> bool: 

145 """Return whether the layer enables the safe lower-bounded gate.""" 

146 return bool(getattr(module, "safe_gate", _gate_lower_bound(module) is not None)) 

147 

148 

149def _uses_short_conv(module: nn.Module) -> bool: 

150 """Return whether Q/K/V pass through the layer's ShortConvolution modules.""" 

151 return bool( 

152 getattr( 

153 module, 

154 "use_short_conv", 

155 all(hasattr(module, name) for name in ("q_conv1d", "k_conv1d", "v_conv1d")), 

156 ) 

157 ) 

158 

159 

160def _forget_gate_projection(module: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor: 

161 """Run the module-specific state-decay gate projection.""" 

162 if hasattr(module, "f_proj"): 

163 return module.f_proj(hidden_states) 

164 return module.f_b_proj(module.f_a_proj(hidden_states)) 

165 

166 

167def _output_gate_projection(module: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor: 

168 """Run the module-specific output-gate projection.""" 

169 if hasattr(module, "g_proj"): 

170 return module.g_proj(hidden_states) 

171 return module.g_b_proj(module.g_a_proj(hidden_states)) 

172 

173 

174def _format_layer_output(module: nn.Module, output: torch.Tensor): 

175 """Preserve the original layer's tensor or three-tuple output contract.""" 

176 if _is_official_kimi_layer(module): 

177 return output 

178 return output, None, None 

179 

180 

181def _prepare_layer_input( 

182 hidden_states: torch.Tensor, 

183 kwargs: dict[str, Any], 

184 cp_rank: int, 

185 cp_size: int, 

186) -> tuple[torch.Tensor, Optional[PackedSequenceInfo], bool]: 

187 """Normalize a local layer input and resolve this invocation's packed metadata.""" 

188 hidden_states = _local_cp_tensor(hidden_states) 

189 # Restore the caller's layout on return; offsets define document boundaries. 

190 is_flat_input = hidden_states.ndim == 2 

191 if is_flat_input: 

192 hidden_states = hidden_states.unsqueeze(0) 

193 sequence_info = resolve_packed_sequence( 

194 kwargs, hidden_states.shape[1], cp_rank=cp_rank, cp_size=cp_size) 

195 if sequence_info is not None and hidden_states.shape[0] != 1: 

196 raise ValueError("Packed KDA requires batch dimension one.") 

197 return hidden_states, sequence_info, is_flat_input 

198 

199 

200def _global_peer_rank(cp_mesh: DeviceMesh, local_rank: int) -> int: 

201 """Map one CP-local rank index to its global distributed rank.""" 

202 return int(cp_mesh.rank_list[local_rank]) 

203 

204 

205def _local_cp_tensor(tensor: torch.Tensor) -> torch.Tensor: 

206 """Return the local sequence shard of a CP input tensor.""" 

207 if isinstance(tensor, DTensor): 

208 return tensor.to_local() 

209 return tensor 

210 

211 

212def _all_to_all_previous_rank_halo( 

213 tail: torch.Tensor, 

214 cp_mesh: DeviceMesh, 

215 cp_rank: int, 

216 cp_size: int, 

217) -> torch.Tensor: 

218 """Send a causal-convolution halo only to the next CP rank.""" 

219 if cp_size == 1: 

220 return torch.zeros_like(tail) 

221 

222 cp_group = cp_mesh.get_group() 

223 group_ranks = tuple(int(rank) for rank in dist.get_process_group_ranks(cp_group)) 

224 rank_list = tuple(int(rank) for rank in cp_mesh.rank_list) 

225 rank_to_group_index = {rank: index for index, rank in enumerate(group_ranks)} 

226 halo_width = tail.shape[1] 

227 

228 input_splits = [0] * cp_size 

229 exchange_input = tail.permute(1, 0, 2).contiguous() 

230 if cp_rank < cp_size - 1: 

231 input_splits[rank_to_group_index[rank_list[cp_rank + 1]]] = halo_width 

232 else: 

233 exchange_input = exchange_input[:0] 

234 

235 output_splits = [0] * cp_size 

236 if cp_rank > 0: 

237 output_splits[rank_to_group_index[rank_list[cp_rank - 1]]] = halo_width 

238 

239 exchange_output = communication.differentiable_all_to_all_single( 

240 exchange_input, 

241 input_splits, 

242 output_splits, 

243 group=cp_group, 

244 ) 

245 if cp_rank == 0: 

246 return torch.zeros_like(tail) + exchange_output.sum().to(tail.dtype) * 0 

247 return exchange_output.permute(1, 0, 2).contiguous() 

248 

249 

250def _run_causal_short_conv( 

251 tensor: torch.Tensor, 

252 convolution: nn.Conv1d, 

253 halo: Optional[torch.Tensor] = None, 

254 sequence_info: Optional[PackedSequenceInfo] = None, 

255 backend: str = "eager", 

256) -> torch.Tensor: 

257 """Apply one local ShortConv, optionally with a preceding-rank halo.""" 

258 if backend != "eager": 

259 return fused_causal_conv1d( 

260 tensor, convolution.weight, convolution.bias, sequence_info, halo, 

261 backend=backend, dilation=convolution.dilation[0]) 

262 if sequence_info is not None: 

263 return packed_causal_conv1d(tensor, convolution, sequence_info, halo) 

264 if halo is None: 

265 conv_input = tensor.transpose(1, 2) 

266 padding = convolution.padding 

267 else: 

268 conv_input = torch.cat((halo, tensor), dim=1).transpose(1, 2) 

269 padding = 0 

270 output = F.conv1d( # pylint: disable=not-callable 

271 input=conv_input, 

272 weight=convolution.weight, 

273 bias=convolution.bias, 

274 stride=convolution.stride, 

275 padding=padding, 

276 dilation=convolution.dilation, 

277 groups=convolution.groups, 

278 ) 

279 if halo is None: 

280 output = output[:, :, : tensor.shape[1]] 

281 return F.silu(output).transpose(1, 2) 

282 

283 

284def _causal_short_convs_with_cp_halo( 

285 projected: tuple[torch.Tensor, torch.Tensor, torch.Tensor], 

286 convolutions: tuple[nn.Conv1d, nn.Conv1d, nn.Conv1d], 

287 cp_mesh: DeviceMesh, 

288 cp_rank: int, 

289 cp_size: int, 

290 sequence_info: Optional[PackedSequenceInfo] = None, 

291 backend: str = "eager", 

292) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 

293 """Run three causal depthwise ShortConvs with one packed halo exchange.""" 

294 halo_widths = { 

295 (conv.kernel_size[0] - 1) * conv.dilation[0] for conv in convolutions 

296 } 

297 if len(halo_widths) != 1: 

298 raise ValueError("KDA P2P requires Q/K/V ShortConv halo widths to match.") 

299 halo_width = halo_widths.pop() 

300 if halo_width == 0 or cp_size == 1: 

301 return tuple( 

302 _run_causal_short_conv(tensor, convolution, sequence_info=sequence_info, backend=backend) 

303 for tensor, convolution in zip(projected, convolutions) 

304 ) 

305 if projected[0].shape[1] < halo_width: 

306 raise ValueError( 

307 "KDA P2P ShortConv requires local_seq_len >= halo width, got " 

308 f"{projected[0].shape[1]} < {halo_width}." 

309 ) 

310 

311 channel_sizes = [tensor.shape[-1] for tensor in projected] 

312 packed_tail = torch.cat( 

313 [tensor[:, -halo_width:, :] for tensor in projected], 

314 dim=-1, 

315 ).contiguous() 

316 packed_halo = _all_to_all_previous_rank_halo( 

317 packed_tail, 

318 cp_mesh, 

319 cp_rank, 

320 cp_size, 

321 ) 

322 halos = torch.split(packed_halo, channel_sizes, dim=-1) 

323 return tuple( 

324 _run_causal_short_conv(tensor, convolution, halo, sequence_info=sequence_info, backend=backend) 

325 for tensor, halo, convolution in zip(projected, halos, convolutions) 

326 ) 

327 

328 

329def _slice_local_heads( 

330 tensor: torch.Tensor, 

331 cp_rank: int, 

332 cp_size: int, 

333) -> torch.Tensor: 

334 """Return the current CP rank's contiguous head slice.""" 

335 if tensor.shape[0] % cp_size != 0: 

336 raise ValueError( 

337 f"KDA head dimension {tensor.shape[0]} must be divisible by " 

338 f"cp_size {cp_size}." 

339 ) 

340 local_heads = tensor.shape[0] // cp_size 

341 return tensor.narrow(0, cp_rank * local_heads, local_heads) 

342 

343 

344def _differentiable_all_to_all_shard( 

345 tensor: torch.Tensor, 

346 device_mesh: DeviceMesh, 

347 *, 

348 split_dim: int, 

349 concat_dim: int, 

350) -> torch.Tensor: 

351 """Redistribute a local tensor from concat-dim shard to split-dim shard.""" 

352 split_count = device_mesh.size() 

353 if split_count == 1: 

354 return tensor 

355 

356 original_shape = tuple(tensor.shape) 

357 if original_shape[split_dim] % split_count != 0: 

358 raise ValueError( 

359 f"KDA all-to-all split dimension {split_dim} with size " 

360 f"{original_shape[split_dim]} must be divisible by cp_size " 

361 f"{split_count}." 

362 ) 

363 split_size = original_shape[split_dim] // split_count 

364 final_shape = list(original_shape) 

365 final_shape[split_dim] = split_size 

366 final_shape[concat_dim] *= split_count 

367 

368 reshape_dims = list(original_shape) 

369 reshape_dims[split_dim] = split_count 

370 reshape_dims.insert(split_dim + 1, split_size) 

371 permutation = list(range(len(reshape_dims))) 

372 permutation.remove(split_dim) 

373 permutation.insert(0, split_dim) 

374 all_to_all_input = tensor.reshape(reshape_dims).permute(permutation).contiguous() 

375 all_to_all_input = all_to_all_input.flatten(0, 1) 

376 

377 split_length = all_to_all_input.shape[0] // split_count 

378 splits = [split_length] * split_count 

379 output = communication.differentiable_all_to_all_single( 

380 all_to_all_input, 

381 splits, 

382 splits, 

383 group=device_mesh.get_group(), 

384 ) 

385 

386 output_reshape = list(output.shape) 

387 output_reshape[0] = split_count 

388 output_reshape.insert(1, output.shape[0] // split_count) 

389 permutation = list(range(len(output_reshape))) 

390 rank_dimension = permutation.pop(0) 

391 permutation.insert(concat_dim, rank_dimension) 

392 output = output.reshape(output_reshape).permute(permutation).contiguous() 

393 

394 output_shape = list(output.shape) 

395 output_shape[concat_dim] *= output_shape[concat_dim + 1] 

396 output_shape.pop(concat_dim + 1) 

397 return output.reshape(output_shape).view(final_shape) 

398 

399 

400class _RecvKDAInitialState(torch.autograd.Function): 

401 """Receive a KDA initial state and return its gradient to the sender.""" 

402 

403 @staticmethod 

404 def forward( # pylint: disable=arguments-differ 

405 ctx: Any, 

406 anchor: torch.Tensor, 

407 cp_group: Any, 

408 prev_rank: int, 

409 state_shape: tuple[int, ...], 

410 ) -> torch.Tensor: 

411 """Receive the initial recurrent state from the previous CP rank. 

412 

413 Args: 

414 ctx: Autograd context holding tensors and communication metadata. 

415 anchor: Tensor supplying the receive device and autograd dependency. 

416 cp_group: Precreated chronological CP process group. 

417 prev_rank: Global rank of the preceding sequence partition. 

418 state_shape: Shape of the FP32 recurrent state. 

419 """ 

420 state = torch.empty(state_shape, device=anchor.device, dtype=torch.float32) 

421 dist.recv(state, src=prev_rank, group=cp_group) 

422 ctx.cp_group = cp_group 

423 ctx.prev_rank = prev_rank 

424 return state 

425 

426 @staticmethod 

427 def backward( 

428 ctx: Any, 

429 grad_state: Optional[torch.Tensor], 

430 ) -> tuple[None, None, None, None]: 

431 """Send the accumulated initial-state gradient to the previous rank. 

432 

433 Args: 

434 ctx: Autograd context holding tensors and communication metadata. 

435 grad_state: Gradient of the received initial state. 

436 """ 

437 if grad_state is None: 

438 raise RuntimeError("KDA P2P backward is missing the initial-state gradient.") 

439 dist.send(grad_state.contiguous(), dst=ctx.prev_rank, group=ctx.cp_group) 

440 return None, None, None, None 

441 

442 

443class _SendKDAFinalState(torch.autograd.Function): 

444 """Send a KDA final state and receive its gradient during backward.""" 

445 

446 @staticmethod 

447 def forward( # pylint: disable=arguments-differ 

448 ctx: Any, 

449 final_state: torch.Tensor, 

450 cp_group: Any, 

451 next_rank: int, 

452 ) -> torch.Tensor: 

453 """Send the final recurrent state to the next CP rank. 

454 

455 Args: 

456 ctx: Autograd context holding tensors and communication metadata. 

457 final_state: Outgoing recurrent state. 

458 cp_group: Precreated chronological CP process group. 

459 next_rank: Global rank of the following sequence partition. 

460 """ 

461 dist.send(final_state.contiguous(), dst=next_rank, group=cp_group) 

462 ctx.cp_group = cp_group 

463 ctx.next_rank = next_rank 

464 ctx.state_shape = tuple(final_state.shape) 

465 ctx.state_dtype = final_state.dtype 

466 return final_state.new_zeros(()) 

467 

468 @staticmethod 

469 def backward( 

470 ctx: Any, 

471 grad_token: torch.Tensor, 

472 ) -> tuple[torch.Tensor, None, None]: 

473 """Receive the final-state gradient from the next rank. 

474 

475 Args: 

476 ctx: Autograd context holding tensors and communication metadata. 

477 grad_token: Gradient token supplying the receive device. 

478 """ 

479 grad_state = torch.empty( 

480 ctx.state_shape, 

481 device=grad_token.device, 

482 dtype=ctx.state_dtype, 

483 ) 

484 dist.recv(grad_state, src=ctx.next_rank, group=ctx.cp_group) 

485 return grad_state, None, None 

486 

487 

488def _eager_kda_packed_p2p( 

489 query: torch.Tensor, 

490 key: torch.Tensor, 

491 value: torch.Tensor, 

492 gate: torch.Tensor, 

493 beta: torch.Tensor, 

494 *, 

495 a_log: torch.Tensor, 

496 dt_bias: torch.Tensor, 

497 scale: Optional[float], 

498 lower_bound: float, 

499 chunk_size: int, 

500 cp_group: Any, 

501 prev_rank: int, 

502 next_rank: int, 

503 cp_rank: int, 

504 cp_size: int, 

505 sequence_info: Optional[PackedSequenceInfo] = None, 

506) -> torch.Tensor: 

507 """Run native per-document recurrence with differentiable neighbor states.""" 

508 # Portable semantic reference: native fragment recurrence, fixed sends. 

509 incoming = None 

510 if cp_rank > 0: 

511 shape = (1, value.shape[2], query.shape[-1], value.shape[-1]) 

512 incoming = _RecvKDAInitialState.apply(query, cp_group, prev_rank, shape) 

513 output, finals = chunk_kda( 

514 query, key, value, gate, beta, a_log=a_log, dt_bias=dt_bias, scale=scale, 

515 initial_state=sequence_info.first_state(incoming), output_final_state=True, 

516 use_qk_l2norm_in_kernel=True, use_gate_in_kernel=True, use_beta_sigmoid_in_kernel=True, 

517 lower_bound=lower_bound, chunk_size=chunk_size, backend="eager", 

518 cu_seqlens=sequence_info.local_offsets, 

519 ) 

520 if incoming is not None: 

521 output = output + incoming.sum().to(output.dtype) * 0 

522 if cp_rank < cp_size - 1: 

523 token = _SendKDAFinalState.apply(finals[-1:].contiguous(), cp_group, next_rank) 

524 output = output + token.to(output.dtype) * 0 

525 return output 

526 

527 

528def _eager_kda_state_p2p( 

529 query: torch.Tensor, 

530 key: torch.Tensor, 

531 value: torch.Tensor, 

532 gate: torch.Tensor, 

533 beta: torch.Tensor, 

534 *, 

535 a_log: torch.Tensor, 

536 dt_bias: torch.Tensor, 

537 scale: Optional[float], 

538 lower_bound: float, 

539 chunk_size: int, 

540 cp_group: Any, 

541 prev_rank: int, 

542 next_rank: int, 

543 cp_rank: int, 

544 cp_size: int, 

545 sequence_info: Optional[PackedSequenceInfo] = None, 

546) -> torch.Tensor: 

547 """Run the differentiable Torch reference for affine state-P2P KDA.""" 

548 if sequence_info is not None: 

549 return _eager_kda_packed_p2p( 

550 query, key, value, gate, beta, a_log=a_log, dt_bias=dt_bias, scale=scale, 

551 lower_bound=lower_bound, chunk_size=chunk_size, cp_group=cp_group, 

552 prev_rank=prev_rank, next_rank=next_rank, cp_rank=cp_rank, cp_size=cp_size, 

553 sequence_info=sequence_info, 

554 ) 

555 if cp_size == 1: 

556 return _run_local_kda( 

557 query, 

558 key, 

559 value, 

560 gate, 

561 beta, 

562 a_log=a_log, 

563 dt_bias=dt_bias, 

564 scale=scale, 

565 lower_bound=lower_bound, 

566 chunk_size=chunk_size, 

567 safe_gate=True, 

568 backend="eager", 

569 ) 

570 

571 state_ext = None 

572 transition = None 

573 if cp_rank < cp_size - 1: 

574 state_ext, transition = torch_kda_state_summary( 

575 query, 

576 key, 

577 value, 

578 gate, 

579 beta, 

580 a_log=a_log, 

581 dt_bias=dt_bias, 

582 scale=scale, 

583 use_qk_l2norm_in_kernel=True, 

584 use_gate_in_kernel=True, 

585 use_beta_sigmoid_in_kernel=True, 

586 lower_bound=lower_bound, 

587 chunk_size=chunk_size, 

588 ) 

589 

590 initial_state = None 

591 if cp_rank > 0: 

592 state_shape = ( 

593 query.shape[0], 

594 value.shape[2], 

595 query.shape[-1], 

596 value.shape[-1], 

597 ) 

598 initial_state = _RecvKDAInitialState.apply( 

599 query, 

600 cp_group, 

601 prev_rank, 

602 state_shape, 

603 ) 

604 

605 send_token = None 

606 if cp_rank < cp_size - 1: 

607 final_state = torch_apply_kda_state_summary(state_ext, transition, initial_state) 

608 send_token = _SendKDAFinalState.apply(final_state, cp_group, next_rank) 

609 

610 output, _ = chunk_kda( 

611 query, 

612 key, 

613 value, 

614 gate, 

615 beta, 

616 a_log=a_log, 

617 dt_bias=dt_bias, 

618 scale=scale, 

619 initial_state=initial_state, 

620 output_final_state=False, 

621 use_qk_l2norm_in_kernel=True, 

622 use_gate_in_kernel=True, 

623 use_beta_sigmoid_in_kernel=True, 

624 lower_bound=lower_bound, 

625 chunk_size=chunk_size, 

626 backend="eager", 

627 ) 

628 if send_token is not None: 

629 output = output + send_token.to(output.dtype) * 0 

630 return output 

631 

632 

633def _validate_projected_inputs( 

634 ulysses_degree: int, 

635 query: torch.Tensor, 

636 key: torch.Tensor, 

637 value: torch.Tensor, 

638 gate: torch.Tensor, 

639 beta: torch.Tensor, 

640 a_log: torch.Tensor, 

641 dt_bias: torch.Tensor, 

642) -> None: 

643 """Validate the projected-tensor contract and Ulysses divisibility.""" 

644 if (query.dim(), key.dim(), value.dim(), gate.dim()) != (4, 4, 4, 4): 

645 raise ValueError("query, key, value, and gate must be rank-4 tensors.") 

646 if beta.dim() != 3: 

647 raise ValueError("beta must be a rank-3 tensor.") 

648 if query.shape != key.shape: 

649 raise ValueError("query and key must have identical shapes.") 

650 

651 batch_size, local_seq_len, num_k_heads, k_head_dim = query.shape 

652 expected_prefix = (batch_size, local_seq_len) 

653 if value.shape[:2] != expected_prefix: 

654 raise ValueError("value batch and sequence dimensions must match query.") 

655 num_v_heads = value.shape[2] 

656 if gate.shape != (batch_size, local_seq_len, num_v_heads, k_head_dim): 

657 raise ValueError( 

658 "gate must have shape [B, S_local, num_v_heads, k_head_dim]." 

659 ) 

660 if beta.shape != (batch_size, local_seq_len, num_v_heads): 

661 raise ValueError("beta must have shape [B, S_local, num_v_heads].") 

662 if num_v_heads % num_k_heads != 0: 

663 raise ValueError("num_v_heads must be divisible by num_k_heads.") 

664 if num_k_heads % ulysses_degree != 0: 

665 raise ValueError( 

666 f"KDA num_k_heads ({num_k_heads}) must be divisible by " 

667 f"cp_size ({ulysses_degree}) for Ulysses CP." 

668 ) 

669 if num_v_heads % ulysses_degree != 0: 

670 raise ValueError( 

671 f"KDA num_v_heads ({num_v_heads}) must be divisible by " 

672 f"cp_size ({ulysses_degree}) for Ulysses CP." 

673 ) 

674 if a_log.numel() != num_v_heads: 

675 raise ValueError( 

676 f"a_log must contain {num_v_heads} elements, got {a_log.numel()}." 

677 ) 

678 if dt_bias.numel() != num_v_heads * k_head_dim: 

679 raise ValueError( 

680 f"dt_bias must contain {num_v_heads * k_head_dim} elements, " 

681 f"got {dt_bias.numel()}." 

682 ) 

683 

684 

685class KimiDeltaAttentionUlyssesCP(nn.Module): 

686 """Run KDA on sequence-sharded projected tensors with Ulysses all-to-all. 

687 

688 The input and output layout is token-first with a local sequence shard: 

689 

690 ``[B, S/P, H, D] -> [B, S, H/P, D] -> [B, S/P, H, D]``. 

691 

692 This first-stage interface intentionally starts after projections and 

693 ShortConv and ends before the output norm/gate/projection. It is therefore 

694 independent of one particular Kimi K3 model class. Training uses a zero 

695 initial recurrent state and does not materialize a final state. 

696 """ 

697 

698 def __init__( 

699 self, 

700 device_mesh: DeviceMesh, 

701 *, 

702 chunk_size: int = 64, 

703 lower_bound: float = -5.0, 

704 safe_gate: bool = True, 

705 backend: str = "eager", 

706 ) -> None: 

707 """Initialize the KDA Ulysses executor for one 1-D CP mesh.""" 

708 super().__init__() 

709 if chunk_size <= 0: 

710 raise ValueError(f"chunk_size must be positive, got {chunk_size}.") 

711 if lower_bound >= 0: 

712 raise ValueError(f"lower_bound must be negative, got {lower_bound}.") 

713 backend = backend.lower() 

714 if backend not in _KDA_BACKENDS: 

715 raise ValueError( 

716 f"unsupported KDA backend {backend!r}; " 

717 f"expected one of {sorted(_KDA_BACKENDS)}." 

718 ) 

719 self.cp_mesh = _ensure_1d(device_mesh) 

720 self.cp_size = self.cp_mesh.size() 

721 self.cp_rank = self.cp_mesh.get_local_rank() 

722 self.chunk_size = chunk_size 

723 self.lower_bound = lower_bound 

724 self.safe_gate = safe_gate 

725 self.backend = backend 

726 self.seq_dim = 1 

727 self.head_dim = 2 

728 

729 def _validate_inputs( 

730 self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, 

731 gate: torch.Tensor, beta: torch.Tensor, a_log: torch.Tensor, dt_bias: torch.Tensor, 

732 ) -> None: 

733 """Validate the projected contract for this Ulysses degree.""" 

734 _validate_projected_inputs(self.cp_size, query, key, value, gate, beta, a_log, dt_bias) 

735 

736 def _seq_to_head(self, tensor: torch.Tensor) -> torch.Tensor: 

737 """Redistribute a local sequence shard into a local head shard.""" 

738 return _differentiable_all_to_all_shard( 

739 tensor, 

740 self.cp_mesh, 

741 split_dim=self.head_dim, 

742 concat_dim=self.seq_dim, 

743 ) 

744 

745 def _head_to_seq(self, tensor: torch.Tensor) -> torch.Tensor: 

746 """Redistribute a local head shard back into a local sequence shard.""" 

747 return _differentiable_all_to_all_shard( 

748 tensor, 

749 self.cp_mesh, 

750 split_dim=self.seq_dim, 

751 concat_dim=self.head_dim, 

752 ) 

753 

754 def forward( 

755 self, 

756 query: torch.Tensor, 

757 key: torch.Tensor, 

758 value: torch.Tensor, 

759 gate: torch.Tensor, 

760 beta: torch.Tensor, 

761 *, 

762 a_log: torch.Tensor, 

763 dt_bias: torch.Tensor, 

764 scale: Optional[float] = None, 

765 sequence_info: Optional[PackedSequenceInfo] = None, 

766 ) -> torch.Tensor: 

767 """Run chunkwise KDA using full sequence and local heads. 

768 

769 Args: 

770 query: Local query tensor [B,T,H,K]. 

771 key: Matching local key tensor. 

772 value: Local value tensor [B,T,HV,V]. 

773 gate: Per-token gate logits [B,T,HV,K]. 

774 beta: Per-token beta logits [B,T,HV]. 

775 a_log: Decay parameter with one value per value head. 

776 dt_bias: Learned bias with one value per gate channel. 

777 scale: Optional attention scaling factor. 

778 """ 

779 is_flat_input = query.ndim == 3 

780 if is_flat_input: 

781 output = self.forward(*(tensor.unsqueeze(0) for tensor in (query, key, value, gate, beta)), 

782 a_log=a_log, dt_bias=dt_bias, scale=scale, sequence_info=sequence_info) 

783 return output.squeeze(0) 

784 self._validate_inputs(query, key, value, gate, beta, a_log, dt_bias) 

785 exchanged_info = None if sequence_info is None else sequence_info.ulysses(self.cp_size, self.cp_rank) 

786 num_v_heads = value.shape[2] 

787 k_head_dim = query.shape[-1] 

788 

789 query = self._seq_to_head(query) 

790 key = self._seq_to_head(key) 

791 value = self._seq_to_head(value) 

792 gate = self._seq_to_head(gate) 

793 beta = self._seq_to_head(beta) 

794 

795 local_a_log = _slice_local_heads( 

796 a_log.reshape(num_v_heads), 

797 self.cp_rank, 

798 self.cp_size, 

799 ) 

800 local_dt_bias = _slice_local_heads( 

801 dt_bias.reshape(num_v_heads, k_head_dim), 

802 self.cp_rank, 

803 self.cp_size, 

804 ) 

805 output = _run_local_kda( 

806 query, 

807 key, 

808 value, 

809 gate, 

810 beta, 

811 a_log=local_a_log, 

812 dt_bias=local_dt_bias, 

813 scale=scale, 

814 lower_bound=self.lower_bound, 

815 chunk_size=self.chunk_size, 

816 safe_gate=self.safe_gate, 

817 backend=self.backend, 

818 sequence_info=exchanged_info, 

819 ) 

820 return self._head_to_seq(output) 

821 

822 

823class _KDAStateCP(nn.Module): 

824 """Run local KDA with the configured state exchange.""" 

825 

826 def __init__( 

827 self, 

828 device_mesh: DeviceMesh, 

829 *, 

830 chunk_size: int = 64, 

831 lower_bound: float = -5.0, 

832 safe_gate: bool = True, 

833 backend: str = "eager", 

834 state_cp_method: str = "p2p", 

835 allgather_group_size: int = 1, 

836 ) -> None: 

837 """Initialize local KDA and its state exchange on one chronological axis.""" 

838 super().__init__() 

839 if chunk_size <= 0: 

840 raise ValueError(f"chunk_size must be positive, got {chunk_size}.") 

841 backend = backend.lower() 

842 if backend not in _KDA_BACKENDS: 

843 raise ValueError( 

844 f"unsupported KDA backend {backend!r}; " 

845 f"expected one of {sorted(_KDA_BACKENDS)}." 

846 ) 

847 self.cp_mesh = _ensure_1d(device_mesh) 

848 self.cp_size = self.cp_mesh.size() 

849 self.cp_rank = self.cp_mesh.get_local_rank() 

850 self.cp_group = self.cp_mesh.get_group() 

851 if backend == "ascendc" and (chunk_size != 64 or not safe_gate): 

852 raise ValueError("Ascend C KDA requires safe_gate=True and chunk_size=64.") 

853 if backend == "eager" and state_cp_method != "p2p": 

854 raise ValueError("Eager KDA state exchange supports P2P only.") 

855 self.state_cp = build_kda_state_cp(self.cp_mesh, state_cp_method, allgather_group_size) 

856 self.prev_rank = _global_peer_rank( 

857 self.cp_mesh, 

858 max(self.cp_rank - 1, 0), 

859 ) 

860 self.next_rank = _global_peer_rank( 

861 self.cp_mesh, 

862 min(self.cp_rank + 1, self.cp_size - 1), 

863 ) 

864 self.chunk_size = chunk_size 

865 self.lower_bound = lower_bound 

866 self.safe_gate = safe_gate 

867 self.backend = backend 

868 

869 def forward( 

870 self, 

871 query: torch.Tensor, 

872 key: torch.Tensor, 

873 value: torch.Tensor, 

874 gate: torch.Tensor, 

875 beta: torch.Tensor, 

876 *, 

877 a_log: torch.Tensor, 

878 dt_bias: torch.Tensor, 

879 scale: Optional[float] = None, 

880 sequence_info: Optional[PackedSequenceInfo] = None, 

881 ) -> torch.Tensor: 

882 """Run one local KDA segment and propagate its recurrent state. 

883 

884 Args: 

885 query: Local query tensor [B,T,H,K]. 

886 key: Matching local key tensor. 

887 value: Local value tensor [B,T,HV,V]. 

888 gate: Per-token gate logits [B,T,HV,K]. 

889 beta: Per-token beta logits [B,T,HV]. 

890 a_log: Decay parameter with one value per value head. 

891 dt_bias: Learned bias with one value per gate channel. 

892 scale: Optional attention scaling factor. 

893 """ 

894 if self.cp_size == 1: 

895 return _run_local_kda( 

896 query, 

897 key, 

898 value, 

899 gate, 

900 beta, 

901 a_log=a_log, 

902 dt_bias=dt_bias, 

903 scale=scale, 

904 lower_bound=self.lower_bound, 

905 chunk_size=self.chunk_size, 

906 safe_gate=self.safe_gate, 

907 backend=self.backend, 

908 sequence_info=sequence_info, 

909 ) 

910 if self.backend == "eager": 

911 return _eager_kda_state_p2p( 

912 query, 

913 key, 

914 value, 

915 gate, 

916 beta, 

917 a_log=a_log, 

918 dt_bias=dt_bias, 

919 scale=scale, 

920 lower_bound=self.lower_bound, 

921 chunk_size=self.chunk_size, 

922 cp_group=self.cp_group, 

923 prev_rank=self.prev_rank, 

924 next_rank=self.next_rank, 

925 cp_rank=self.cp_rank, 

926 cp_size=self.cp_size, 

927 sequence_info=sequence_info, 

928 ) 

929 

930 if self.backend == "ascendc": 

931 # The native package remains optional; loading it belongs to the selected backend. 

932 from hyper_parallel.components.functional.kimi_delta_attention_ascendc import ( # pylint: disable=import-outside-toplevel 

933 ascendc_state_kda, 

934 ) 

935 return ascendc_state_kda( 

936 query, key, value, gate, beta, a_log=a_log, dt_bias=dt_bias, scale=scale, 

937 lower_bound=self.lower_bound, state_cp=self.state_cp, cp_mesh=self.cp_mesh, 

938 sequence_info=sequence_info) 

939 

940 from hyper_parallel.components.functional.kimi_delta_attention import ( # pylint: disable=import-outside-toplevel 

941 fused_chunk_kda_p2p, 

942 ) 

943 return fused_chunk_kda_p2p( 

944 query, 

945 key, 

946 value, 

947 gate, 

948 beta, 

949 a_log=a_log.float(), 

950 dt_bias=dt_bias.float(), 

951 cp_group=self.cp_group, 

952 prev_rank=self.prev_rank, 

953 next_rank=self.next_rank, 

954 cp_rank=self.cp_rank, 

955 cp_size=self.cp_size, 

956 scale=scale, 

957 lower_bound=self.lower_bound, 

958 chunk_size=self.chunk_size, 

959 safe_gate=self.safe_gate, 

960 state_cp=self.state_cp, 

961 sequence_info=sequence_info, 

962 ) 

963 

964 

965class KimiDeltaAttentionP2PCP(_KDAStateCP): 

966 """Preserve the public projected-P2P type while sharing state execution. 

967 

968 Inherited defaults select P2P. Keeping a named subclass preserves the old 

969 import and type name without duplicating the common implementation. 

970 New hybrid configurations use KimiDeltaAttentionCoreCP. 

971 """ 

972 

973 

974def _validate_layer_short_conv(module: nn.Module, mode: str) -> None: 

975 """Check the shared depthwise-convolution contract before building execution.""" 

976 if _uses_short_conv(module): 

977 for name in ("q_conv1d", "k_conv1d", "v_conv1d"): 

978 if not hasattr(module, name): 

979 raise TypeError(f"Kimi K3 layer is missing {name}.") 

980 convolution = getattr(module, name) 

981 if convolution.stride != (1,) or convolution.dilation != (1,): 

982 raise ValueError( 

983 f"Kimi K3 {mode} CP supports ShortConv stride=1 and " 

984 "dilation=1 only." 

985 ) 

986 if convolution.groups != convolution.in_channels or convolution.out_channels != convolution.in_channels: 

987 raise ValueError(f"Kimi K3 {mode} CP expects depthwise ShortConv with equal input/output channels.") 

988 if convolution.padding != (convolution.kernel_size[0] - 1,) or convolution.padding_mode != "zeros": 

989 raise ValueError(f"Kimi K3 {mode} CP expects causal zero-padded ShortConv.") 

990 if getattr(convolution, "activation", None) not in ("silu", "swish"): 

991 raise ValueError( 

992 f"Kimi K3 {mode} CP expects ShortConv SiLU activation." 

993 ) 

994 

995 

996class KimiDeltaAttentionLayerUlyssesCP(KimiDeltaAttentionUlyssesCP): 

997 """Wrap a training-time Kimi K3 attention layer with Ulysses CP. 

998 

999 The wrapped module keeps ownership of all model parameters. Its local 

1000 sequence shard is projected first, then Q/K/V, per-dimension gate logits, 

1001 and beta logits are packed into one sequence-to-head all-to-all. Each CP 

1002 rank runs the wrapped layer's ShortConv weights and fused KDA core for its 

1003 local head shard and the full sequence. The reverse all-to-all restores a 

1004 local sequence shard before the original output gate, norm, and projection. 

1005 

1006 Packed samples use global document offsets. ``backend`` selects both the 

1007 attention core and local convolution for dense and packed inputs. 

1008 Inference cache and padded sequences are unsupported. 

1009 """ 

1010 

1011 def __init__( 

1012 self, 

1013 module: nn.Module, 

1014 device_mesh: DeviceMesh, 

1015 *, 

1016 chunk_size: int = 64, 

1017 backend: str = "eager", 

1018 ) -> None: 

1019 """Initialize the full-layer adapter without replacing its parameters. 

1020 

1021 Args: 

1022 module: Original Kimi attention layer that owns the parameters. 

1023 device_mesh: One-dimensional Ulysses mesh. 

1024 chunk_size: Chunk length for the attention core. 

1025 backend: Attention and convolution backend: ``eager`` or ``triton``. 

1026 

1027 Raises: 

1028 ValueError: If a backend or the wrapped layer contract is unsupported. 

1029 """ 

1030 lower_bound = _gate_lower_bound(module) 

1031 if lower_bound is None: 

1032 raise ValueError( 

1033 "Kimi K3 Ulysses CP requires the lower-bounded gate parameterization." 

1034 ) 

1035 super().__init__( 

1036 device_mesh, 

1037 chunk_size=chunk_size, 

1038 lower_bound=float(lower_bound), 

1039 safe_gate=_uses_safe_gate(module), 

1040 backend=backend, 

1041 ) 

1042 self.module = module 

1043 self._validate_module() 

1044 

1045 def _validate_module(self) -> None: 

1046 """Validate the first-stage KimiDeltaAttention module contract.""" 

1047 required_attributes = ( 

1048 "q_proj", 

1049 "k_proj", 

1050 "v_proj", 

1051 "b_proj", 

1052 "o_norm", 

1053 "o_proj", 

1054 "A_log", 

1055 "dt_bias", 

1056 "num_heads", 

1057 "head_k_dim", 

1058 ) 

1059 missing = [name for name in required_attributes if not hasattr(self.module, name)] 

1060 if missing: 

1061 raise TypeError( 

1062 "Kimi K3 layer is missing required attributes: " 

1063 + ", ".join(missing) 

1064 ) 

1065 if self.module.num_heads % self.cp_size != 0: 

1066 raise ValueError( 

1067 f"KDA num_heads ({self.module.num_heads}) must be divisible by " 

1068 f"cp_size ({self.cp_size})." 

1069 ) 

1070 num_v_heads = _num_value_heads(self.module) 

1071 if num_v_heads % self.cp_size != 0: 

1072 raise ValueError( 

1073 f"KDA num_v_heads ({num_v_heads}) must be divisible " 

1074 f"by cp_size ({self.cp_size})." 

1075 ) 

1076 if num_v_heads % self.module.num_heads != 0: 

1077 raise ValueError("KDA num_v_heads must be divisible by num_heads.") 

1078 if not (hasattr(self.module, "f_proj") or _is_official_kimi_layer(self.module)): 

1079 raise TypeError("Kimi K3 layer is missing its forget-gate projection.") 

1080 if not ( 

1081 hasattr(self.module, "g_proj") 

1082 or all(hasattr(self.module, name) for name in ("g_a_proj", "g_b_proj")) 

1083 ): 

1084 raise TypeError("Kimi K3 layer is missing its output-gate projection.") 

1085 if getattr(self.module, "allow_neg_eigval", False): 

1086 raise NotImplementedError( 

1087 "Kimi K3 Ulysses CP does not yet support allow_neg_eigval=True." 

1088 ) 

1089 

1090 _validate_layer_short_conv(self.module, "Ulysses") 

1091 

1092 def _pack_projected_inputs( 

1093 self, 

1094 query: torch.Tensor, 

1095 key: torch.Tensor, 

1096 value: torch.Tensor, 

1097 gate: torch.Tensor, 

1098 beta: torch.Tensor, 

1099 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 

1100 """Pack all projected paths into one sequence-to-head all-to-all.""" 

1101 if self.cp_size == 1: 

1102 return query, key, value, gate, beta 

1103 

1104 base = self.module 

1105 split_sizes = ( 

1106 _key_projection_dim(base) // self.cp_size, 

1107 _key_projection_dim(base) // self.cp_size, 

1108 _value_projection_dim(base) // self.cp_size, 

1109 _num_value_heads(base) * base.head_k_dim // self.cp_size, 

1110 _num_value_heads(base) // self.cp_size, 

1111 ) 

1112 projected = (query, key, value, gate, beta) 

1113 chunks = [torch.split(tensor, size, dim=-1) for tensor, size in zip(projected, split_sizes)] 

1114 rank_major = [ 

1115 torch.cat(parts, dim=-1) 

1116 for parts in zip(*chunks) 

1117 ] 

1118 packed = self._seq_to_head(torch.cat(rank_major, dim=-1).contiguous()) 

1119 return torch.split(packed, split_sizes, dim=-1) 

1120 

1121 def _local_short_conv( 

1122 self, 

1123 projected: torch.Tensor, 

1124 convolution: nn.Conv1d, 

1125 sequence_info: Optional[PackedSequenceInfo] = None, 

1126 ) -> torch.Tensor: 

1127 """Run one wrapped ShortConv on this rank's contiguous channel slice.""" 

1128 local_channels = projected.shape[-1] 

1129 channel_start = self.cp_rank * local_channels 

1130 weight = convolution.weight.narrow(0, channel_start, local_channels) 

1131 bias = convolution.bias 

1132 if bias is not None: 

1133 bias = bias.narrow(0, channel_start, local_channels) 

1134 if sequence_info is not None: 

1135 return packed_causal_conv1d( 

1136 projected, convolution, sequence_info, weight=weight, bias=bias, 

1137 backend=self.backend, 

1138 ) 

1139 if self.backend != "eager": 

1140 return fused_causal_conv1d( 

1141 projected, weight, bias, backend=self.backend, 

1142 dilation=convolution.dilation[0], 

1143 ) 

1144 output = F.conv1d( # pylint: disable=not-callable 

1145 projected.transpose(1, 2), 

1146 weight, 

1147 bias, 

1148 stride=convolution.stride, 

1149 padding=convolution.padding, 

1150 dilation=convolution.dilation, 

1151 groups=local_channels, 

1152 ) 

1153 output = output[:, :, : projected.shape[1]].transpose(1, 2) 

1154 return F.silu(output) 

1155 

1156 def forward( # pylint: disable=arguments-renamed 

1157 self, 

1158 hidden_states: torch.Tensor, 

1159 attention_mask: Optional[torch.Tensor] = None, 

1160 past_key_values: Optional[Any] = None, 

1161 use_cache: bool = False, 

1162 output_attentions: bool = False, 

1163 **kwargs: Any, 

1164 ) -> tuple[torch.Tensor, None, None]: 

1165 """Run the real Kimi layer using KDA and Ulysses CP. 

1166 

1167 Args: 

1168 hidden_states: Local sequence shard [B,T,hidden_size]. 

1169 attention_mask: Optional dense mask; padding is unsupported. 

1170 past_key_values: Recurrent inference cache, currently unsupported. 

1171 use_cache: Must be False for this training adapter. 

1172 output_attentions: Must be False; KDA does not materialize attention weights. 

1173 kwargs: Optional global sample offsets or prebuilt packed_sequence_info. 

1174 """ 

1175 if not self.module.training: 

1176 raise NotImplementedError("Kimi K3 Ulysses CP currently supports training only.") 

1177 if past_key_values is not None or use_cache: 

1178 raise NotImplementedError("Kimi K3 Ulysses CP does not support recurrent cache.") 

1179 if output_attentions: 

1180 raise NotImplementedError("KDA does not materialize attention weights.") 

1181 if attention_mask is not None and not bool(attention_mask.bool().all().item()): 

1182 raise NotImplementedError("Padded KDA CP batches are not implemented.") 

1183 

1184 hidden_states, sequence_info, is_flat_input = _prepare_layer_input( 

1185 hidden_states, kwargs, self.cp_rank, self.cp_size) 

1186 exchanged_info = None if sequence_info is None else sequence_info.ulysses(self.cp_size, self.cp_rank) 

1187 base = self.module 

1188 batch_size, local_seq_len, _ = hidden_states.shape 

1189 query = base.q_proj(hidden_states) 

1190 key = base.k_proj(hidden_states) 

1191 value = base.v_proj(hidden_states) 

1192 gate = _forget_gate_projection(base, hidden_states) 

1193 beta = base.b_proj(hidden_states) 

1194 output_gate = _output_gate_projection(base, hidden_states).reshape( 

1195 batch_size, 

1196 local_seq_len, 

1197 _num_value_heads(base), 

1198 _value_head_dim(base), 

1199 ) 

1200 

1201 query, key, value, gate, beta = self._pack_projected_inputs(query, key, value, gate, beta) 

1202 if _uses_short_conv(base): 

1203 query = self._local_short_conv(query, base.q_conv1d, sequence_info=exchanged_info) 

1204 key = self._local_short_conv(key, base.k_conv1d, sequence_info=exchanged_info) 

1205 value = self._local_short_conv(value, base.v_conv1d, sequence_info=exchanged_info) 

1206 else: 

1207 query, key, value = (F.silu(tensor) for tensor in (query, key, value)) 

1208 

1209 full_seq_len = query.shape[1] 

1210 local_k_heads = base.num_heads // self.cp_size 

1211 local_v_heads = _num_value_heads(base) // self.cp_size 

1212 query = query.reshape( 

1213 batch_size, full_seq_len, local_k_heads, base.head_k_dim 

1214 ) 

1215 key = key.reshape(batch_size, full_seq_len, local_k_heads, base.head_k_dim) 

1216 value = value.reshape( 

1217 batch_size, full_seq_len, local_v_heads, _value_head_dim(base) 

1218 ) 

1219 gate = gate.reshape( 

1220 batch_size, full_seq_len, local_v_heads, base.head_k_dim 

1221 ) 

1222 beta = beta.reshape(batch_size, full_seq_len, local_v_heads) 

1223 

1224 local_a_log = _slice_local_heads( 

1225 base.A_log.reshape(_num_value_heads(base)), 

1226 self.cp_rank, 

1227 self.cp_size, 

1228 ) 

1229 local_dt_bias = _slice_local_heads( 

1230 base.dt_bias.reshape(_num_value_heads(base), base.head_k_dim), 

1231 self.cp_rank, 

1232 self.cp_size, 

1233 ) 

1234 output = _run_local_kda( 

1235 query, 

1236 key, 

1237 value, 

1238 gate, 

1239 beta, 

1240 a_log=local_a_log, 

1241 dt_bias=local_dt_bias, 

1242 scale=None, 

1243 lower_bound=self.lower_bound, 

1244 chunk_size=self.chunk_size, 

1245 safe_gate=self.safe_gate, 

1246 backend=self.backend, 

1247 sequence_info=exchanged_info, 

1248 ) 

1249 output = self._head_to_seq(output) 

1250 output = base.o_norm(output, output_gate) 

1251 output = output.reshape( 

1252 batch_size, local_seq_len, _value_projection_dim(base) 

1253 ) 

1254 output = base.o_proj(output) 

1255 return _format_layer_output(base, output.squeeze(0) if is_flat_input else output) 

1256 

1257 

1258def _validate_cp_degrees(cp_size: int, ulysses_degree: int, allgather_group_size: int) -> None: 

1259 """Validate both split extents before head division or process-group creation.""" 

1260 if (not isinstance(ulysses_degree, int) or isinstance(ulysses_degree, bool) 

1261 or ulysses_degree < 1 or cp_size % ulysses_degree): 

1262 raise ValueError("ulysses_degree must be a positive integer dividing the CP size.") 

1263 if (not isinstance(allgather_group_size, int) or isinstance(allgather_group_size, bool) 

1264 or allgather_group_size < 1 or (cp_size // ulysses_degree) % allgather_group_size): 

1265 raise ValueError("allgather_group_size must be a positive integer dividing the state CP size.") 

1266 

1267 

1268class KimiDeltaAttentionLayerCP(nn.Module): 

1269 """Project local tokens, exchange convolution halos, and invoke the configured CP core.""" 

1270 

1271 def __init__( 

1272 self, 

1273 module: nn.Module, 

1274 device_mesh: DeviceMesh, 

1275 *, 

1276 chunk_size: int = 64, 

1277 backend: str = "eager", 

1278 ulysses_degree: int = 1, 

1279 state_cp_method: str = "p2p", 

1280 allgather_group_size: int = 1, 

1281 ) -> None: 

1282 """Keep layer ownership and full-CP halo metadata separate from the core.""" 

1283 super().__init__() 

1284 lower_bound = _gate_lower_bound(module) 

1285 if lower_bound is None: 

1286 raise ValueError("Kimi K3 CP requires the lower-bounded gate parameterization.") 

1287 self.cp_mesh = _ensure_1d(device_mesh) 

1288 self.cp_rank, self.cp_size = self.cp_mesh.get_local_rank(), self.cp_mesh.size() 

1289 self.module, self.backend = module, backend.lower() 

1290 self._validate_module() 

1291 _validate_cp_degrees(self.cp_size, ulysses_degree, allgather_group_size) 

1292 if module.num_heads % ulysses_degree or _num_value_heads(module) % ulysses_degree: 

1293 raise ValueError("KDA query and value heads must both be divisible by ulysses_degree.") 

1294 self.core = KimiDeltaAttentionCoreCP( 

1295 device_mesh, ulysses_degree=ulysses_degree, chunk_size=chunk_size, 

1296 lower_bound=float(lower_bound), safe_gate=_uses_safe_gate(module), backend=backend, 

1297 state_cp_method=state_cp_method, allgather_group_size=allgather_group_size) 

1298 

1299 def _validate_module(self) -> None: 

1300 """Validate the KimiDeltaAttention contract used by state CP.""" 

1301 required_attributes = ( 

1302 "q_proj", 

1303 "k_proj", 

1304 "v_proj", 

1305 "b_proj", 

1306 "o_norm", 

1307 "o_proj", 

1308 "A_log", 

1309 "dt_bias", 

1310 "num_heads", 

1311 "head_k_dim", 

1312 ) 

1313 missing = [name for name in required_attributes if not hasattr(self.module, name)] 

1314 if missing: 

1315 raise TypeError( 

1316 "Kimi K3 layer is missing required attributes: " 

1317 + ", ".join(missing) 

1318 ) 

1319 num_v_heads = _num_value_heads(self.module) 

1320 if num_v_heads % self.module.num_heads: 

1321 raise ValueError("KDA num_v_heads must be divisible by num_heads.") 

1322 if self.backend in ("triton", "ascendc") and ( 

1323 self.module.head_k_dim != 128 or _value_head_dim(self.module) != 128 

1324 ): 

1325 raise NotImplementedError( 

1326 "Kimi K3 CP currently requires key/value head dimensions 128." 

1327 ) 

1328 if getattr(self.module, "allow_neg_eigval", False): 

1329 raise NotImplementedError( 

1330 "Kimi K3 CP does not support allow_neg_eigval=True." 

1331 ) 

1332 if not (hasattr(self.module, "f_proj") or _is_official_kimi_layer(self.module)): 

1333 raise TypeError("Kimi K3 layer is missing its forget-gate projection.") 

1334 if not ( 

1335 hasattr(self.module, "g_proj") 

1336 or all(hasattr(self.module, name) for name in ("g_a_proj", "g_b_proj")) 

1337 ): 

1338 raise TypeError("Kimi K3 layer is missing its output-gate projection.") 

1339 _validate_layer_short_conv(self.module, "state") 

1340 

1341 def forward( # pylint: disable=arguments-renamed 

1342 self, 

1343 hidden_states: torch.Tensor, 

1344 attention_mask: Optional[torch.Tensor] = None, 

1345 past_key_values: Optional[Any] = None, 

1346 use_cache: bool = False, 

1347 output_attentions: bool = False, 

1348 **kwargs: Any, 

1349 ) -> tuple[torch.Tensor, None, None]: 

1350 """Run projections, full-CP halos, the configured CP core, and output transforms. 

1351 

1352 Args: 

1353 hidden_states: Local sequence shard [B,T,hidden_size]. 

1354 attention_mask: Optional dense mask; padding is unsupported. 

1355 past_key_values: Recurrent inference cache, currently unsupported. 

1356 use_cache: Must be False for this training adapter. 

1357 output_attentions: Must be False; KDA does not materialize attention weights. 

1358 kwargs: Optional global sample offsets or prebuilt packed_sequence_info. 

1359 """ 

1360 if not self.module.training: 

1361 raise NotImplementedError("Kimi K3 CP currently supports training only.") 

1362 if past_key_values is not None or use_cache: 

1363 raise NotImplementedError("Kimi K3 CP does not support recurrent cache.") 

1364 if output_attentions: 

1365 raise NotImplementedError("KDA does not materialize attention weights.") 

1366 if attention_mask is not None and not bool(attention_mask.bool().all().item()): 

1367 raise NotImplementedError("Padded KDA CP batches are not implemented.") 

1368 

1369 hidden_states, sequence_info, is_flat_input = _prepare_layer_input( 

1370 hidden_states, kwargs, self.cp_rank, self.cp_size) 

1371 base = self.module 

1372 batch_size, local_seq_len, _ = hidden_states.shape 

1373 query = base.q_proj(hidden_states) 

1374 key = base.k_proj(hidden_states) 

1375 value = base.v_proj(hidden_states) 

1376 if _uses_short_conv(base): 

1377 query, key, value = _causal_short_convs_with_cp_halo( # pylint: disable=unbalanced-tuple-unpacking 

1378 (query, key, value), 

1379 (base.q_conv1d, base.k_conv1d, base.v_conv1d), 

1380 self.cp_mesh, 

1381 self.cp_rank, 

1382 self.cp_size, 

1383 sequence_info=sequence_info, 

1384 backend="ascendc" if self.backend == "ascendc" else "eager", 

1385 ) 

1386 else: 

1387 query, key, value = (F.silu(tensor) for tensor in (query, key, value)) 

1388 

1389 query = query.reshape( 

1390 batch_size, 

1391 local_seq_len, 

1392 base.num_heads, 

1393 base.head_k_dim, 

1394 ) 

1395 key = key.reshape( 

1396 batch_size, 

1397 local_seq_len, 

1398 base.num_heads, 

1399 base.head_k_dim, 

1400 ) 

1401 value = value.reshape( 

1402 batch_size, 

1403 local_seq_len, 

1404 _num_value_heads(base), 

1405 _value_head_dim(base), 

1406 ) 

1407 gate = _forget_gate_projection(base, hidden_states).reshape( 

1408 batch_size, 

1409 local_seq_len, 

1410 _num_value_heads(base), 

1411 base.head_k_dim, 

1412 ) 

1413 beta = base.b_proj(hidden_states).reshape( 

1414 batch_size, 

1415 local_seq_len, 

1416 _num_value_heads(base), 

1417 ) 

1418 output_gate = _output_gate_projection(base, hidden_states).reshape( 

1419 batch_size, 

1420 local_seq_len, 

1421 _num_value_heads(base), 

1422 _value_head_dim(base), 

1423 ) 

1424 

1425 output = self.core( 

1426 query, 

1427 key, 

1428 value, 

1429 gate, 

1430 beta, 

1431 a_log=base.A_log, 

1432 dt_bias=base.dt_bias, 

1433 sequence_info=sequence_info, 

1434 ) 

1435 output = base.o_norm(output, output_gate) 

1436 output = output.reshape( 

1437 batch_size, local_seq_len, _value_projection_dim(base) 

1438 ) 

1439 output = base.o_proj(output) 

1440 return _format_layer_output(base, output.squeeze(0) if is_flat_input else output) 

1441 

1442 

1443class KimiDeltaAttentionCoreCP(nn.Module): 

1444 """Compose optional Ulysses redistribution with local KDA and state exchange. 

1445 

1446 Inputs are projected Q/K/V/gate/beta after ShortConv. The layer adapter 

1447 handles projections and full-sequence halo neighbors before calling here. 

1448 """ 

1449 

1450 def __init__(self, device_mesh: DeviceMesh, *, ulysses_degree: int = 1, 

1451 chunk_size: int = 64, lower_bound: float = -5.0, safe_gate: bool = True, 

1452 backend: str = "triton", state_cp_method: str = "p2p", 

1453 allgather_group_size: int = 1) -> None: 

1454 """Build all communication groups once, before forward or checkpoint replay.""" 

1455 super().__init__() 

1456 device_mesh = _ensure_1d(device_mesh) 

1457 if state_cp_method not in ("p2p", "allgather", "grouped_allgather_p2p", "recursive_doubling"): 

1458 raise ValueError("KDA requires a supported state_cp_method.") 

1459 if backend.lower() == "eager" and state_cp_method != "p2p": 

1460 raise ValueError("Eager KDA state exchange supports P2P only.") 

1461 _validate_cp_degrees(device_mesh.size(), ulysses_degree, allgather_group_size) 

1462 if state_cp_method != "grouped_allgather_p2p" and allgather_group_size != 1: 

1463 raise ValueError("allgather_group_size is only valid for grouped_allgather_p2p.") 

1464 if lower_bound >= 0: 

1465 raise ValueError("lower_bound must be negative.") 

1466 self.cp_size = device_mesh.size() 

1467 self.cp_rank = device_mesh.get_local_rank() 

1468 self.ulysses_degree = ulysses_degree 

1469 self.ulysses_mesh = None 

1470 self.ulysses_rank = 0 

1471 state_mesh = device_mesh 

1472 if ulysses_degree > 1: 

1473 hybrid = split_kda_mesh(device_mesh, ulysses_degree, ("kda_state", "kda_ulysses")) 

1474 self.ulysses_mesh = hybrid["kda_ulysses"] 

1475 self.ulysses_rank = self.ulysses_mesh.get_local_rank() 

1476 state_mesh = hybrid["kda_state"] 

1477 self.state = _KDAStateCP( 

1478 state_mesh, chunk_size=chunk_size, lower_bound=lower_bound, safe_gate=safe_gate, 

1479 backend=backend, state_cp_method=state_cp_method, allgather_group_size=allgather_group_size) 

1480 

1481 def forward(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, 

1482 gate: torch.Tensor, beta: torch.Tensor, *, a_log: torch.Tensor, 

1483 dt_bias: torch.Tensor, scale: float | None = None, 

1484 sequence_info: Optional[PackedSequenceInfo] = None) -> torch.Tensor: 

1485 """Run state CP with optional token/head redistribution. 

1486 

1487 Args: 

1488 query: Projected query tensor [B,T,H,K]. 

1489 key: Matching local key tensor. 

1490 value: Projected value tensor [B,T,HV,V]. 

1491 gate: Per-token gate logits [B,T,HV,K]. 

1492 beta: Per-token beta logits [B,T,HV]. 

1493 a_log: Learned decay parameter with one entry per value head. 

1494 dt_bias: Learned gate bias with HV times K entries. 

1495 scale: Attention scaling factor; None selects the local default. 

1496 

1497 Returns: 

1498 Output restored to this rank's original token/head layout. 

1499 """ 

1500 is_flat_input = query.ndim == 3 

1501 if is_flat_input: 

1502 output = self.forward(*(tensor.unsqueeze(0) for tensor in (query, key, value, gate, beta)), 

1503 a_log=a_log, dt_bias=dt_bias, scale=scale, sequence_info=sequence_info) 

1504 return output.squeeze(0) 

1505 if sequence_info is not None: 

1506 if query.shape[0] != 1: 

1507 raise ValueError("Packed KDA core requires batch dimension one.") 

1508 resolve_packed_sequence({"packed_sequence_info": sequence_info}, query.shape[1], 

1509 cp_rank=self.cp_rank, cp_size=self.cp_size) 

1510 if self.ulysses_mesh is None: 

1511 return self.state(query, key, value, gate, beta, a_log=a_log, dt_bias=dt_bias, scale=scale, 

1512 sequence_info=sequence_info) 

1513 _validate_projected_inputs(self.ulysses_degree, query, key, value, gate, beta, a_log, dt_bias) 

1514 heads, key_dim = value.shape[2], query.shape[-1] 

1515 inputs = self._exchange_inputs((query, key, value, gate, beta)) 

1516 local_a = _slice_local_heads(a_log.reshape(heads), self.ulysses_rank, self.ulysses_degree) 

1517 local_bias = _slice_local_heads(dt_bias.reshape(heads, key_dim), self.ulysses_rank, self.ulysses_degree) 

1518 exchanged_info = None if sequence_info is None else sequence_info.ulysses( 

1519 self.ulysses_degree, self.ulysses_rank) 

1520 output = self.state(*inputs, a_log=local_a, dt_bias=local_bias, scale=scale, sequence_info=exchanged_info) 

1521 return _differentiable_all_to_all_shard(output, self.ulysses_mesh, split_dim=1, concat_dim=2) 

1522 

1523 def _exchange_inputs(self, inputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]: 

1524 """Redistribute projected inputs using the established synchronous schedule.""" 

1525 return tuple(_differentiable_all_to_all_shard(tensor, self.ulysses_mesh, split_dim=2, concat_dim=1) 

1526 for tensor in inputs) 

1527 

1528 

1529class KimiDeltaAttentionLayerP2PCP(KimiDeltaAttentionLayerCP): 

1530 """Preserve the public layer-P2P type while sharing layer execution. 

1531 

1532 Inherited defaults select P2P with no Ulysses. The subclass retains the old 

1533 import and type name; new hybrid configurations use KimiDeltaAttentionLayerCP. 

1534 """