Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / dsa_attention.py: 0%

405 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Model-compatible DSA modules using the Hyper high-performance operators.""" 

16 

17from __future__ import annotations 

18 

19from collections.abc import Mapping, Sequence 

20from typing import Any 

21 

22# This package provides PyTorch-specific high-performance modules. 

23import torch 

24from torch import nn 

25from torch.nn import functional as F 

26from hyper_parallel.components.checkpoint.weight_conversion import WeightConverter 

27 

28from hyper_parallel.components.checkpoint import ConcatenateWithSections 

29from hyper_parallel.models.replacement import module_replacement 

30from hyper_parallel.components.functional import aggregate_hidden, aux_loss_auto_scale 

31from hyper_parallel.components.functional import apply_rotary_pos_emb, apply_rotary_pos_emb_interleave 

32from hyper_parallel.components.functional import ( 

33 dsa_indexer, 

34 dsa_kl_loss, 

35 dsa_sparse_attention, 

36 dsa_sparse_attention_rescale, 

37) 

38from hyper_parallel.components.functional.npu_fusion_attention import resolve_packed_sequence_lengths 

39 

40 

41def _dense_flash_attention( 

42 module: nn.Module, 

43 query: torch.Tensor, 

44 key: torch.Tensor, 

45 value: torch.Tensor, 

46 sequence_lengths: tuple[torch.Tensor, torch.Tensor], 

47 *, 

48 need_stats: bool, 

49) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]: 

50 """Run one TND FA and return dense-indexer statistics.""" 

51 query_ends, key_ends = (lengths.tolist() for lengths in sequence_lengths) 

52 q_flat, k_flat, v_flat = (tensor.flatten(0, 1) for tensor in (query, key, value)) 

53 num_heads = q_flat.shape[1] 

54 causal_mask = torch.ones((2048, 2048), dtype=torch.bool, device=query.device).triu(diagonal=1) 

55 import torch_npu # pylint: disable=import-outside-toplevel 

56 

57 result = torch_npu.npu_fusion_attention( 

58 q_flat.contiguous(), k_flat.contiguous(), v_flat.contiguous(), num_heads, "TND", 

59 pse=None, padding_mask=None, atten_mask=causal_mask, scale=module.scaling, 

60 pre_tockens=getattr(module, "pre_tockens", 1048576), 

61 next_tockens=getattr(module, "next_tockens", 0), keep_prob=1.0, inner_precise=0, 

62 actual_seq_qlen=query_ends, actual_seq_kvlen=key_ends, sparse_mode=3, 

63 ) 

64 output = result[0].reshape(*query.shape[:3], value.shape[-1]) 

65 if not need_stats: 

66 return output, None, None 

67 

68 # TND FA stores each packed segment as [head, token, 8], while exposing a 

69 # physical [token, head, 8] shape. Dense KL consumes [head, token, 1]. 

70 statistics = [] 

71 starts = [0, *query_ends[:-1]] 

72 for statistic in result[1:3]: 

73 rows = statistic.reshape(-1, statistic.shape[-1]) 

74 segments = [ 

75 rows[start * num_heads : end * num_heads].reshape( 

76 num_heads, end - start, statistic.shape[-1] 

77 )[..., :1] 

78 for start, end in zip(starts, query_ends) 

79 ] 

80 statistics.append(torch.cat(segments, dim=1).detach()) 

81 return output, statistics[0], statistics[1] 

82 

83 

84def _deepseek_v32_dense_attention( 

85 module: nn.Module, 

86 index_states: tuple[torch.Tensor, ...], 

87 attention_states: tuple[torch.Tensor, ...], 

88 sequence_lengths: tuple[torch.Tensor, torch.Tensor], 

89) -> torch.Tensor: 

90 """Run unabsorbed MLA FA and the fused dense indexer KL on local states.""" 

91 query, compressed_kv, query_rope, key_rope = attention_states 

92 projected_kv = module.kv_b_proj(compressed_kv.squeeze(2)).view( 

93 *compressed_kv.shape[:2], module.num_heads, module.qk_nope_head_dim + module.v_head_dim, 

94 ) 

95 key, value = projected_kv.split((module.qk_nope_head_dim, module.v_head_dim), dim=-1) 

96 key_rope = key_rope.expand(-1, -1, module.num_heads, -1) 

97 need_stats = bool(module.training and not module.freeze_dsa and module.dsa_loss_coeff) 

98 output, maximum, total = _dense_flash_attention( 

99 module, torch.cat((query, query_rope), dim=-1), torch.cat((key, key_rope), dim=-1), 

100 value, sequence_lengths, need_stats=need_stats, 

101 ) 

102 if need_stats: 

103 # Dense CANN operators are optional for sparse and Omni execution. 

104 # pylint: disable=import-outside-toplevel 

105 from hyper_parallel.components.functional.dsa_dense import ( 

106 dsa_dense_indexer_kl_loss, 

107 dsa_dense_indexer_lse, 

108 ) 

109 # pylint: enable=import-outside-toplevel 

110 

111 query_lengths, key_lengths = sequence_lengths 

112 index_query, index_key, weights = (tensor.flatten(0, 1) for tensor in index_states) 

113 indexer_stats = dsa_dense_indexer_lse( 

114 index_query, index_key, weights, query_lengths=query_lengths, key_lengths=key_lengths, 

115 ) 

116 loss = dsa_dense_indexer_kl_loss( 

117 index_query, index_key, weights, query.flatten(0, 1), key.flatten(0, 1), 

118 maximum, total, indexer_stats, scale=module.scaling, 

119 query_rope=query_rope.flatten(0, 1), key_rope=key_rope.flatten(0, 1), 

120 query_lengths=query_lengths, key_lengths=key_lengths, loss_coefficient=module.dsa_loss_coeff, 

121 ) 

122 output = aux_loss_auto_scale(output, loss) 

123 return output 

124 

125 

126def apply_mome( 

127 hidden_states: torch.Tensor, 

128 mome_mask: torch.Tensor | None, 

129 convolution: nn.Conv1d, 

130 *, 

131 fused: bool, 

132) -> torch.Tensor: 

133 """Apply masked causal depthwise convolution and residual.""" 

134 if mome_mask is None: 

135 raise ValueError("mome_mask is required when MOME is enabled") 

136 if mome_mask.shape != hidden_states.shape[:2]: 

137 raise ValueError( 

138 f"mome_mask must have shape {hidden_states.shape[:2]}, " 

139 f"but got {tuple(mome_mask.shape)}" 

140 ) 

141 

142 mome_mask = mome_mask.to(device=hidden_states.device, dtype=torch.bool) 

143 padding = convolution.kernel_size[0] - 1 

144 padded_states = F.pad(hidden_states, (0, 0, padding, 0)) 

145 if fused: 

146 padded_mask = F.pad(mome_mask, (padding, 0), value=False) 

147 weight = convolution.weight.squeeze(1).transpose(0, 1) 

148 mixed_states = aggregate_hidden( # pylint: disable=not-callable 

149 padded_states.transpose(0, 1).contiguous(), 

150 weight, 

151 padded_mask, 

152 ) 

153 mixed_states = mixed_states[padding:].transpose(0, 1).contiguous() 

154 else: 

155 mixed_states = convolution(padded_states.transpose(1, 2)).transpose(1, 2) 

156 mixed_states = mixed_states * mome_mask.unsqueeze(-1).to(mixed_states.dtype) 

157 return hidden_states + mixed_states 

158 

159 

160def _apply_attention_rope( 

161 query: torch.Tensor, 

162 key: torch.Tensor, 

163 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, 

164 *, 

165 interleaved: bool, 

166) -> tuple[torch.Tensor, torch.Tensor]: 

167 """Apply the configured RoPE convention to an attention query/key pair.""" 

168 if position_embeddings is None: 

169 return query, key 

170 cos, sin = position_embeddings 

171 rotary_op = apply_rotary_pos_emb_interleave if interleaved else apply_rotary_pos_emb 

172 return rotary_op(query, key, cos, sin, unsqueeze_dim=2) 

173 

174 

175def _apply_index_rope( 

176 query: torch.Tensor, 

177 key: torch.Tensor, 

178 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, 

179 *, 

180 rope_head_dim: int, 

181 index_head_dim: int, 

182 interleaved: bool, 

183) -> tuple[torch.Tensor, torch.Tensor]: 

184 """Apply RoPE to the rotary portion of an indexer query/key pair.""" 

185 if position_embeddings is None: 

186 return query, key 

187 cos, sin = position_embeddings 

188 split_sizes = (rope_head_dim, index_head_dim - rope_head_dim) 

189 query_rot, query_pass = torch.split(query, split_sizes, dim=-1) 

190 key_rot, key_pass = torch.split(key, split_sizes, dim=-1) 

191 rotary_op = apply_rotary_pos_emb_interleave if interleaved else apply_rotary_pos_emb 

192 query_rot, key_rot = rotary_op(query_rot, key_rot, cos, sin, unsqueeze_dim=2) 

193 return torch.cat((query_rot, query_pass), dim=-1), torch.cat((key_rot, key_pass), dim=-1) 

194 

195 

196def _absorb_query_projection( 

197 query: torch.Tensor, 

198 key_up_weight: torch.Tensor, 

199 *, 

200 num_heads: int, 

201 batch_size: int, 

202 seq_length: int, 

203 input_dim: int, 

204 output_dim: int, 

205) -> torch.Tensor: 

206 """Absorb the key up-projection weight into query states.""" 

207 query = query.permute(2, 0, 1, 3).reshape(num_heads, batch_size * seq_length, input_dim) 

208 query = torch.bmm(query, key_up_weight).view(num_heads, batch_size, seq_length, output_dim) 

209 return query.permute(1, 2, 0, 3) 

210 

211 

212def _restore_attention_projection( 

213 output: torch.Tensor, 

214 value_up_weight: torch.Tensor, 

215 *, 

216 num_heads: int, 

217 batch_size: int, 

218 seq_length: int, 

219 kv_lora_rank: int, 

220 value_head_dim: int, 

221) -> torch.Tensor: 

222 """Apply the absorbed value up-projection and restore the hidden layout.""" 

223 output = output[..., :kv_lora_rank] 

224 output = output.permute(2, 0, 1, 3).reshape(num_heads, batch_size * seq_length, kv_lora_rank) 

225 output = torch.bmm(output, value_up_weight) 

226 return output.view(num_heads, batch_size, seq_length, value_head_dim).permute(1, 2, 0, 3).reshape( 

227 batch_size, seq_length, -1 

228 ) 

229 

230 

231@module_replacement 

232class DeepseekV32DSAAttention(nn.Module): 

233 """NPU DSA replacement for Transformers ``DeepseekV32Attention``. 

234 

235 It supports causal training and packed sequences. Packed boundaries must 

236 be supplied through ``actual_seq_len``. The optional ``attention_mask`` is 

237 assumed to be the standard causal mask created by Transformers; padding, 

238 custom attention masks, dropout, and KV cache are not supported. 

239 """ 

240 

241 @staticmethod 

242 def _validate_source(module: nn.Module) -> None: 

243 """Validate the required DSA module and indexer surface.""" 

244 required = ( 

245 "q_a_proj", "q_a_layernorm", "q_b_proj", "kv_a_proj_with_mqa", 

246 "kv_a_layernorm", "kv_b_proj", "o_proj", "indexer", 

247 ) 

248 missing = [name for name in required if not hasattr(module, name)] 

249 if missing: 

250 raise TypeError(f"DeepseekV32DSAAttention source module is missing: {missing}") 

251 indexer_required = ("wq_b", "wk", "k_norm", "weights_proj") 

252 indexer_missing = [name for name in indexer_required if not hasattr(module.indexer, name)] 

253 if indexer_missing: 

254 raise TypeError(f"DeepseekV32DSAAttention source indexer is missing: {indexer_missing}") 

255 

256 def _initialize_dimensions(self, module: nn.Module) -> None: 

257 """Copy main-attention and indexer dimensions from the source module.""" 

258 config = self.config 

259 self.layer_idx = getattr(module, "layer_idx", getattr(module, "layer_number", None)) 

260 self.num_heads = getattr(module, "num_heads", config.num_attention_heads) 

261 self.q_lora_rank = getattr(module, "q_lora_rank", config.q_lora_rank) 

262 self.kv_lora_rank = getattr(module, "kv_lora_rank", config.kv_lora_rank) 

263 self.qk_rope_head_dim = getattr(module, "qk_rope_head_dim", config.qk_rope_head_dim) 

264 self.qk_nope_head_dim = getattr(module, "qk_nope_head_dim", config.qk_nope_head_dim) 

265 self.v_head_dim = getattr(module, "v_head_dim", config.v_head_dim) 

266 self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim 

267 if self.q_lora_rank is None: 

268 raise ValueError("DeepseekV32DSAAttention requires the Q-LoRA projection path") 

269 self.index_head_dim = getattr(module.indexer, "head_dim", config.index_head_dim) 

270 self.num_index_heads = getattr( 

271 module.indexer, 

272 "n_heads", 

273 getattr(config, "index_n_heads", getattr(config, "index_num_attention_heads", None)), 

274 ) 

275 if self.num_index_heads is None: 

276 raise ValueError("DeepseekV32DSAAttention requires the number of indexer heads") 

277 self.index_topk = getattr(module.indexer, "index_topk", config.index_topk) 

278 

279 def _initialize_options(self, module: nn.Module) -> None: 

280 """Copy supported DSA options and reject unsupported source modes.""" 

281 config = self.config 

282 self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5) 

283 attention_dropout = getattr(module, "attention_dropout", getattr(config, "attention_dropout", 0.0)) 

284 if isinstance(attention_dropout, nn.Dropout): 

285 attention_dropout = attention_dropout.p 

286 if attention_dropout != 0.0: 

287 raise ValueError( 

288 "DeepseekV32DSAAttention requires attention_dropout=0 because " 

289 "the sparse-attention operator does not apply dropout" 

290 ) 

291 source_rotary_interleaved = getattr(module, "rotary_interleaved", None) 

292 if source_rotary_interleaved is None: 

293 source_rotary_interleaved = getattr(config, "rope_interleave", None) 

294 if source_rotary_interleaved is None: 

295 source_rotary_interleaved = getattr(config, "rope_interleaved", None) 

296 self.rotary_interleaved = bool(True if source_rotary_interleaved is None else source_rotary_interleaved) 

297 self.index_rotary_interleaved = bool( 

298 getattr(module.indexer, "rotary_interleaved", getattr(config, "index_rope_interleaved", False)) 

299 ) 

300 self.dsa_loss_coeff = getattr(config, "dsa_loss_coeff", 0.0) 

301 self.freeze_dsa = bool(getattr(config, "freeze_DSA", getattr(config, "freeze_dsa", False))) 

302 self.dsa_dense_warm_up = bool( 

303 getattr(module, "dsa_dense_warm_up", getattr(config, "dsa_dense_warm_up", False)) 

304 ) 

305 if getattr(module, "param_sink_number", 0) > 0: 

306 raise ValueError("DeepseekV32DSAAttention does not support the parameter-sink path") 

307 if getattr(module, "use_mome", False): 

308 raise ValueError("DeepseekV32DSAAttention does not support the MOME path") 

309 

310 def _initialize_source_layers(self, module: nn.Module) -> None: 

311 """Fuse latent projections and attach the remaining source layers.""" 

312 q_a = module.q_a_proj 

313 kv_a = module.kv_a_proj_with_mqa 

314 self._q_latent_output_size = q_a.out_features 

315 self._kv_latent_output_size = kv_a.out_features 

316 can_fuse = ( 

317 q_a.in_features == kv_a.in_features 

318 and q_a.weight.requires_grad == kv_a.weight.requires_grad 

319 and (q_a.bias is None) == (kv_a.bias is None) 

320 and (q_a.bias is None or q_a.bias.requires_grad == kv_a.bias.requires_grad) 

321 ) 

322 if not can_fuse: 

323 raise ValueError("Q and KV latent projections cannot be represented by one fused projection") 

324 self.linear_qkv = nn.Linear( 

325 q_a.in_features, 

326 q_a.out_features + kv_a.out_features, 

327 bias=q_a.bias is not None, 

328 device=q_a.weight.device, 

329 dtype=q_a.weight.dtype, 

330 ) 

331 self.linear_qkv.weight.requires_grad_(q_a.weight.requires_grad) 

332 if q_a.bias is not None: 

333 self.linear_qkv.bias.requires_grad_(q_a.bias.requires_grad) 

334 for name in ("q_a_layernorm", "kv_a_layernorm", "q_b_proj", "kv_b_proj", "o_proj", "indexer"): 

335 setattr(self, name, getattr(module, name)) 

336 

337 def __init__( 

338 self, 

339 *, 

340 module: nn.Module, 

341 module_fqn: str = "", 

342 context: Mapping[str, Any] | None = None, 

343 ) -> None: 

344 """Build the high-performance module from a Transformers DSA module.""" 

345 super().__init__() 

346 del module_fqn, context 

347 self._validate_source(module) 

348 config = module.config 

349 self.config = config 

350 if hasattr(config, "indexer_types"): 

351 raise TypeError( 

352 "DeepseekV32DSAAttention supports the DeepSeek-V3.2 DSA contract; " 

353 "GLM-MOE-DSA shared indexers and three-value forward result require " 

354 "a dedicated replacement" 

355 ) 

356 self._initialize_dimensions(module) 

357 self._initialize_options(module) 

358 self._initialize_source_layers(module) 

359 self.train(module.training) 

360 

361 def make_transforms(self) -> list[WeightConverter]: 

362 """Describe reversible source-checkpoint to high-performance conversion.""" 

363 transforms: list[WeightConverter] = [ 

364 WeightConverter( 

365 source_patterns=[ 

366 "q_a_proj.weight", 

367 "kv_a_proj_with_mqa.weight", 

368 ], 

369 target_patterns="linear_qkv.weight", 

370 operations=[ 

371 ConcatenateWithSections( 

372 sections=( 

373 self._q_latent_output_size, 

374 self._kv_latent_output_size, 

375 ), 

376 dim=0, 

377 ) 

378 ], 

379 ), 

380 ] 

381 if self.linear_qkv.bias is not None: 

382 transforms.insert( 

383 1, 

384 WeightConverter( 

385 source_patterns=["q_a_proj.bias", "kv_a_proj_with_mqa.bias"], 

386 target_patterns="linear_qkv.bias", 

387 operations=[ 

388 ConcatenateWithSections( 

389 sections=( 

390 self._q_latent_output_size, 

391 self._kv_latent_output_size, 

392 ), 

393 dim=0, 

394 ) 

395 ], 

396 ), 

397 ) 

398 return transforms 

399 

400 @staticmethod 

401 def _get_actual_seq_len( 

402 actual_seq_len: torch.Tensor | Sequence[int] | None, 

403 batch_size: int, 

404 seq_length: int, 

405 device: torch.device, 

406 ) -> torch.Tensor: 

407 """Normalize cumulative sequence lengths for the TND custom operators.""" 

408 if isinstance(actual_seq_len, torch.Tensor): 

409 return actual_seq_len.to(device=device, dtype=torch.int32) 

410 if actual_seq_len is not None: 

411 return torch.tensor(actual_seq_len, device=device, dtype=torch.int32) 

412 return torch.arange( 

413 seq_length, 

414 (batch_size + 1) * seq_length, 

415 seq_length, 

416 dtype=torch.int32, 

417 device=device, 

418 ) 

419 

420 @staticmethod 

421 def _invalid_position_ids( 

422 position_ids: torch.Tensor | None, 

423 batch_size: int, 

424 seq_length: int, 

425 ) -> bool: 

426 """Return whether position IDs violate the supported dense layout.""" 

427 if position_ids is None: 

428 return False 

429 if position_ids.ndim != 2: 

430 return True 

431 return position_ids.shape[0] not in (1, batch_size) or position_ids.shape[1] != seq_length 

432 

433 @staticmethod 

434 def _validate_forward_inputs( 

435 hidden_states: torch.Tensor, 

436 attention_mask: torch.Tensor | None, 

437 past_key_values: Any | None, 

438 position_ids: torch.Tensor | None, 

439 kwargs: dict[str, Any], 

440 ) -> None: 

441 """Validate inputs supported by the DeepSeek sparse-attention path.""" 

442 batch_size, seq_length = hidden_states.shape[:-1] 

443 if past_key_values is not None: 

444 raise NotImplementedError("DeepseekV32DSAAttention does not support KV cache") 

445 if DeepseekV32DSAAttention._invalid_position_ids(position_ids, batch_size, seq_length): 

446 raise ValueError("position_ids must have shape [1 or batch_size, sequence_length]") 

447 if attention_mask is not None: 

448 expected_shape = (batch_size, 1, seq_length, seq_length) 

449 if tuple(attention_mask.shape) != expected_shape: 

450 raise ValueError( 

451 "DeepseekV32DSAAttention supports only a Transformers 4D causal mask; " 

452 "padding and custom masks are not supported" 

453 ) 

454 if kwargs.pop("output_attentions", False): 

455 raise NotImplementedError("DeepseekV32DSAAttention does not expose attention weights") 

456 

457 def _project_attention_states( 

458 self, 

459 hidden_states: torch.Tensor, 

460 *, 

461 use_dense: bool = False, 

462 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 

463 """Project MLA Q/KV, retaining Q before absorption during dense warm-up.""" 

464 batch_size, seq_length = hidden_states.shape[:-1] 

465 latent_states = self.linear_qkv(hidden_states) 

466 q_latent, kv_nope, k_rot = torch.split( 

467 latent_states, (self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim), dim=-1 

468 ) 

469 q_resid = self.q_a_layernorm(q_latent) 

470 q_states = self.q_b_proj(q_resid).view(batch_size, seq_length, self.num_heads, self.qk_head_dim) 

471 q_pass, q_rot = torch.split(q_states, (self.qk_nope_head_dim, self.qk_rope_head_dim), dim=-1) 

472 kv_weight = self.kv_b_proj.weight.view( 

473 self.num_heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank 

474 ) 

475 absorbed_query = q_pass if use_dense else _absorb_query_projection( 

476 q_pass, 

477 kv_weight[:, : self.qk_nope_head_dim], 

478 num_heads=self.num_heads, 

479 batch_size=batch_size, 

480 seq_length=seq_length, 

481 input_dim=self.qk_nope_head_dim, 

482 output_dim=self.kv_lora_rank, 

483 ) 

484 kv_nope = self.kv_a_layernorm(kv_nope).view(batch_size, seq_length, 1, self.kv_lora_rank) 

485 k_rot = k_rot.view(batch_size, seq_length, 1, self.qk_rope_head_dim) 

486 return q_resid, absorbed_query, kv_nope, q_rot, k_rot, kv_weight 

487 

488 def _project_index_states( 

489 self, 

490 hidden_states: torch.Tensor, 

491 q_resid: torch.Tensor, 

492 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, 

493 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 

494 """Build indexer query, key, and merge-weight states.""" 

495 batch_size, seq_length = hidden_states.shape[:-1] 

496 index_query = self.indexer.wq_b(q_resid.detach()).view( 

497 batch_size, seq_length, self.num_index_heads, self.index_head_dim 

498 ) 

499 index_key = self.indexer.k_norm(self.indexer.wk(hidden_states.detach())).unsqueeze(2) 

500 merge_weight = self.indexer.weights_proj(hidden_states.detach()) 

501 merge_weight = merge_weight * self.num_index_heads**-0.5 * self.index_head_dim**-0.5 

502 index_query, index_key = _apply_index_rope( 

503 index_query, 

504 index_key, 

505 position_embeddings, 

506 rope_head_dim=self.qk_rope_head_dim, 

507 index_head_dim=self.index_head_dim, 

508 interleaved=self.index_rotary_interleaved, 

509 ) 

510 return index_query, index_key, merge_weight 

511 

512 def _apply_auxiliary_loss( 

513 self, 

514 attn_output: torch.Tensor, 

515 index_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor], 

516 attention_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], 

517 topk_indices: torch.Tensor, 

518 softmax_stats: tuple[torch.Tensor, torch.Tensor], 

519 actual_q_len: torch.Tensor, 

520 actual_kv_len: torch.Tensor, 

521 ) -> torch.Tensor: 

522 """Attach the DSA KL auxiliary loss when training enables it.""" 

523 if not (self.training and not self.freeze_dsa and self.dsa_loss_coeff): 

524 return attn_output 

525 query_tnd, key_tnd, q_rot_tnd, k_rot_tnd = ( 

526 tensor.reshape(-1, tensor.shape[2], tensor.shape[3]) 

527 for tensor in attention_states 

528 ) 

529 aux_loss = dsa_kl_loss( 

530 *index_states, query_tnd, key_tnd, 

531 topk_indices, *softmax_stats, q_rot_tnd, k_rot_tnd, 

532 actual_q_len, actual_kv_len, self.scaling, self.dsa_loss_coeff, 

533 backend="cann", 

534 ) 

535 return aux_loss_auto_scale(attn_output, aux_loss) 

536 

537 def _prepare_attention_states(self, hidden_states, position_embeddings): 

538 """Project attention states and apply their rotary embeddings.""" 

539 q_resid, absorbed_query_states, kv_nope, q_rot, k_rot, kv_weight = self._project_attention_states( 

540 hidden_states, use_dense=self.dsa_dense_warm_up, 

541 ) 

542 q_rot, k_rot = _apply_attention_rope( 

543 q_rot, k_rot, position_embeddings, interleaved=self.rotary_interleaved 

544 ) 

545 return q_resid, (absorbed_query_states, kv_nope, q_rot, k_rot), kv_weight 

546 

547 def _run_indexer(self, hidden_states, q_resid, position_embeddings, actual_seq_len, kwargs): 

548 """Resolve packed boundaries and select sparse attention indices.""" 

549 index_states = self._project_index_states(hidden_states, q_resid, position_embeddings) 

550 packed_kwargs = dict(kwargs) 

551 packed_kwargs["actual_seq_len"] = actual_seq_len 

552 actual_q_len, actual_kv_len = resolve_packed_sequence_lengths( 

553 packed_kwargs, 

554 hidden_states.shape[0] * hidden_states.shape[1], 

555 hidden_states.shape[0] * hidden_states.shape[1], 

556 ) 

557 actual_q_len = self._get_actual_seq_len( 

558 actual_q_len, *hidden_states.shape[:-1], hidden_states.device, 

559 ) 

560 actual_kv_len = self._get_actual_seq_len( 

561 actual_kv_len, *hidden_states.shape[:-1], hidden_states.device, 

562 ) 

563 indexed = dsa_indexer( 

564 *index_states, actual_q_len, actual_kv_len, self.index_topk, backend="cann", 

565 ) 

566 return indexed[0], indexed[1:], actual_q_len, actual_kv_len 

567 

568 def _compute_sparse_attention( 

569 self, hidden_states, q_resid, attention_states, position_embeddings, actual_seq_len, kwargs, 

570 ): 

571 """Run sparse attention and attach its optional auxiliary loss.""" 

572 topk_indices, index_states, actual_q_len, actual_kv_len = self._run_indexer( 

573 hidden_states, q_resid, position_embeddings, actual_seq_len, kwargs, 

574 ) 

575 attn_output, softmax_max, softmax_sum = dsa_sparse_attention( 

576 *attention_states, topk_indices, self.scaling, actual_q_len, actual_kv_len, 

577 backend="cann", 

578 ) 

579 return self._apply_auxiliary_loss( 

580 attn_output, index_states, attention_states, topk_indices, 

581 (softmax_max, softmax_sum), actual_q_len, actual_kv_len, 

582 ) 

583 

584 def _compute_attention_output( 

585 self, hidden_states, position_embeddings, actual_seq_len, kwargs, 

586 ): 

587 """Run the selected dense/sparse stage and restore the value projection.""" 

588 batch_size, seq_length = hidden_states.shape[:-1] 

589 q_resid, attention_states, kv_weight = self._prepare_attention_states( 

590 hidden_states, position_embeddings, 

591 ) 

592 if self.dsa_dense_warm_up: 

593 index_states = self._project_index_states(hidden_states, q_resid, position_embeddings) 

594 packed_kwargs = {**kwargs, "actual_seq_len": actual_seq_len} 

595 lengths = resolve_packed_sequence_lengths( 

596 packed_kwargs, batch_size * seq_length, batch_size * seq_length, 

597 ) 

598 lengths = tuple(self._get_actual_seq_len( 

599 value, batch_size, seq_length, hidden_states.device, 

600 ) for value in lengths) 

601 output = _deepseek_v32_dense_attention(self, index_states, attention_states, lengths) 

602 return output.flatten(2) 

603 attn_output = self._compute_sparse_attention( 

604 hidden_states, q_resid, attention_states, position_embeddings, actual_seq_len, kwargs, 

605 ) 

606 value_up_weight = kv_weight[:, self.qk_nope_head_dim :].transpose(1, 2) 

607 return _restore_attention_projection( 

608 attn_output, 

609 value_up_weight, 

610 num_heads=self.num_heads, 

611 batch_size=batch_size, 

612 seq_length=seq_length, 

613 kv_lora_rank=self.kv_lora_rank, 

614 value_head_dim=self.v_head_dim, 

615 ) 

616 

617 def forward( 

618 self, 

619 hidden_states: torch.Tensor, 

620 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, 

621 attention_mask: torch.Tensor | None = None, 

622 past_key_values: Any | None = None, 

623 position_ids: torch.Tensor | None = None, 

624 actual_seq_len: torch.Tensor | Sequence[int] | None = None, 

625 **kwargs: Any, 

626 ) -> tuple[torch.Tensor, None]: 

627 """Run causal or packed DSA with the NPU sparse-attention kernels.""" 

628 self._validate_forward_inputs(hidden_states, attention_mask, past_key_values, position_ids, kwargs) 

629 attn_output = self._compute_attention_output( 

630 hidden_states, position_embeddings, actual_seq_len, kwargs, 

631 ) 

632 attn_output = self.o_proj(attn_output) 

633 return attn_output, None 

634 

635 

636@module_replacement 

637class DSAAttention(nn.Module): 

638 """NPU-optimized replacement for a DSA attention module. 

639 

640 Packed boundaries are supplied through ``actual_seq_len``. Attention masks, 

641 KV reuse, and KV cache are not supported. The source parameter layout is 

642 preserved, so no checkpoint conversion is required. 

643 """ 

644 

645 def __init__( 

646 self, 

647 *, 

648 module: nn.Module, 

649 module_fqn: str = "", 

650 context: Mapping[str, Any] | None = None, 

651 ) -> None: 

652 """Build the replacement from a compatible DSA attention module.""" 

653 super().__init__() 

654 del module_fqn, context 

655 required = ( 

656 "linear_qkv", 

657 "q_layernorm", 

658 "k_layernorm", 

659 "linear_qb", 

660 "linear_kvb", 

661 "linear_proj", 

662 "index_linear_qb", 

663 "index_linear_k", 

664 "index_k_layernorm", 

665 "linear_merge_weight", 

666 ) 

667 missing = [name for name in required if not hasattr(module, name)] 

668 if missing: 

669 raise TypeError(f"DSAAttention source module is missing: {missing}") 

670 

671 unsupported = { 

672 "mla_mm_split": bool(getattr(module, "mla_mm_split", False)), 

673 "dense warm-up": bool(getattr(module, "dsa_dense_warm_up", False)), 

674 } 

675 enabled = [name for name, value in unsupported.items() if value] 

676 if enabled: 

677 raise NotImplementedError( 

678 f"DSAAttention does not yet support: {', '.join(enabled)}" 

679 ) 

680 

681 self.config = module.config 

682 self.layer_number = module.layer_number 

683 self.num_heads = module.num_heads 

684 self.q_lora_rank = module.q_lora_rank 

685 self.kv_lora_rank = module.kv_lora_rank 

686 self.qk_rope_head_dim = module.qk_rope_head_dim 

687 self.qk_nope_head_dim = module.qk_nope_head_dim 

688 self.qk_head_dim = module.qk_head_dim 

689 self.v_head_dim = module.v_head_dim 

690 self.index_head_dim = module.index_head_dim 

691 self.num_index_heads = module.num_index_heads 

692 self.index_topk = module.index_topk 

693 self.rotary_interleaved = module.rotary_interleaved 

694 self.dsa_loss_coeff = module.dsa_loss_coeff 

695 self.freeze_dsa = module.freeze_dsa 

696 self.use_flash_attn = bool(getattr(module, "use_flash_attn", True)) 

697 self.use_mome = bool(getattr(module, "use_mome", False)) 

698 self.use_fused_mome = bool(getattr(module, "use_fused_mome", False)) 

699 self.param_sink_number = int(getattr(module, "param_sink_number", 0)) 

700 self.param_sink_scalar = getattr(module, "param_sink_scalar", None) 

701 self.apply_fa_rescale = bool( 

702 getattr(module, "apply_fa_rescale", False) 

703 ) 

704 self.attention_dropout = getattr(module, "attention_dropout", nn.Dropout(0.0)) 

705 if self.param_sink_number > 0: 

706 if self.param_sink_scalar: 

707 raise NotImplementedError( 

708 "DSAAttention does not support scalar parameter sink" 

709 ) 

710 if not self.apply_fa_rescale: 

711 raise NotImplementedError( 

712 "DSAAttention supports parameter sink through " 

713 "apply_fa_rescale only" 

714 ) 

715 

716 self.linear_qkv = module.linear_qkv 

717 self.q_layernorm = module.q_layernorm 

718 self.k_layernorm = module.k_layernorm 

719 self.linear_qb = module.linear_qb 

720 self.linear_kvb = module.linear_kvb 

721 self.linear_proj = module.linear_proj 

722 self.index_linear_qb = module.index_linear_qb 

723 self.index_linear_k = module.index_linear_k 

724 self.index_k_layernorm = module.index_k_layernorm 

725 self.linear_merge_weight = module.linear_merge_weight 

726 if self.use_mome: 

727 self.qa_conv = module.qa_conv 

728 self.compresskv_conv = module.compresskv_conv 

729 self.o_conv = module.o_conv 

730 if self.param_sink_number > 0: 

731 self.param_sink_k_pe = module.param_sink_k_pe 

732 self.param_sink_compressed_kv = module.param_sink_compressed_kv 

733 self.train(module.training) 

734 

735 @staticmethod 

736 def _get_actual_seq_len( 

737 actual_seq_len: torch.Tensor | Sequence[int] | None, 

738 batch_size: int, 

739 seq_length: int, 

740 device: torch.device, 

741 ) -> torch.Tensor: 

742 """Normalize cumulative sequence lengths for the TND custom operators.""" 

743 if actual_seq_len is None: 

744 return torch.arange( 

745 seq_length, 

746 (batch_size + 1) * seq_length, 

747 seq_length, 

748 dtype=torch.int32, 

749 device=device, 

750 ) 

751 return torch.as_tensor(actual_seq_len, dtype=torch.int32, device=device) 

752 

753 def _prepare_param_sink( 

754 self, batch_size: int 

755 ) -> tuple[torch.Tensor, torch.Tensor]: 

756 """Construct the DSA sink key and value in BSND layout.""" 

757 sink_rotary_key = self.param_sink_k_pe.unsqueeze(0).unsqueeze(2).expand( 

758 batch_size, -1, -1, -1 

759 ) 

760 sink_latent = self.param_sink_compressed_kv.unsqueeze(0).unsqueeze(2).expand( 

761 batch_size, -1, -1, -1 

762 ) 

763 sink_latent = self.k_layernorm(sink_latent) 

764 sink_value = ( 

765 F.pad(sink_latent, (0, self.qk_rope_head_dim)) 

766 if self.use_flash_attn and self.qk_rope_head_dim > 0 

767 else sink_latent 

768 ) 

769 sink_key = torch.cat((sink_latent, sink_rotary_key), dim=-1) 

770 return sink_key, sink_value 

771 

772 @staticmethod 

773 def _validate_forward_inputs( 

774 attention_mask: torch.Tensor | None, 

775 kv_reuse_states: Any | None, 

776 past_key_values: Any | None, 

777 cache_position: torch.Tensor | None, 

778 output_attentions: bool, 

779 ) -> None: 

780 """Validate inputs supported by the optimized DSA path.""" 

781 if attention_mask is not None: 

782 raise ValueError( 

783 "DSAAttention does not consume attention_mask; pass packed sequence " 

784 "boundaries through actual_seq_len" 

785 ) 

786 if kv_reuse_states is not None: 

787 raise NotImplementedError("DSAAttention does not support KV reuse") 

788 if past_key_values is not None or cache_position is not None: 

789 raise NotImplementedError("DSAAttention does not support KV cache") 

790 if output_attentions: 

791 raise NotImplementedError("DSAAttention does not expose attention weights") 

792 

793 def _project_attention_states( 

794 self, 

795 hidden_states: torch.Tensor, 

796 mome_mask: torch.Tensor | None, 

797 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: 

798 """Project hidden states into absorbed-query and latent KV states.""" 

799 batch_size, seq_length = hidden_states.shape[:-1] 

800 latent_states, _ = self.linear_qkv(hidden_states) 

801 q_latent, kv_nope, k_rot = torch.split( 

802 latent_states, (self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim), dim=-1 

803 ) 

804 if self.use_mome: 

805 q_latent = apply_mome(q_latent, mome_mask, self.qa_conv, fused=False) 

806 kv_nope = apply_mome(kv_nope, mome_mask, self.compresskv_conv, fused=False) 

807 q_resid = self.q_layernorm(q_latent) 

808 q_states, _ = self.linear_qb(q_resid.contiguous()) 

809 q_states = q_states.view(batch_size, seq_length, self.num_heads, self.qk_head_dim) 

810 q_pass, q_rot = torch.split(q_states, (self.qk_nope_head_dim, self.qk_rope_head_dim), dim=-1) 

811 kv_weight = self.linear_kvb.weight.view( 

812 self.num_heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank 

813 ) 

814 absorbed_query = _absorb_query_projection( 

815 q_pass, 

816 kv_weight[:, : self.qk_nope_head_dim], 

817 num_heads=self.num_heads, 

818 batch_size=batch_size, 

819 seq_length=seq_length, 

820 input_dim=self.qk_nope_head_dim, 

821 output_dim=self.kv_lora_rank, 

822 ) 

823 kv_nope = self.k_layernorm(kv_nope).view(batch_size, seq_length, 1, self.kv_lora_rank) 

824 k_rot = k_rot.view(batch_size, seq_length, 1, self.qk_rope_head_dim) 

825 return q_resid, absorbed_query, kv_nope, q_rot, k_rot, kv_weight 

826 

827 def _project_index_states( 

828 self, 

829 hidden_states: torch.Tensor, 

830 q_resid: torch.Tensor, 

831 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, 

832 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 

833 """Build indexer states and apply their rotary embedding.""" 

834 batch_size, seq_length = hidden_states.shape[:-1] 

835 index_query, _ = self.index_linear_qb(q_resid.detach()) 

836 index_query = index_query.view(batch_size, seq_length, self.num_index_heads, self.index_head_dim) 

837 index_key, _ = self.index_linear_k(hidden_states.detach()) 

838 index_key = self.index_k_layernorm(index_key.unsqueeze(2)) 

839 merge_weight, _ = self.linear_merge_weight(hidden_states.detach()) 

840 merge_weight = merge_weight * self.num_index_heads**-0.5 * self.index_head_dim**-0.5 

841 index_query, index_key = _apply_index_rope( 

842 index_query, 

843 index_key, 

844 position_embeddings, 

845 rope_head_dim=self.qk_rope_head_dim, 

846 index_head_dim=self.index_head_dim, 

847 interleaved=self.rotary_interleaved, 

848 ) 

849 return index_query, index_key, merge_weight 

850 

851 def _run_sparse_attention( 

852 self, 

853 attention_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], 

854 topk_indices: torch.Tensor, 

855 actual_q_len: torch.Tensor, 

856 actual_kv_len: torch.Tensor, 

857 batch_size: int, 

858 seq_length: int, 

859 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 

860 """Run sparse attention with or without parameter-sink rescaling.""" 

861 absorbed_query, kv_nope, q_rot, k_rot = attention_states 

862 sparse_scale = self.qk_head_dim**-0.5 

863 if self.param_sink_number <= 0: 

864 return dsa_sparse_attention( # pylint: disable=not-callable 

865 absorbed_query, kv_nope, q_rot, k_rot, topk_indices, 

866 sparse_scale, actual_q_len, actual_kv_len, 

867 ) 

868 sink_key, sink_value = self._prepare_param_sink(batch_size) 

869 return dsa_sparse_attention_rescale( # pylint: disable=not-callable 

870 absorbed_query, kv_nope, q_rot, k_rot, sink_key, sink_value, 

871 topk_indices, batch_size, seq_length, self.num_heads, sparse_scale, 

872 1 - self.attention_dropout.p, actual_q_len, actual_kv_len, 

873 ) 

874 

875 def _apply_auxiliary_loss( 

876 self, 

877 attn_output: torch.Tensor, 

878 index_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor], 

879 attention_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], 

880 topk_indices: torch.Tensor, 

881 softmax_stats: tuple[torch.Tensor, torch.Tensor], 

882 actual_q_len: torch.Tensor, 

883 actual_kv_len: torch.Tensor, 

884 ) -> torch.Tensor: 

885 """Attach the DSA KL auxiliary loss when training enables it.""" 

886 if not (self.training and not self.freeze_dsa and self.dsa_loss_coeff): 

887 return attn_output 

888 query_tnd, key_tnd, q_rot_tnd, k_rot_tnd = ( 

889 tensor.reshape(-1, tensor.shape[2], tensor.shape[3]) 

890 for tensor in attention_states 

891 ) 

892 aux_loss = dsa_kl_loss( # pylint: disable=not-callable 

893 *index_states, query_tnd, key_tnd, 

894 topk_indices, *softmax_stats, q_rot_tnd, k_rot_tnd, 

895 actual_q_len, actual_kv_len, self.qk_head_dim**-0.5, self.dsa_loss_coeff, 

896 ) 

897 return aux_loss_auto_scale(attn_output, aux_loss) 

898 

899 def _prepare_attention_states(self, hidden_states, position_embeddings, mome_mask): 

900 """Project attention states and apply their rotary embeddings.""" 

901 q_resid, absorbed_query_states, kv_nope, q_rot, k_rot, kv_weight = self._project_attention_states( 

902 hidden_states, mome_mask 

903 ) 

904 q_rot, k_rot = _apply_attention_rope( 

905 q_rot, k_rot, position_embeddings, interleaved=self.rotary_interleaved 

906 ) 

907 return q_resid, (absorbed_query_states, kv_nope, q_rot, k_rot), kv_weight 

908 

909 def _run_indexer(self, hidden_states, q_resid, position_embeddings, actual_seq_len, kwargs): 

910 """Resolve packed boundaries and select sparse attention indices.""" 

911 index_states = self._project_index_states(hidden_states, q_resid, position_embeddings) 

912 packed_kwargs = dict(kwargs) 

913 packed_kwargs["actual_seq_len"] = actual_seq_len 

914 actual_q_len, actual_kv_len = resolve_packed_sequence_lengths( 

915 packed_kwargs, 

916 hidden_states.shape[0] * hidden_states.shape[1], 

917 hidden_states.shape[0] * hidden_states.shape[1], 

918 ) 

919 actual_q_len = self._get_actual_seq_len( 

920 actual_q_len, *hidden_states.shape[:-1], hidden_states.device, 

921 ) 

922 actual_kv_len = self._get_actual_seq_len( 

923 actual_kv_len, *hidden_states.shape[:-1], hidden_states.device, 

924 ) 

925 indexed = dsa_indexer( # pylint: disable=not-callable 

926 *index_states, actual_q_len, actual_kv_len, self.index_topk 

927 ) 

928 return indexed[0], indexed[1:], actual_q_len, actual_kv_len 

929 

930 def _compute_sparse_attention( 

931 self, hidden_states, q_resid, attention_states, position_embeddings, actual_seq_len, kwargs, 

932 ): 

933 """Run sparse attention and attach its optional auxiliary loss.""" 

934 topk_indices, index_states, actual_q_len, actual_kv_len = self._run_indexer( 

935 hidden_states, q_resid, position_embeddings, actual_seq_len, kwargs, 

936 ) 

937 attn_output, softmax_max, softmax_sum = self._run_sparse_attention( 

938 attention_states, topk_indices, actual_q_len, actual_kv_len, *hidden_states.shape[:-1], 

939 ) 

940 return self._apply_auxiliary_loss( 

941 attn_output, index_states, attention_states, topk_indices, 

942 (softmax_max, softmax_sum), actual_q_len, actual_kv_len, 

943 ) 

944 

945 def _compute_attention_output( 

946 self, hidden_states, position_embeddings, actual_seq_len, kwargs, mome_mask, 

947 ): 

948 """Compute sparse attention and restore its value projection.""" 

949 batch_size, seq_length = hidden_states.shape[:-1] 

950 q_resid, attention_states, kv_weight = self._prepare_attention_states( 

951 hidden_states, position_embeddings, mome_mask, 

952 ) 

953 attn_output = self._compute_sparse_attention( 

954 hidden_states, q_resid, attention_states, position_embeddings, actual_seq_len, kwargs, 

955 ) 

956 value_up_weight = kv_weight[:, self.qk_nope_head_dim :].transpose(1, 2) 

957 return _restore_attention_projection( 

958 attn_output, 

959 value_up_weight, 

960 num_heads=self.num_heads, 

961 batch_size=batch_size, 

962 seq_length=seq_length, 

963 kv_lora_rank=self.kv_lora_rank, 

964 value_head_dim=self.v_head_dim, 

965 ) 

966 

967 def forward( 

968 self, 

969 hidden_states: torch.Tensor, 

970 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, 

971 attention_mask: torch.Tensor | None = None, 

972 past_key_values: Any | None = None, 

973 cache_position: torch.Tensor | None = None, 

974 actual_seq_len: torch.Tensor | Sequence[int] | None = None, 

975 kv_reuse_states: Any | None = None, 

976 output_attentions: bool = False, 

977 return_bias: bool = False, 

978 mome_mask: torch.Tensor | None = None, 

979 **kwargs: Any, 

980 ) -> tuple[torch.Tensor, torch.Tensor | None]: 

981 """Run DSA with its configured MOME and parameter-sink paths.""" 

982 self._validate_forward_inputs( 

983 attention_mask, kv_reuse_states, past_key_values, cache_position, output_attentions 

984 ) 

985 attn_output = self._compute_attention_output( 

986 hidden_states, position_embeddings, actual_seq_len, kwargs, mome_mask, 

987 ) 

988 if self.use_mome: 

989 attn_output = apply_mome( 

990 attn_output, 

991 mome_mask, 

992 self.o_conv, 

993 fused=self.use_fused_mome, 

994 ) 

995 output, bias = self.linear_proj(attn_output) 

996 return (output, bias) if return_bias else (output, None)