Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / dsa_attention.py: 0%
405 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Model-compatible DSA modules using the Hyper high-performance operators."""
17from __future__ import annotations
19from collections.abc import Mapping, Sequence
20from typing import Any
22# This package provides PyTorch-specific high-performance modules.
23import torch
24from torch import nn
25from torch.nn import functional as F
26from hyper_parallel.components.checkpoint.weight_conversion import WeightConverter
28from hyper_parallel.components.checkpoint import ConcatenateWithSections
29from hyper_parallel.models.replacement import module_replacement
30from hyper_parallel.components.functional import aggregate_hidden, aux_loss_auto_scale
31from hyper_parallel.components.functional import apply_rotary_pos_emb, apply_rotary_pos_emb_interleave
32from hyper_parallel.components.functional import (
33 dsa_indexer,
34 dsa_kl_loss,
35 dsa_sparse_attention,
36 dsa_sparse_attention_rescale,
37)
38from hyper_parallel.components.functional.npu_fusion_attention import resolve_packed_sequence_lengths
41def _dense_flash_attention(
42 module: nn.Module,
43 query: torch.Tensor,
44 key: torch.Tensor,
45 value: torch.Tensor,
46 sequence_lengths: tuple[torch.Tensor, torch.Tensor],
47 *,
48 need_stats: bool,
49) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
50 """Run one TND FA and return dense-indexer statistics."""
51 query_ends, key_ends = (lengths.tolist() for lengths in sequence_lengths)
52 q_flat, k_flat, v_flat = (tensor.flatten(0, 1) for tensor in (query, key, value))
53 num_heads = q_flat.shape[1]
54 causal_mask = torch.ones((2048, 2048), dtype=torch.bool, device=query.device).triu(diagonal=1)
55 import torch_npu # pylint: disable=import-outside-toplevel
57 result = torch_npu.npu_fusion_attention(
58 q_flat.contiguous(), k_flat.contiguous(), v_flat.contiguous(), num_heads, "TND",
59 pse=None, padding_mask=None, atten_mask=causal_mask, scale=module.scaling,
60 pre_tockens=getattr(module, "pre_tockens", 1048576),
61 next_tockens=getattr(module, "next_tockens", 0), keep_prob=1.0, inner_precise=0,
62 actual_seq_qlen=query_ends, actual_seq_kvlen=key_ends, sparse_mode=3,
63 )
64 output = result[0].reshape(*query.shape[:3], value.shape[-1])
65 if not need_stats:
66 return output, None, None
68 # TND FA stores each packed segment as [head, token, 8], while exposing a
69 # physical [token, head, 8] shape. Dense KL consumes [head, token, 1].
70 statistics = []
71 starts = [0, *query_ends[:-1]]
72 for statistic in result[1:3]:
73 rows = statistic.reshape(-1, statistic.shape[-1])
74 segments = [
75 rows[start * num_heads : end * num_heads].reshape(
76 num_heads, end - start, statistic.shape[-1]
77 )[..., :1]
78 for start, end in zip(starts, query_ends)
79 ]
80 statistics.append(torch.cat(segments, dim=1).detach())
81 return output, statistics[0], statistics[1]
84def _deepseek_v32_dense_attention(
85 module: nn.Module,
86 index_states: tuple[torch.Tensor, ...],
87 attention_states: tuple[torch.Tensor, ...],
88 sequence_lengths: tuple[torch.Tensor, torch.Tensor],
89) -> torch.Tensor:
90 """Run unabsorbed MLA FA and the fused dense indexer KL on local states."""
91 query, compressed_kv, query_rope, key_rope = attention_states
92 projected_kv = module.kv_b_proj(compressed_kv.squeeze(2)).view(
93 *compressed_kv.shape[:2], module.num_heads, module.qk_nope_head_dim + module.v_head_dim,
94 )
95 key, value = projected_kv.split((module.qk_nope_head_dim, module.v_head_dim), dim=-1)
96 key_rope = key_rope.expand(-1, -1, module.num_heads, -1)
97 need_stats = bool(module.training and not module.freeze_dsa and module.dsa_loss_coeff)
98 output, maximum, total = _dense_flash_attention(
99 module, torch.cat((query, query_rope), dim=-1), torch.cat((key, key_rope), dim=-1),
100 value, sequence_lengths, need_stats=need_stats,
101 )
102 if need_stats:
103 # Dense CANN operators are optional for sparse and Omni execution.
104 # pylint: disable=import-outside-toplevel
105 from hyper_parallel.components.functional.dsa_dense import (
106 dsa_dense_indexer_kl_loss,
107 dsa_dense_indexer_lse,
108 )
109 # pylint: enable=import-outside-toplevel
111 query_lengths, key_lengths = sequence_lengths
112 index_query, index_key, weights = (tensor.flatten(0, 1) for tensor in index_states)
113 indexer_stats = dsa_dense_indexer_lse(
114 index_query, index_key, weights, query_lengths=query_lengths, key_lengths=key_lengths,
115 )
116 loss = dsa_dense_indexer_kl_loss(
117 index_query, index_key, weights, query.flatten(0, 1), key.flatten(0, 1),
118 maximum, total, indexer_stats, scale=module.scaling,
119 query_rope=query_rope.flatten(0, 1), key_rope=key_rope.flatten(0, 1),
120 query_lengths=query_lengths, key_lengths=key_lengths, loss_coefficient=module.dsa_loss_coeff,
121 )
122 output = aux_loss_auto_scale(output, loss)
123 return output
126def apply_mome(
127 hidden_states: torch.Tensor,
128 mome_mask: torch.Tensor | None,
129 convolution: nn.Conv1d,
130 *,
131 fused: bool,
132) -> torch.Tensor:
133 """Apply masked causal depthwise convolution and residual."""
134 if mome_mask is None:
135 raise ValueError("mome_mask is required when MOME is enabled")
136 if mome_mask.shape != hidden_states.shape[:2]:
137 raise ValueError(
138 f"mome_mask must have shape {hidden_states.shape[:2]}, "
139 f"but got {tuple(mome_mask.shape)}"
140 )
142 mome_mask = mome_mask.to(device=hidden_states.device, dtype=torch.bool)
143 padding = convolution.kernel_size[0] - 1
144 padded_states = F.pad(hidden_states, (0, 0, padding, 0))
145 if fused:
146 padded_mask = F.pad(mome_mask, (padding, 0), value=False)
147 weight = convolution.weight.squeeze(1).transpose(0, 1)
148 mixed_states = aggregate_hidden( # pylint: disable=not-callable
149 padded_states.transpose(0, 1).contiguous(),
150 weight,
151 padded_mask,
152 )
153 mixed_states = mixed_states[padding:].transpose(0, 1).contiguous()
154 else:
155 mixed_states = convolution(padded_states.transpose(1, 2)).transpose(1, 2)
156 mixed_states = mixed_states * mome_mask.unsqueeze(-1).to(mixed_states.dtype)
157 return hidden_states + mixed_states
160def _apply_attention_rope(
161 query: torch.Tensor,
162 key: torch.Tensor,
163 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
164 *,
165 interleaved: bool,
166) -> tuple[torch.Tensor, torch.Tensor]:
167 """Apply the configured RoPE convention to an attention query/key pair."""
168 if position_embeddings is None:
169 return query, key
170 cos, sin = position_embeddings
171 rotary_op = apply_rotary_pos_emb_interleave if interleaved else apply_rotary_pos_emb
172 return rotary_op(query, key, cos, sin, unsqueeze_dim=2)
175def _apply_index_rope(
176 query: torch.Tensor,
177 key: torch.Tensor,
178 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
179 *,
180 rope_head_dim: int,
181 index_head_dim: int,
182 interleaved: bool,
183) -> tuple[torch.Tensor, torch.Tensor]:
184 """Apply RoPE to the rotary portion of an indexer query/key pair."""
185 if position_embeddings is None:
186 return query, key
187 cos, sin = position_embeddings
188 split_sizes = (rope_head_dim, index_head_dim - rope_head_dim)
189 query_rot, query_pass = torch.split(query, split_sizes, dim=-1)
190 key_rot, key_pass = torch.split(key, split_sizes, dim=-1)
191 rotary_op = apply_rotary_pos_emb_interleave if interleaved else apply_rotary_pos_emb
192 query_rot, key_rot = rotary_op(query_rot, key_rot, cos, sin, unsqueeze_dim=2)
193 return torch.cat((query_rot, query_pass), dim=-1), torch.cat((key_rot, key_pass), dim=-1)
196def _absorb_query_projection(
197 query: torch.Tensor,
198 key_up_weight: torch.Tensor,
199 *,
200 num_heads: int,
201 batch_size: int,
202 seq_length: int,
203 input_dim: int,
204 output_dim: int,
205) -> torch.Tensor:
206 """Absorb the key up-projection weight into query states."""
207 query = query.permute(2, 0, 1, 3).reshape(num_heads, batch_size * seq_length, input_dim)
208 query = torch.bmm(query, key_up_weight).view(num_heads, batch_size, seq_length, output_dim)
209 return query.permute(1, 2, 0, 3)
212def _restore_attention_projection(
213 output: torch.Tensor,
214 value_up_weight: torch.Tensor,
215 *,
216 num_heads: int,
217 batch_size: int,
218 seq_length: int,
219 kv_lora_rank: int,
220 value_head_dim: int,
221) -> torch.Tensor:
222 """Apply the absorbed value up-projection and restore the hidden layout."""
223 output = output[..., :kv_lora_rank]
224 output = output.permute(2, 0, 1, 3).reshape(num_heads, batch_size * seq_length, kv_lora_rank)
225 output = torch.bmm(output, value_up_weight)
226 return output.view(num_heads, batch_size, seq_length, value_head_dim).permute(1, 2, 0, 3).reshape(
227 batch_size, seq_length, -1
228 )
231@module_replacement
232class DeepseekV32DSAAttention(nn.Module):
233 """NPU DSA replacement for Transformers ``DeepseekV32Attention``.
235 It supports causal training and packed sequences. Packed boundaries must
236 be supplied through ``actual_seq_len``. The optional ``attention_mask`` is
237 assumed to be the standard causal mask created by Transformers; padding,
238 custom attention masks, dropout, and KV cache are not supported.
239 """
241 @staticmethod
242 def _validate_source(module: nn.Module) -> None:
243 """Validate the required DSA module and indexer surface."""
244 required = (
245 "q_a_proj", "q_a_layernorm", "q_b_proj", "kv_a_proj_with_mqa",
246 "kv_a_layernorm", "kv_b_proj", "o_proj", "indexer",
247 )
248 missing = [name for name in required if not hasattr(module, name)]
249 if missing:
250 raise TypeError(f"DeepseekV32DSAAttention source module is missing: {missing}")
251 indexer_required = ("wq_b", "wk", "k_norm", "weights_proj")
252 indexer_missing = [name for name in indexer_required if not hasattr(module.indexer, name)]
253 if indexer_missing:
254 raise TypeError(f"DeepseekV32DSAAttention source indexer is missing: {indexer_missing}")
256 def _initialize_dimensions(self, module: nn.Module) -> None:
257 """Copy main-attention and indexer dimensions from the source module."""
258 config = self.config
259 self.layer_idx = getattr(module, "layer_idx", getattr(module, "layer_number", None))
260 self.num_heads = getattr(module, "num_heads", config.num_attention_heads)
261 self.q_lora_rank = getattr(module, "q_lora_rank", config.q_lora_rank)
262 self.kv_lora_rank = getattr(module, "kv_lora_rank", config.kv_lora_rank)
263 self.qk_rope_head_dim = getattr(module, "qk_rope_head_dim", config.qk_rope_head_dim)
264 self.qk_nope_head_dim = getattr(module, "qk_nope_head_dim", config.qk_nope_head_dim)
265 self.v_head_dim = getattr(module, "v_head_dim", config.v_head_dim)
266 self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
267 if self.q_lora_rank is None:
268 raise ValueError("DeepseekV32DSAAttention requires the Q-LoRA projection path")
269 self.index_head_dim = getattr(module.indexer, "head_dim", config.index_head_dim)
270 self.num_index_heads = getattr(
271 module.indexer,
272 "n_heads",
273 getattr(config, "index_n_heads", getattr(config, "index_num_attention_heads", None)),
274 )
275 if self.num_index_heads is None:
276 raise ValueError("DeepseekV32DSAAttention requires the number of indexer heads")
277 self.index_topk = getattr(module.indexer, "index_topk", config.index_topk)
279 def _initialize_options(self, module: nn.Module) -> None:
280 """Copy supported DSA options and reject unsupported source modes."""
281 config = self.config
282 self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5)
283 attention_dropout = getattr(module, "attention_dropout", getattr(config, "attention_dropout", 0.0))
284 if isinstance(attention_dropout, nn.Dropout):
285 attention_dropout = attention_dropout.p
286 if attention_dropout != 0.0:
287 raise ValueError(
288 "DeepseekV32DSAAttention requires attention_dropout=0 because "
289 "the sparse-attention operator does not apply dropout"
290 )
291 source_rotary_interleaved = getattr(module, "rotary_interleaved", None)
292 if source_rotary_interleaved is None:
293 source_rotary_interleaved = getattr(config, "rope_interleave", None)
294 if source_rotary_interleaved is None:
295 source_rotary_interleaved = getattr(config, "rope_interleaved", None)
296 self.rotary_interleaved = bool(True if source_rotary_interleaved is None else source_rotary_interleaved)
297 self.index_rotary_interleaved = bool(
298 getattr(module.indexer, "rotary_interleaved", getattr(config, "index_rope_interleaved", False))
299 )
300 self.dsa_loss_coeff = getattr(config, "dsa_loss_coeff", 0.0)
301 self.freeze_dsa = bool(getattr(config, "freeze_DSA", getattr(config, "freeze_dsa", False)))
302 self.dsa_dense_warm_up = bool(
303 getattr(module, "dsa_dense_warm_up", getattr(config, "dsa_dense_warm_up", False))
304 )
305 if getattr(module, "param_sink_number", 0) > 0:
306 raise ValueError("DeepseekV32DSAAttention does not support the parameter-sink path")
307 if getattr(module, "use_mome", False):
308 raise ValueError("DeepseekV32DSAAttention does not support the MOME path")
310 def _initialize_source_layers(self, module: nn.Module) -> None:
311 """Fuse latent projections and attach the remaining source layers."""
312 q_a = module.q_a_proj
313 kv_a = module.kv_a_proj_with_mqa
314 self._q_latent_output_size = q_a.out_features
315 self._kv_latent_output_size = kv_a.out_features
316 can_fuse = (
317 q_a.in_features == kv_a.in_features
318 and q_a.weight.requires_grad == kv_a.weight.requires_grad
319 and (q_a.bias is None) == (kv_a.bias is None)
320 and (q_a.bias is None or q_a.bias.requires_grad == kv_a.bias.requires_grad)
321 )
322 if not can_fuse:
323 raise ValueError("Q and KV latent projections cannot be represented by one fused projection")
324 self.linear_qkv = nn.Linear(
325 q_a.in_features,
326 q_a.out_features + kv_a.out_features,
327 bias=q_a.bias is not None,
328 device=q_a.weight.device,
329 dtype=q_a.weight.dtype,
330 )
331 self.linear_qkv.weight.requires_grad_(q_a.weight.requires_grad)
332 if q_a.bias is not None:
333 self.linear_qkv.bias.requires_grad_(q_a.bias.requires_grad)
334 for name in ("q_a_layernorm", "kv_a_layernorm", "q_b_proj", "kv_b_proj", "o_proj", "indexer"):
335 setattr(self, name, getattr(module, name))
337 def __init__(
338 self,
339 *,
340 module: nn.Module,
341 module_fqn: str = "",
342 context: Mapping[str, Any] | None = None,
343 ) -> None:
344 """Build the high-performance module from a Transformers DSA module."""
345 super().__init__()
346 del module_fqn, context
347 self._validate_source(module)
348 config = module.config
349 self.config = config
350 if hasattr(config, "indexer_types"):
351 raise TypeError(
352 "DeepseekV32DSAAttention supports the DeepSeek-V3.2 DSA contract; "
353 "GLM-MOE-DSA shared indexers and three-value forward result require "
354 "a dedicated replacement"
355 )
356 self._initialize_dimensions(module)
357 self._initialize_options(module)
358 self._initialize_source_layers(module)
359 self.train(module.training)
361 def make_transforms(self) -> list[WeightConverter]:
362 """Describe reversible source-checkpoint to high-performance conversion."""
363 transforms: list[WeightConverter] = [
364 WeightConverter(
365 source_patterns=[
366 "q_a_proj.weight",
367 "kv_a_proj_with_mqa.weight",
368 ],
369 target_patterns="linear_qkv.weight",
370 operations=[
371 ConcatenateWithSections(
372 sections=(
373 self._q_latent_output_size,
374 self._kv_latent_output_size,
375 ),
376 dim=0,
377 )
378 ],
379 ),
380 ]
381 if self.linear_qkv.bias is not None:
382 transforms.insert(
383 1,
384 WeightConverter(
385 source_patterns=["q_a_proj.bias", "kv_a_proj_with_mqa.bias"],
386 target_patterns="linear_qkv.bias",
387 operations=[
388 ConcatenateWithSections(
389 sections=(
390 self._q_latent_output_size,
391 self._kv_latent_output_size,
392 ),
393 dim=0,
394 )
395 ],
396 ),
397 )
398 return transforms
400 @staticmethod
401 def _get_actual_seq_len(
402 actual_seq_len: torch.Tensor | Sequence[int] | None,
403 batch_size: int,
404 seq_length: int,
405 device: torch.device,
406 ) -> torch.Tensor:
407 """Normalize cumulative sequence lengths for the TND custom operators."""
408 if isinstance(actual_seq_len, torch.Tensor):
409 return actual_seq_len.to(device=device, dtype=torch.int32)
410 if actual_seq_len is not None:
411 return torch.tensor(actual_seq_len, device=device, dtype=torch.int32)
412 return torch.arange(
413 seq_length,
414 (batch_size + 1) * seq_length,
415 seq_length,
416 dtype=torch.int32,
417 device=device,
418 )
420 @staticmethod
421 def _invalid_position_ids(
422 position_ids: torch.Tensor | None,
423 batch_size: int,
424 seq_length: int,
425 ) -> bool:
426 """Return whether position IDs violate the supported dense layout."""
427 if position_ids is None:
428 return False
429 if position_ids.ndim != 2:
430 return True
431 return position_ids.shape[0] not in (1, batch_size) or position_ids.shape[1] != seq_length
433 @staticmethod
434 def _validate_forward_inputs(
435 hidden_states: torch.Tensor,
436 attention_mask: torch.Tensor | None,
437 past_key_values: Any | None,
438 position_ids: torch.Tensor | None,
439 kwargs: dict[str, Any],
440 ) -> None:
441 """Validate inputs supported by the DeepSeek sparse-attention path."""
442 batch_size, seq_length = hidden_states.shape[:-1]
443 if past_key_values is not None:
444 raise NotImplementedError("DeepseekV32DSAAttention does not support KV cache")
445 if DeepseekV32DSAAttention._invalid_position_ids(position_ids, batch_size, seq_length):
446 raise ValueError("position_ids must have shape [1 or batch_size, sequence_length]")
447 if attention_mask is not None:
448 expected_shape = (batch_size, 1, seq_length, seq_length)
449 if tuple(attention_mask.shape) != expected_shape:
450 raise ValueError(
451 "DeepseekV32DSAAttention supports only a Transformers 4D causal mask; "
452 "padding and custom masks are not supported"
453 )
454 if kwargs.pop("output_attentions", False):
455 raise NotImplementedError("DeepseekV32DSAAttention does not expose attention weights")
457 def _project_attention_states(
458 self,
459 hidden_states: torch.Tensor,
460 *,
461 use_dense: bool = False,
462 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
463 """Project MLA Q/KV, retaining Q before absorption during dense warm-up."""
464 batch_size, seq_length = hidden_states.shape[:-1]
465 latent_states = self.linear_qkv(hidden_states)
466 q_latent, kv_nope, k_rot = torch.split(
467 latent_states, (self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim), dim=-1
468 )
469 q_resid = self.q_a_layernorm(q_latent)
470 q_states = self.q_b_proj(q_resid).view(batch_size, seq_length, self.num_heads, self.qk_head_dim)
471 q_pass, q_rot = torch.split(q_states, (self.qk_nope_head_dim, self.qk_rope_head_dim), dim=-1)
472 kv_weight = self.kv_b_proj.weight.view(
473 self.num_heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank
474 )
475 absorbed_query = q_pass if use_dense else _absorb_query_projection(
476 q_pass,
477 kv_weight[:, : self.qk_nope_head_dim],
478 num_heads=self.num_heads,
479 batch_size=batch_size,
480 seq_length=seq_length,
481 input_dim=self.qk_nope_head_dim,
482 output_dim=self.kv_lora_rank,
483 )
484 kv_nope = self.kv_a_layernorm(kv_nope).view(batch_size, seq_length, 1, self.kv_lora_rank)
485 k_rot = k_rot.view(batch_size, seq_length, 1, self.qk_rope_head_dim)
486 return q_resid, absorbed_query, kv_nope, q_rot, k_rot, kv_weight
488 def _project_index_states(
489 self,
490 hidden_states: torch.Tensor,
491 q_resid: torch.Tensor,
492 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
493 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
494 """Build indexer query, key, and merge-weight states."""
495 batch_size, seq_length = hidden_states.shape[:-1]
496 index_query = self.indexer.wq_b(q_resid.detach()).view(
497 batch_size, seq_length, self.num_index_heads, self.index_head_dim
498 )
499 index_key = self.indexer.k_norm(self.indexer.wk(hidden_states.detach())).unsqueeze(2)
500 merge_weight = self.indexer.weights_proj(hidden_states.detach())
501 merge_weight = merge_weight * self.num_index_heads**-0.5 * self.index_head_dim**-0.5
502 index_query, index_key = _apply_index_rope(
503 index_query,
504 index_key,
505 position_embeddings,
506 rope_head_dim=self.qk_rope_head_dim,
507 index_head_dim=self.index_head_dim,
508 interleaved=self.index_rotary_interleaved,
509 )
510 return index_query, index_key, merge_weight
512 def _apply_auxiliary_loss(
513 self,
514 attn_output: torch.Tensor,
515 index_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
516 attention_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor],
517 topk_indices: torch.Tensor,
518 softmax_stats: tuple[torch.Tensor, torch.Tensor],
519 actual_q_len: torch.Tensor,
520 actual_kv_len: torch.Tensor,
521 ) -> torch.Tensor:
522 """Attach the DSA KL auxiliary loss when training enables it."""
523 if not (self.training and not self.freeze_dsa and self.dsa_loss_coeff):
524 return attn_output
525 query_tnd, key_tnd, q_rot_tnd, k_rot_tnd = (
526 tensor.reshape(-1, tensor.shape[2], tensor.shape[3])
527 for tensor in attention_states
528 )
529 aux_loss = dsa_kl_loss(
530 *index_states, query_tnd, key_tnd,
531 topk_indices, *softmax_stats, q_rot_tnd, k_rot_tnd,
532 actual_q_len, actual_kv_len, self.scaling, self.dsa_loss_coeff,
533 backend="cann",
534 )
535 return aux_loss_auto_scale(attn_output, aux_loss)
537 def _prepare_attention_states(self, hidden_states, position_embeddings):
538 """Project attention states and apply their rotary embeddings."""
539 q_resid, absorbed_query_states, kv_nope, q_rot, k_rot, kv_weight = self._project_attention_states(
540 hidden_states, use_dense=self.dsa_dense_warm_up,
541 )
542 q_rot, k_rot = _apply_attention_rope(
543 q_rot, k_rot, position_embeddings, interleaved=self.rotary_interleaved
544 )
545 return q_resid, (absorbed_query_states, kv_nope, q_rot, k_rot), kv_weight
547 def _run_indexer(self, hidden_states, q_resid, position_embeddings, actual_seq_len, kwargs):
548 """Resolve packed boundaries and select sparse attention indices."""
549 index_states = self._project_index_states(hidden_states, q_resid, position_embeddings)
550 packed_kwargs = dict(kwargs)
551 packed_kwargs["actual_seq_len"] = actual_seq_len
552 actual_q_len, actual_kv_len = resolve_packed_sequence_lengths(
553 packed_kwargs,
554 hidden_states.shape[0] * hidden_states.shape[1],
555 hidden_states.shape[0] * hidden_states.shape[1],
556 )
557 actual_q_len = self._get_actual_seq_len(
558 actual_q_len, *hidden_states.shape[:-1], hidden_states.device,
559 )
560 actual_kv_len = self._get_actual_seq_len(
561 actual_kv_len, *hidden_states.shape[:-1], hidden_states.device,
562 )
563 indexed = dsa_indexer(
564 *index_states, actual_q_len, actual_kv_len, self.index_topk, backend="cann",
565 )
566 return indexed[0], indexed[1:], actual_q_len, actual_kv_len
568 def _compute_sparse_attention(
569 self, hidden_states, q_resid, attention_states, position_embeddings, actual_seq_len, kwargs,
570 ):
571 """Run sparse attention and attach its optional auxiliary loss."""
572 topk_indices, index_states, actual_q_len, actual_kv_len = self._run_indexer(
573 hidden_states, q_resid, position_embeddings, actual_seq_len, kwargs,
574 )
575 attn_output, softmax_max, softmax_sum = dsa_sparse_attention(
576 *attention_states, topk_indices, self.scaling, actual_q_len, actual_kv_len,
577 backend="cann",
578 )
579 return self._apply_auxiliary_loss(
580 attn_output, index_states, attention_states, topk_indices,
581 (softmax_max, softmax_sum), actual_q_len, actual_kv_len,
582 )
584 def _compute_attention_output(
585 self, hidden_states, position_embeddings, actual_seq_len, kwargs,
586 ):
587 """Run the selected dense/sparse stage and restore the value projection."""
588 batch_size, seq_length = hidden_states.shape[:-1]
589 q_resid, attention_states, kv_weight = self._prepare_attention_states(
590 hidden_states, position_embeddings,
591 )
592 if self.dsa_dense_warm_up:
593 index_states = self._project_index_states(hidden_states, q_resid, position_embeddings)
594 packed_kwargs = {**kwargs, "actual_seq_len": actual_seq_len}
595 lengths = resolve_packed_sequence_lengths(
596 packed_kwargs, batch_size * seq_length, batch_size * seq_length,
597 )
598 lengths = tuple(self._get_actual_seq_len(
599 value, batch_size, seq_length, hidden_states.device,
600 ) for value in lengths)
601 output = _deepseek_v32_dense_attention(self, index_states, attention_states, lengths)
602 return output.flatten(2)
603 attn_output = self._compute_sparse_attention(
604 hidden_states, q_resid, attention_states, position_embeddings, actual_seq_len, kwargs,
605 )
606 value_up_weight = kv_weight[:, self.qk_nope_head_dim :].transpose(1, 2)
607 return _restore_attention_projection(
608 attn_output,
609 value_up_weight,
610 num_heads=self.num_heads,
611 batch_size=batch_size,
612 seq_length=seq_length,
613 kv_lora_rank=self.kv_lora_rank,
614 value_head_dim=self.v_head_dim,
615 )
617 def forward(
618 self,
619 hidden_states: torch.Tensor,
620 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
621 attention_mask: torch.Tensor | None = None,
622 past_key_values: Any | None = None,
623 position_ids: torch.Tensor | None = None,
624 actual_seq_len: torch.Tensor | Sequence[int] | None = None,
625 **kwargs: Any,
626 ) -> tuple[torch.Tensor, None]:
627 """Run causal or packed DSA with the NPU sparse-attention kernels."""
628 self._validate_forward_inputs(hidden_states, attention_mask, past_key_values, position_ids, kwargs)
629 attn_output = self._compute_attention_output(
630 hidden_states, position_embeddings, actual_seq_len, kwargs,
631 )
632 attn_output = self.o_proj(attn_output)
633 return attn_output, None
636@module_replacement
637class DSAAttention(nn.Module):
638 """NPU-optimized replacement for a DSA attention module.
640 Packed boundaries are supplied through ``actual_seq_len``. Attention masks,
641 KV reuse, and KV cache are not supported. The source parameter layout is
642 preserved, so no checkpoint conversion is required.
643 """
645 def __init__(
646 self,
647 *,
648 module: nn.Module,
649 module_fqn: str = "",
650 context: Mapping[str, Any] | None = None,
651 ) -> None:
652 """Build the replacement from a compatible DSA attention module."""
653 super().__init__()
654 del module_fqn, context
655 required = (
656 "linear_qkv",
657 "q_layernorm",
658 "k_layernorm",
659 "linear_qb",
660 "linear_kvb",
661 "linear_proj",
662 "index_linear_qb",
663 "index_linear_k",
664 "index_k_layernorm",
665 "linear_merge_weight",
666 )
667 missing = [name for name in required if not hasattr(module, name)]
668 if missing:
669 raise TypeError(f"DSAAttention source module is missing: {missing}")
671 unsupported = {
672 "mla_mm_split": bool(getattr(module, "mla_mm_split", False)),
673 "dense warm-up": bool(getattr(module, "dsa_dense_warm_up", False)),
674 }
675 enabled = [name for name, value in unsupported.items() if value]
676 if enabled:
677 raise NotImplementedError(
678 f"DSAAttention does not yet support: {', '.join(enabled)}"
679 )
681 self.config = module.config
682 self.layer_number = module.layer_number
683 self.num_heads = module.num_heads
684 self.q_lora_rank = module.q_lora_rank
685 self.kv_lora_rank = module.kv_lora_rank
686 self.qk_rope_head_dim = module.qk_rope_head_dim
687 self.qk_nope_head_dim = module.qk_nope_head_dim
688 self.qk_head_dim = module.qk_head_dim
689 self.v_head_dim = module.v_head_dim
690 self.index_head_dim = module.index_head_dim
691 self.num_index_heads = module.num_index_heads
692 self.index_topk = module.index_topk
693 self.rotary_interleaved = module.rotary_interleaved
694 self.dsa_loss_coeff = module.dsa_loss_coeff
695 self.freeze_dsa = module.freeze_dsa
696 self.use_flash_attn = bool(getattr(module, "use_flash_attn", True))
697 self.use_mome = bool(getattr(module, "use_mome", False))
698 self.use_fused_mome = bool(getattr(module, "use_fused_mome", False))
699 self.param_sink_number = int(getattr(module, "param_sink_number", 0))
700 self.param_sink_scalar = getattr(module, "param_sink_scalar", None)
701 self.apply_fa_rescale = bool(
702 getattr(module, "apply_fa_rescale", False)
703 )
704 self.attention_dropout = getattr(module, "attention_dropout", nn.Dropout(0.0))
705 if self.param_sink_number > 0:
706 if self.param_sink_scalar:
707 raise NotImplementedError(
708 "DSAAttention does not support scalar parameter sink"
709 )
710 if not self.apply_fa_rescale:
711 raise NotImplementedError(
712 "DSAAttention supports parameter sink through "
713 "apply_fa_rescale only"
714 )
716 self.linear_qkv = module.linear_qkv
717 self.q_layernorm = module.q_layernorm
718 self.k_layernorm = module.k_layernorm
719 self.linear_qb = module.linear_qb
720 self.linear_kvb = module.linear_kvb
721 self.linear_proj = module.linear_proj
722 self.index_linear_qb = module.index_linear_qb
723 self.index_linear_k = module.index_linear_k
724 self.index_k_layernorm = module.index_k_layernorm
725 self.linear_merge_weight = module.linear_merge_weight
726 if self.use_mome:
727 self.qa_conv = module.qa_conv
728 self.compresskv_conv = module.compresskv_conv
729 self.o_conv = module.o_conv
730 if self.param_sink_number > 0:
731 self.param_sink_k_pe = module.param_sink_k_pe
732 self.param_sink_compressed_kv = module.param_sink_compressed_kv
733 self.train(module.training)
735 @staticmethod
736 def _get_actual_seq_len(
737 actual_seq_len: torch.Tensor | Sequence[int] | None,
738 batch_size: int,
739 seq_length: int,
740 device: torch.device,
741 ) -> torch.Tensor:
742 """Normalize cumulative sequence lengths for the TND custom operators."""
743 if actual_seq_len is None:
744 return torch.arange(
745 seq_length,
746 (batch_size + 1) * seq_length,
747 seq_length,
748 dtype=torch.int32,
749 device=device,
750 )
751 return torch.as_tensor(actual_seq_len, dtype=torch.int32, device=device)
753 def _prepare_param_sink(
754 self, batch_size: int
755 ) -> tuple[torch.Tensor, torch.Tensor]:
756 """Construct the DSA sink key and value in BSND layout."""
757 sink_rotary_key = self.param_sink_k_pe.unsqueeze(0).unsqueeze(2).expand(
758 batch_size, -1, -1, -1
759 )
760 sink_latent = self.param_sink_compressed_kv.unsqueeze(0).unsqueeze(2).expand(
761 batch_size, -1, -1, -1
762 )
763 sink_latent = self.k_layernorm(sink_latent)
764 sink_value = (
765 F.pad(sink_latent, (0, self.qk_rope_head_dim))
766 if self.use_flash_attn and self.qk_rope_head_dim > 0
767 else sink_latent
768 )
769 sink_key = torch.cat((sink_latent, sink_rotary_key), dim=-1)
770 return sink_key, sink_value
772 @staticmethod
773 def _validate_forward_inputs(
774 attention_mask: torch.Tensor | None,
775 kv_reuse_states: Any | None,
776 past_key_values: Any | None,
777 cache_position: torch.Tensor | None,
778 output_attentions: bool,
779 ) -> None:
780 """Validate inputs supported by the optimized DSA path."""
781 if attention_mask is not None:
782 raise ValueError(
783 "DSAAttention does not consume attention_mask; pass packed sequence "
784 "boundaries through actual_seq_len"
785 )
786 if kv_reuse_states is not None:
787 raise NotImplementedError("DSAAttention does not support KV reuse")
788 if past_key_values is not None or cache_position is not None:
789 raise NotImplementedError("DSAAttention does not support KV cache")
790 if output_attentions:
791 raise NotImplementedError("DSAAttention does not expose attention weights")
793 def _project_attention_states(
794 self,
795 hidden_states: torch.Tensor,
796 mome_mask: torch.Tensor | None,
797 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
798 """Project hidden states into absorbed-query and latent KV states."""
799 batch_size, seq_length = hidden_states.shape[:-1]
800 latent_states, _ = self.linear_qkv(hidden_states)
801 q_latent, kv_nope, k_rot = torch.split(
802 latent_states, (self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim), dim=-1
803 )
804 if self.use_mome:
805 q_latent = apply_mome(q_latent, mome_mask, self.qa_conv, fused=False)
806 kv_nope = apply_mome(kv_nope, mome_mask, self.compresskv_conv, fused=False)
807 q_resid = self.q_layernorm(q_latent)
808 q_states, _ = self.linear_qb(q_resid.contiguous())
809 q_states = q_states.view(batch_size, seq_length, self.num_heads, self.qk_head_dim)
810 q_pass, q_rot = torch.split(q_states, (self.qk_nope_head_dim, self.qk_rope_head_dim), dim=-1)
811 kv_weight = self.linear_kvb.weight.view(
812 self.num_heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank
813 )
814 absorbed_query = _absorb_query_projection(
815 q_pass,
816 kv_weight[:, : self.qk_nope_head_dim],
817 num_heads=self.num_heads,
818 batch_size=batch_size,
819 seq_length=seq_length,
820 input_dim=self.qk_nope_head_dim,
821 output_dim=self.kv_lora_rank,
822 )
823 kv_nope = self.k_layernorm(kv_nope).view(batch_size, seq_length, 1, self.kv_lora_rank)
824 k_rot = k_rot.view(batch_size, seq_length, 1, self.qk_rope_head_dim)
825 return q_resid, absorbed_query, kv_nope, q_rot, k_rot, kv_weight
827 def _project_index_states(
828 self,
829 hidden_states: torch.Tensor,
830 q_resid: torch.Tensor,
831 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
832 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
833 """Build indexer states and apply their rotary embedding."""
834 batch_size, seq_length = hidden_states.shape[:-1]
835 index_query, _ = self.index_linear_qb(q_resid.detach())
836 index_query = index_query.view(batch_size, seq_length, self.num_index_heads, self.index_head_dim)
837 index_key, _ = self.index_linear_k(hidden_states.detach())
838 index_key = self.index_k_layernorm(index_key.unsqueeze(2))
839 merge_weight, _ = self.linear_merge_weight(hidden_states.detach())
840 merge_weight = merge_weight * self.num_index_heads**-0.5 * self.index_head_dim**-0.5
841 index_query, index_key = _apply_index_rope(
842 index_query,
843 index_key,
844 position_embeddings,
845 rope_head_dim=self.qk_rope_head_dim,
846 index_head_dim=self.index_head_dim,
847 interleaved=self.rotary_interleaved,
848 )
849 return index_query, index_key, merge_weight
851 def _run_sparse_attention(
852 self,
853 attention_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor],
854 topk_indices: torch.Tensor,
855 actual_q_len: torch.Tensor,
856 actual_kv_len: torch.Tensor,
857 batch_size: int,
858 seq_length: int,
859 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
860 """Run sparse attention with or without parameter-sink rescaling."""
861 absorbed_query, kv_nope, q_rot, k_rot = attention_states
862 sparse_scale = self.qk_head_dim**-0.5
863 if self.param_sink_number <= 0:
864 return dsa_sparse_attention( # pylint: disable=not-callable
865 absorbed_query, kv_nope, q_rot, k_rot, topk_indices,
866 sparse_scale, actual_q_len, actual_kv_len,
867 )
868 sink_key, sink_value = self._prepare_param_sink(batch_size)
869 return dsa_sparse_attention_rescale( # pylint: disable=not-callable
870 absorbed_query, kv_nope, q_rot, k_rot, sink_key, sink_value,
871 topk_indices, batch_size, seq_length, self.num_heads, sparse_scale,
872 1 - self.attention_dropout.p, actual_q_len, actual_kv_len,
873 )
875 def _apply_auxiliary_loss(
876 self,
877 attn_output: torch.Tensor,
878 index_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
879 attention_states: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor],
880 topk_indices: torch.Tensor,
881 softmax_stats: tuple[torch.Tensor, torch.Tensor],
882 actual_q_len: torch.Tensor,
883 actual_kv_len: torch.Tensor,
884 ) -> torch.Tensor:
885 """Attach the DSA KL auxiliary loss when training enables it."""
886 if not (self.training and not self.freeze_dsa and self.dsa_loss_coeff):
887 return attn_output
888 query_tnd, key_tnd, q_rot_tnd, k_rot_tnd = (
889 tensor.reshape(-1, tensor.shape[2], tensor.shape[3])
890 for tensor in attention_states
891 )
892 aux_loss = dsa_kl_loss( # pylint: disable=not-callable
893 *index_states, query_tnd, key_tnd,
894 topk_indices, *softmax_stats, q_rot_tnd, k_rot_tnd,
895 actual_q_len, actual_kv_len, self.qk_head_dim**-0.5, self.dsa_loss_coeff,
896 )
897 return aux_loss_auto_scale(attn_output, aux_loss)
899 def _prepare_attention_states(self, hidden_states, position_embeddings, mome_mask):
900 """Project attention states and apply their rotary embeddings."""
901 q_resid, absorbed_query_states, kv_nope, q_rot, k_rot, kv_weight = self._project_attention_states(
902 hidden_states, mome_mask
903 )
904 q_rot, k_rot = _apply_attention_rope(
905 q_rot, k_rot, position_embeddings, interleaved=self.rotary_interleaved
906 )
907 return q_resid, (absorbed_query_states, kv_nope, q_rot, k_rot), kv_weight
909 def _run_indexer(self, hidden_states, q_resid, position_embeddings, actual_seq_len, kwargs):
910 """Resolve packed boundaries and select sparse attention indices."""
911 index_states = self._project_index_states(hidden_states, q_resid, position_embeddings)
912 packed_kwargs = dict(kwargs)
913 packed_kwargs["actual_seq_len"] = actual_seq_len
914 actual_q_len, actual_kv_len = resolve_packed_sequence_lengths(
915 packed_kwargs,
916 hidden_states.shape[0] * hidden_states.shape[1],
917 hidden_states.shape[0] * hidden_states.shape[1],
918 )
919 actual_q_len = self._get_actual_seq_len(
920 actual_q_len, *hidden_states.shape[:-1], hidden_states.device,
921 )
922 actual_kv_len = self._get_actual_seq_len(
923 actual_kv_len, *hidden_states.shape[:-1], hidden_states.device,
924 )
925 indexed = dsa_indexer( # pylint: disable=not-callable
926 *index_states, actual_q_len, actual_kv_len, self.index_topk
927 )
928 return indexed[0], indexed[1:], actual_q_len, actual_kv_len
930 def _compute_sparse_attention(
931 self, hidden_states, q_resid, attention_states, position_embeddings, actual_seq_len, kwargs,
932 ):
933 """Run sparse attention and attach its optional auxiliary loss."""
934 topk_indices, index_states, actual_q_len, actual_kv_len = self._run_indexer(
935 hidden_states, q_resid, position_embeddings, actual_seq_len, kwargs,
936 )
937 attn_output, softmax_max, softmax_sum = self._run_sparse_attention(
938 attention_states, topk_indices, actual_q_len, actual_kv_len, *hidden_states.shape[:-1],
939 )
940 return self._apply_auxiliary_loss(
941 attn_output, index_states, attention_states, topk_indices,
942 (softmax_max, softmax_sum), actual_q_len, actual_kv_len,
943 )
945 def _compute_attention_output(
946 self, hidden_states, position_embeddings, actual_seq_len, kwargs, mome_mask,
947 ):
948 """Compute sparse attention and restore its value projection."""
949 batch_size, seq_length = hidden_states.shape[:-1]
950 q_resid, attention_states, kv_weight = self._prepare_attention_states(
951 hidden_states, position_embeddings, mome_mask,
952 )
953 attn_output = self._compute_sparse_attention(
954 hidden_states, q_resid, attention_states, position_embeddings, actual_seq_len, kwargs,
955 )
956 value_up_weight = kv_weight[:, self.qk_nope_head_dim :].transpose(1, 2)
957 return _restore_attention_projection(
958 attn_output,
959 value_up_weight,
960 num_heads=self.num_heads,
961 batch_size=batch_size,
962 seq_length=seq_length,
963 kv_lora_rank=self.kv_lora_rank,
964 value_head_dim=self.v_head_dim,
965 )
967 def forward(
968 self,
969 hidden_states: torch.Tensor,
970 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
971 attention_mask: torch.Tensor | None = None,
972 past_key_values: Any | None = None,
973 cache_position: torch.Tensor | None = None,
974 actual_seq_len: torch.Tensor | Sequence[int] | None = None,
975 kv_reuse_states: Any | None = None,
976 output_attentions: bool = False,
977 return_bias: bool = False,
978 mome_mask: torch.Tensor | None = None,
979 **kwargs: Any,
980 ) -> tuple[torch.Tensor, torch.Tensor | None]:
981 """Run DSA with its configured MOME and parameter-sink paths."""
982 self._validate_forward_inputs(
983 attention_mask, kv_reuse_states, past_key_values, cache_position, output_attentions
984 )
985 attn_output = self._compute_attention_output(
986 hidden_states, position_embeddings, actual_seq_len, kwargs, mome_mask,
987 )
988 if self.use_mome:
989 attn_output = apply_mome(
990 attn_output,
991 mome_mask,
992 self.o_conv,
993 fused=self.use_fused_mome,
994 )
995 output, bias = self.linear_proj(attn_output)
996 return (output, bias) if return_bias else (output, None)