Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / mla_attention.py: 93%
117 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Multi-head latent attention module using the Hyper projection layout."""
17from __future__ import annotations
19from collections.abc import Callable, Mapping, Sequence
20from dataclasses import dataclass
21from typing import Any
23# This package provides PyTorch-specific high-performance modules.
24import torch
25from torch import nn
26from hyper_parallel.components.checkpoint.weight_conversion import WeightConverter
28from hyper_parallel.components.checkpoint import ConcatenateWithSections
29from hyper_parallel.components.functional.npu_fusion_attention import npu_fusion_attention_forward
30from hyper_parallel.models.replacement import module_replacement
33def prepare_mla_attention_mask(
34 attention_mask: torch.Tensor | None,
35 query_length: int,
36 key_length: int,
37) -> torch.Tensor | None:
38 """Disambiguate MLA Q/K masks before calling a shared attention interface.
40 Args:
41 attention_mask: Optional mask. A two-dimensional Q/K matrix takes
42 precedence over padding in MLA, including when batch equals query length.
43 query_length: Number of query tokens visible to the attention interface.
44 key_length: Number of key tokens visible to the attention interface.
46 Returns:
47 A broadcastable four-dimensional view for a Q/K matrix. Other inputs
48 pass through unchanged, preserving padding and compressed causal paths.
49 """
50 if (attention_mask is not None and attention_mask.ndim == 2
51 and attention_mask.shape == (query_length, key_length)):
52 # Shared FA interprets 2D masks as batch padding; MLA also accepts Q/K matrices.
53 return attention_mask[None, None, :, :]
54 return attention_mask
57@dataclass
58class _MLALatents:
59 batch_size: int
60 sequence_length: int
61 query_pass: torch.Tensor
62 query_rope: torch.Tensor
63 kv_nope: torch.Tensor
64 key_rope: torch.Tensor
67@module_replacement
68class MLAAttention(nn.Module):
69 """Transformers-compatible MLA using a fused latent projection.
71 ``q_a_proj`` and ``kv_a_proj_with_mqa`` are fused into ``linear_qkv``.
72 Unchanged child modules retain their source names and checkpoint layout.
73 Construction only creates the target structure; ``make_transforms``
74 declares the checkpoint conversion.
75 """
77 def __init__(
78 self,
79 *,
80 module: nn.Module,
81 module_fqn: str = "",
82 context: Mapping[str, Any] | None = None,
83 attention_interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]] = (
84 npu_fusion_attention_forward
85 ),
86 ) -> None:
87 """Build the high-performance module from a Transformers MLA module."""
88 super().__init__()
89 del module_fqn, context
90 self.attention_interface = attention_interface
91 required = (
92 "q_a_proj",
93 "q_a_layernorm",
94 "q_b_proj",
95 "kv_a_proj_with_mqa",
96 "kv_a_layernorm",
97 "kv_b_proj",
98 "o_proj",
99 )
100 missing = [name for name in required if not hasattr(module, name)]
101 if missing:
102 raise TypeError(f"MLAAttention source module is missing: {missing}")
104 config = module.config
105 self.config = config
106 self.layer_idx = getattr(module, "layer_idx", getattr(module, "layer_number", None))
107 self.num_heads = getattr(module, "num_heads", config.num_attention_heads)
108 self.num_key_value_heads = getattr(
109 module,
110 "num_key_value_heads",
111 getattr(config, "num_key_value_heads", self.num_heads),
112 )
113 if self.num_key_value_heads is None:
114 self.num_key_value_heads = self.num_heads
115 self.num_key_value_groups = self.num_heads // self.num_key_value_heads
116 self.q_lora_rank = getattr(module, "q_lora_rank", config.q_lora_rank)
117 self.kv_lora_rank = getattr(module, "kv_lora_rank", config.kv_lora_rank)
118 if self.q_lora_rank is None:
119 raise ValueError("MLAAttention requires the Q-LoRA projection path")
120 self.qk_rope_head_dim = getattr(module, "qk_rope_head_dim", config.qk_rope_head_dim)
121 self.qk_nope_head_dim = getattr(module, "qk_nope_head_dim", config.qk_nope_head_dim)
122 self.v_head_dim = getattr(module, "v_head_dim", config.v_head_dim)
123 self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
124 self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5)
125 self.attention_dropout = getattr(module, "attention_dropout", 0.0)
126 if isinstance(self.attention_dropout, nn.Dropout):
127 self.attention_dropout = self.attention_dropout.p
128 self.is_causal = getattr(module, "is_causal", True)
129 self.sliding_window = getattr(
130 module, "sliding_window", getattr(config, "sliding_window", None)
131 )
132 self.rotary_interleaved = bool(
133 getattr(module, "rotary_interleaved", getattr(config, "rope_interleave", False))
134 )
135 if getattr(module, "param_sink_number", 0) > 0:
136 raise ValueError(
137 "MLAAttention only implements standard MLA; parameter-sink attention "
138 "must use the sink high-performance interfaces."
139 )
140 self.use_mome = bool(getattr(module, "use_mome", False))
141 if self.use_mome:
142 raise NotImplementedError(
143 "MLAAttention only implements standard MLA; MOME must use the "
144 "aggregate_hidden high-performance path."
145 )
147 q_a = module.q_a_proj
148 kv_a = module.kv_a_proj_with_mqa
149 self._q_latent_output_size = q_a.out_features
150 self._kv_latent_output_size = kv_a.out_features
151 can_fuse = (
152 q_a.in_features == kv_a.in_features
153 and q_a.weight.requires_grad == kv_a.weight.requires_grad
154 and (q_a.bias is None) == (kv_a.bias is None)
155 and (q_a.bias is None or q_a.bias.requires_grad == kv_a.bias.requires_grad)
156 )
157 if not can_fuse:
158 raise ValueError(
159 "Q and KV latent projections cannot be represented by one fused projection"
160 )
161 self.linear_qkv = nn.Linear(
162 q_a.in_features,
163 q_a.out_features + kv_a.out_features,
164 bias=q_a.bias is not None,
165 device=q_a.weight.device,
166 dtype=q_a.weight.dtype,
167 )
168 self.linear_qkv.weight.requires_grad_(q_a.weight.requires_grad)
169 if q_a.bias is not None:
170 self.linear_qkv.bias.requires_grad_(q_a.bias.requires_grad)
172 self.q_a_layernorm = module.q_a_layernorm
173 self.kv_a_layernorm = module.kv_a_layernorm
174 self.q_b_proj = module.q_b_proj
175 self.kv_b_proj = module.kv_b_proj
176 self.o_proj = module.o_proj
178 self.train(module.training)
180 def make_transforms(self) -> list[WeightConverter]:
181 """Describe reversible source-checkpoint to high-performance conversion."""
182 transforms: list[WeightConverter] = [
183 WeightConverter(
184 source_patterns=[
185 "q_a_proj.weight",
186 "kv_a_proj_with_mqa.weight",
187 ],
188 target_patterns="linear_qkv.weight",
189 operations=[
190 ConcatenateWithSections(
191 sections=(
192 self._q_latent_output_size,
193 self._kv_latent_output_size,
194 ),
195 dim=0,
196 )
197 ],
198 ),
199 ]
200 if self.linear_qkv.bias is not None:
201 transforms.insert(
202 1,
203 WeightConverter(
204 source_patterns=[
205 "q_a_proj.bias",
206 "kv_a_proj_with_mqa.bias",
207 ],
208 target_patterns="linear_qkv.bias",
209 operations=[
210 ConcatenateWithSections(
211 sections=(
212 self._q_latent_output_size,
213 self._kv_latent_output_size,
214 ),
215 dim=0,
216 )
217 ],
218 ),
219 )
220 return transforms
222 def project_latent_inputs(
223 self, hidden_states: torch.Tensor,
224 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
225 """Compute shared latent inputs before head expansion.
227 Args:
228 hidden_states: Input tokens with shape [batch, sequence, hidden].
230 Returns:
231 Normalized query and KV latents, and the unrotated shared RoPE key.
232 """
233 latent_states = self.linear_qkv(hidden_states)
234 query_latent, kv_nope, key_rope = torch.split(
235 latent_states,
236 (self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim),
237 dim=-1,
238 )
239 return self.q_a_layernorm(query_latent), self.kv_a_layernorm(kv_nope), key_rope
241 def _project_latents(self, hidden_states: torch.Tensor) -> _MLALatents:
242 """Project hidden states into query and compressed KV latent states."""
243 batch_size, sequence_length = hidden_states.shape[:-1]
244 query_latent, kv_nope, key_rope = self.project_latent_inputs(hidden_states)
245 query_states = self.q_b_proj(query_latent).view(
246 batch_size,
247 sequence_length,
248 self.num_heads,
249 self.qk_head_dim,
250 )
251 query_pass, query_rope = torch.split(
252 query_states,
253 (self.qk_nope_head_dim, self.qk_rope_head_dim),
254 dim=-1,
255 )
256 return _MLALatents(
257 batch_size,
258 sequence_length,
259 query_pass,
260 query_rope.transpose(1, 2),
261 kv_nope.view(
262 batch_size, 1, sequence_length, self.kv_lora_rank
263 ),
264 key_rope.view(batch_size, 1, sequence_length, self.qk_rope_head_dim),
265 )
267 def _project_attention_inputs(
268 self,
269 hidden_states: torch.Tensor,
270 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
271 past_key_values: Any | None,
272 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
273 """Build query, key, and value tensors from compressed latent states."""
274 latents = self._project_latents(hidden_states)
275 query_rope, key_rope = latents.query_rope, latents.key_rope
276 if position_embeddings is not None:
277 # Defer the optional NPU dependency until rotary embedding is used.
278 from hyper_parallel.components.functional import ( # pylint: disable=C0415
279 apply_rotary_pos_emb, apply_rotary_pos_emb_interleave,
280 )
282 cos, sin = position_embeddings
283 rope_fn = apply_rotary_pos_emb_interleave if self.rotary_interleaved else apply_rotary_pos_emb
284 query_rope, key_rope = rope_fn(query_rope, key_rope, cos, sin)
285 kv_nope = latents.kv_nope
286 if past_key_values is not None:
287 kv_nope, key_rope = past_key_values.update(kv_nope, key_rope, self.layer_idx)
288 kv_states = self.kv_b_proj(kv_nope).view(
289 latents.batch_size,
290 kv_nope.shape[2],
291 self.num_heads,
292 self.qk_nope_head_dim + self.v_head_dim,
293 ).transpose(1, 2)
294 key_nope, value_states = torch.split(
295 kv_states,
296 (self.qk_nope_head_dim, self.v_head_dim),
297 dim=-1,
298 )
299 return (
300 torch.cat((latents.query_pass.transpose(1, 2), query_rope), dim=-1),
301 torch.cat((key_nope, key_rope.expand(-1, self.num_heads, -1, -1)), dim=-1),
302 value_states,
303 )
305 def project_output(self, attention_output: torch.Tensor, hidden_states: torch.Tensor) -> torch.Tensor:
306 """Project [B, local_tokens, heads * value_dim] to [B, local_tokens, hidden].
308 This model-math extension point lets gated MLA multiply its gate before
309 Wo; the default MLA keeps the original output projection unchanged.
310 This method owns model math only; CP communication belongs to its adapter.
312 Args:
313 attention_output: Flattened local-token attention output.
314 hidden_states: Original local-token input, available to gated variants.
316 Returns:
317 Projected local hidden states.
318 """
319 del hidden_states
320 return self.o_proj(attention_output)
322 def forward(
323 self,
324 hidden_states: torch.Tensor,
325 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
326 attention_mask: torch.Tensor | None = None,
327 past_key_values: Any | None = None,
328 actual_seq_len: torch.Tensor | Sequence[int] | None = None,
329 **kwargs: Any,
330 ) -> tuple[torch.Tensor, torch.Tensor | None]:
331 """Run MLA with the same external contract as Transformers attention.
333 Args:
334 hidden_states: Input tokens in batch, sequence, hidden order.
335 position_embeddings: Optional precomputed RoPE cosine and sine.
336 attention_mask: Optional mask in the attention interface convention.
337 past_key_values: Optional cache for incremental decoding.
338 actual_seq_len: Optional cumulative packed document ends.
339 **kwargs: Additional arguments for the attention interface.
341 Returns:
342 Projected attention output and optional attention weights.
343 """
344 batch_size, seq_length = hidden_states.shape[:-1]
345 query_states, key_states, value_states = self._project_attention_inputs(
346 hidden_states,
347 position_embeddings,
348 past_key_values,
349 )
351 attention_mask = prepare_mla_attention_mask(
352 attention_mask, query_states.shape[2], key_states.shape[2],
353 )
354 attn_output, attn_weights = self.attention_interface(
355 self,
356 query_states,
357 key_states,
358 value_states,
359 attention_mask,
360 dropout=self.attention_dropout if self.training else 0.0,
361 scaling=self.scaling,
362 sliding_window=self.sliding_window,
363 actual_seq_len=actual_seq_len,
364 **kwargs,
365 )
366 attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous()
367 # Gated MLA derives its gate from the original input before Wo; ordinary
368 # MLA only applies Wo. Keep this math extension shared by CP1 and CP.
369 attn_output = self.project_output(attn_output, hidden_states)
370 return attn_output, attn_weights