Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / mla_attention.py: 93%

117 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Multi-head latent attention module using the Hyper projection layout.""" 

16 

17from __future__ import annotations 

18 

19from collections.abc import Callable, Mapping, Sequence 

20from dataclasses import dataclass 

21from typing import Any 

22 

23# This package provides PyTorch-specific high-performance modules. 

24import torch 

25from torch import nn 

26from hyper_parallel.components.checkpoint.weight_conversion import WeightConverter 

27 

28from hyper_parallel.components.checkpoint import ConcatenateWithSections 

29from hyper_parallel.components.functional.npu_fusion_attention import npu_fusion_attention_forward 

30from hyper_parallel.models.replacement import module_replacement 

31 

32 

33def prepare_mla_attention_mask( 

34 attention_mask: torch.Tensor | None, 

35 query_length: int, 

36 key_length: int, 

37) -> torch.Tensor | None: 

38 """Disambiguate MLA Q/K masks before calling a shared attention interface. 

39 

40 Args: 

41 attention_mask: Optional mask. A two-dimensional Q/K matrix takes 

42 precedence over padding in MLA, including when batch equals query length. 

43 query_length: Number of query tokens visible to the attention interface. 

44 key_length: Number of key tokens visible to the attention interface. 

45 

46 Returns: 

47 A broadcastable four-dimensional view for a Q/K matrix. Other inputs 

48 pass through unchanged, preserving padding and compressed causal paths. 

49 """ 

50 if (attention_mask is not None and attention_mask.ndim == 2 

51 and attention_mask.shape == (query_length, key_length)): 

52 # Shared FA interprets 2D masks as batch padding; MLA also accepts Q/K matrices. 

53 return attention_mask[None, None, :, :] 

54 return attention_mask 

55 

56 

57@dataclass 

58class _MLALatents: 

59 batch_size: int 

60 sequence_length: int 

61 query_pass: torch.Tensor 

62 query_rope: torch.Tensor 

63 kv_nope: torch.Tensor 

64 key_rope: torch.Tensor 

65 

66 

67@module_replacement 

68class MLAAttention(nn.Module): 

69 """Transformers-compatible MLA using a fused latent projection. 

70 

71 ``q_a_proj`` and ``kv_a_proj_with_mqa`` are fused into ``linear_qkv``. 

72 Unchanged child modules retain their source names and checkpoint layout. 

73 Construction only creates the target structure; ``make_transforms`` 

74 declares the checkpoint conversion. 

75 """ 

76 

77 def __init__( 

78 self, 

79 *, 

80 module: nn.Module, 

81 module_fqn: str = "", 

82 context: Mapping[str, Any] | None = None, 

83 attention_interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]] = ( 

84 npu_fusion_attention_forward 

85 ), 

86 ) -> None: 

87 """Build the high-performance module from a Transformers MLA module.""" 

88 super().__init__() 

89 del module_fqn, context 

90 self.attention_interface = attention_interface 

91 required = ( 

92 "q_a_proj", 

93 "q_a_layernorm", 

94 "q_b_proj", 

95 "kv_a_proj_with_mqa", 

96 "kv_a_layernorm", 

97 "kv_b_proj", 

98 "o_proj", 

99 ) 

100 missing = [name for name in required if not hasattr(module, name)] 

101 if missing: 

102 raise TypeError(f"MLAAttention source module is missing: {missing}") 

103 

104 config = module.config 

105 self.config = config 

106 self.layer_idx = getattr(module, "layer_idx", getattr(module, "layer_number", None)) 

107 self.num_heads = getattr(module, "num_heads", config.num_attention_heads) 

108 self.num_key_value_heads = getattr( 

109 module, 

110 "num_key_value_heads", 

111 getattr(config, "num_key_value_heads", self.num_heads), 

112 ) 

113 if self.num_key_value_heads is None: 

114 self.num_key_value_heads = self.num_heads 

115 self.num_key_value_groups = self.num_heads // self.num_key_value_heads 

116 self.q_lora_rank = getattr(module, "q_lora_rank", config.q_lora_rank) 

117 self.kv_lora_rank = getattr(module, "kv_lora_rank", config.kv_lora_rank) 

118 if self.q_lora_rank is None: 

119 raise ValueError("MLAAttention requires the Q-LoRA projection path") 

120 self.qk_rope_head_dim = getattr(module, "qk_rope_head_dim", config.qk_rope_head_dim) 

121 self.qk_nope_head_dim = getattr(module, "qk_nope_head_dim", config.qk_nope_head_dim) 

122 self.v_head_dim = getattr(module, "v_head_dim", config.v_head_dim) 

123 self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim 

124 self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5) 

125 self.attention_dropout = getattr(module, "attention_dropout", 0.0) 

126 if isinstance(self.attention_dropout, nn.Dropout): 

127 self.attention_dropout = self.attention_dropout.p 

128 self.is_causal = getattr(module, "is_causal", True) 

129 self.sliding_window = getattr( 

130 module, "sliding_window", getattr(config, "sliding_window", None) 

131 ) 

132 self.rotary_interleaved = bool( 

133 getattr(module, "rotary_interleaved", getattr(config, "rope_interleave", False)) 

134 ) 

135 if getattr(module, "param_sink_number", 0) > 0: 

136 raise ValueError( 

137 "MLAAttention only implements standard MLA; parameter-sink attention " 

138 "must use the sink high-performance interfaces." 

139 ) 

140 self.use_mome = bool(getattr(module, "use_mome", False)) 

141 if self.use_mome: 

142 raise NotImplementedError( 

143 "MLAAttention only implements standard MLA; MOME must use the " 

144 "aggregate_hidden high-performance path." 

145 ) 

146 

147 q_a = module.q_a_proj 

148 kv_a = module.kv_a_proj_with_mqa 

149 self._q_latent_output_size = q_a.out_features 

150 self._kv_latent_output_size = kv_a.out_features 

151 can_fuse = ( 

152 q_a.in_features == kv_a.in_features 

153 and q_a.weight.requires_grad == kv_a.weight.requires_grad 

154 and (q_a.bias is None) == (kv_a.bias is None) 

155 and (q_a.bias is None or q_a.bias.requires_grad == kv_a.bias.requires_grad) 

156 ) 

157 if not can_fuse: 

158 raise ValueError( 

159 "Q and KV latent projections cannot be represented by one fused projection" 

160 ) 

161 self.linear_qkv = nn.Linear( 

162 q_a.in_features, 

163 q_a.out_features + kv_a.out_features, 

164 bias=q_a.bias is not None, 

165 device=q_a.weight.device, 

166 dtype=q_a.weight.dtype, 

167 ) 

168 self.linear_qkv.weight.requires_grad_(q_a.weight.requires_grad) 

169 if q_a.bias is not None: 

170 self.linear_qkv.bias.requires_grad_(q_a.bias.requires_grad) 

171 

172 self.q_a_layernorm = module.q_a_layernorm 

173 self.kv_a_layernorm = module.kv_a_layernorm 

174 self.q_b_proj = module.q_b_proj 

175 self.kv_b_proj = module.kv_b_proj 

176 self.o_proj = module.o_proj 

177 

178 self.train(module.training) 

179 

180 def make_transforms(self) -> list[WeightConverter]: 

181 """Describe reversible source-checkpoint to high-performance conversion.""" 

182 transforms: list[WeightConverter] = [ 

183 WeightConverter( 

184 source_patterns=[ 

185 "q_a_proj.weight", 

186 "kv_a_proj_with_mqa.weight", 

187 ], 

188 target_patterns="linear_qkv.weight", 

189 operations=[ 

190 ConcatenateWithSections( 

191 sections=( 

192 self._q_latent_output_size, 

193 self._kv_latent_output_size, 

194 ), 

195 dim=0, 

196 ) 

197 ], 

198 ), 

199 ] 

200 if self.linear_qkv.bias is not None: 

201 transforms.insert( 

202 1, 

203 WeightConverter( 

204 source_patterns=[ 

205 "q_a_proj.bias", 

206 "kv_a_proj_with_mqa.bias", 

207 ], 

208 target_patterns="linear_qkv.bias", 

209 operations=[ 

210 ConcatenateWithSections( 

211 sections=( 

212 self._q_latent_output_size, 

213 self._kv_latent_output_size, 

214 ), 

215 dim=0, 

216 ) 

217 ], 

218 ), 

219 ) 

220 return transforms 

221 

222 def project_latent_inputs( 

223 self, hidden_states: torch.Tensor, 

224 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 

225 """Compute shared latent inputs before head expansion. 

226 

227 Args: 

228 hidden_states: Input tokens with shape [batch, sequence, hidden]. 

229 

230 Returns: 

231 Normalized query and KV latents, and the unrotated shared RoPE key. 

232 """ 

233 latent_states = self.linear_qkv(hidden_states) 

234 query_latent, kv_nope, key_rope = torch.split( 

235 latent_states, 

236 (self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim), 

237 dim=-1, 

238 ) 

239 return self.q_a_layernorm(query_latent), self.kv_a_layernorm(kv_nope), key_rope 

240 

241 def _project_latents(self, hidden_states: torch.Tensor) -> _MLALatents: 

242 """Project hidden states into query and compressed KV latent states.""" 

243 batch_size, sequence_length = hidden_states.shape[:-1] 

244 query_latent, kv_nope, key_rope = self.project_latent_inputs(hidden_states) 

245 query_states = self.q_b_proj(query_latent).view( 

246 batch_size, 

247 sequence_length, 

248 self.num_heads, 

249 self.qk_head_dim, 

250 ) 

251 query_pass, query_rope = torch.split( 

252 query_states, 

253 (self.qk_nope_head_dim, self.qk_rope_head_dim), 

254 dim=-1, 

255 ) 

256 return _MLALatents( 

257 batch_size, 

258 sequence_length, 

259 query_pass, 

260 query_rope.transpose(1, 2), 

261 kv_nope.view( 

262 batch_size, 1, sequence_length, self.kv_lora_rank 

263 ), 

264 key_rope.view(batch_size, 1, sequence_length, self.qk_rope_head_dim), 

265 ) 

266 

267 def _project_attention_inputs( 

268 self, 

269 hidden_states: torch.Tensor, 

270 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, 

271 past_key_values: Any | None, 

272 ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: 

273 """Build query, key, and value tensors from compressed latent states.""" 

274 latents = self._project_latents(hidden_states) 

275 query_rope, key_rope = latents.query_rope, latents.key_rope 

276 if position_embeddings is not None: 

277 # Defer the optional NPU dependency until rotary embedding is used. 

278 from hyper_parallel.components.functional import ( # pylint: disable=C0415 

279 apply_rotary_pos_emb, apply_rotary_pos_emb_interleave, 

280 ) 

281 

282 cos, sin = position_embeddings 

283 rope_fn = apply_rotary_pos_emb_interleave if self.rotary_interleaved else apply_rotary_pos_emb 

284 query_rope, key_rope = rope_fn(query_rope, key_rope, cos, sin) 

285 kv_nope = latents.kv_nope 

286 if past_key_values is not None: 

287 kv_nope, key_rope = past_key_values.update(kv_nope, key_rope, self.layer_idx) 

288 kv_states = self.kv_b_proj(kv_nope).view( 

289 latents.batch_size, 

290 kv_nope.shape[2], 

291 self.num_heads, 

292 self.qk_nope_head_dim + self.v_head_dim, 

293 ).transpose(1, 2) 

294 key_nope, value_states = torch.split( 

295 kv_states, 

296 (self.qk_nope_head_dim, self.v_head_dim), 

297 dim=-1, 

298 ) 

299 return ( 

300 torch.cat((latents.query_pass.transpose(1, 2), query_rope), dim=-1), 

301 torch.cat((key_nope, key_rope.expand(-1, self.num_heads, -1, -1)), dim=-1), 

302 value_states, 

303 ) 

304 

305 def project_output(self, attention_output: torch.Tensor, hidden_states: torch.Tensor) -> torch.Tensor: 

306 """Project [B, local_tokens, heads * value_dim] to [B, local_tokens, hidden]. 

307 

308 This model-math extension point lets gated MLA multiply its gate before 

309 Wo; the default MLA keeps the original output projection unchanged. 

310 This method owns model math only; CP communication belongs to its adapter. 

311 

312 Args: 

313 attention_output: Flattened local-token attention output. 

314 hidden_states: Original local-token input, available to gated variants. 

315 

316 Returns: 

317 Projected local hidden states. 

318 """ 

319 del hidden_states 

320 return self.o_proj(attention_output) 

321 

322 def forward( 

323 self, 

324 hidden_states: torch.Tensor, 

325 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, 

326 attention_mask: torch.Tensor | None = None, 

327 past_key_values: Any | None = None, 

328 actual_seq_len: torch.Tensor | Sequence[int] | None = None, 

329 **kwargs: Any, 

330 ) -> tuple[torch.Tensor, torch.Tensor | None]: 

331 """Run MLA with the same external contract as Transformers attention. 

332 

333 Args: 

334 hidden_states: Input tokens in batch, sequence, hidden order. 

335 position_embeddings: Optional precomputed RoPE cosine and sine. 

336 attention_mask: Optional mask in the attention interface convention. 

337 past_key_values: Optional cache for incremental decoding. 

338 actual_seq_len: Optional cumulative packed document ends. 

339 **kwargs: Additional arguments for the attention interface. 

340 

341 Returns: 

342 Projected attention output and optional attention weights. 

343 """ 

344 batch_size, seq_length = hidden_states.shape[:-1] 

345 query_states, key_states, value_states = self._project_attention_inputs( 

346 hidden_states, 

347 position_embeddings, 

348 past_key_values, 

349 ) 

350 

351 attention_mask = prepare_mla_attention_mask( 

352 attention_mask, query_states.shape[2], key_states.shape[2], 

353 ) 

354 attn_output, attn_weights = self.attention_interface( 

355 self, 

356 query_states, 

357 key_states, 

358 value_states, 

359 attention_mask, 

360 dropout=self.attention_dropout if self.training else 0.0, 

361 scaling=self.scaling, 

362 sliding_window=self.sliding_window, 

363 actual_seq_len=actual_seq_len, 

364 **kwargs, 

365 ) 

366 attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous() 

367 # Gated MLA derives its gate from the original input before Wo; ordinary 

368 # MLA only applies Wo. Keep this math extension shared by CP1 and CP. 

369 attn_output = self.project_output(attn_output, hidden_states) 

370 return attn_output, attn_weights