Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / gqa_attention.py: 49%
225 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Grouped-query attention module using the Hyper high-performance layout."""
17from __future__ import annotations
19from collections.abc import Callable, Mapping, Sequence
20from typing import Any
22# This package provides PyTorch-specific high-performance modules.
23import torch
24from torch import nn
25from hyper_parallel.components.checkpoint.weight_conversion import WeightConverter
27from hyper_parallel.components.checkpoint import InterleaveGateQKV, InterleaveQKV
28from hyper_parallel.models.replacement import module_replacement
29from hyper_parallel.components.functional import (
30 apply_rotary_pos_emb,
31 apply_rotary_pos_emb_interleave,
32 npu_fusion_attention_forward,
33)
36def rotate_half(x: torch.Tensor) -> torch.Tensor:
37 """Rotate the two halves of the last dimension as Transformers does."""
38 first = x[..., : x.shape[-1] // 2]
39 second = x[..., x.shape[-1] // 2 :]
40 return torch.cat((-second, first), dim=-1)
43def _projections_can_fuse(projections: tuple[nn.Linear, ...]) -> bool:
44 """Return whether projections share the layout needed for fusion."""
45 biases = tuple(projection.bias for projection in projections)
46 return (
47 len({projection.in_features for projection in projections}) == 1
48 and len({projection.weight.requires_grad for projection in projections}) == 1
49 and (all(bias is None for bias in biases) or all(bias is not None for bias in biases))
50 and (biases[0] is None or len({bias.requires_grad for bias in biases}) == 1)
51 )
54def _apply_gqa_rope(
55 query: torch.Tensor,
56 key: torch.Tensor,
57 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None,
58 rotary_interleaved: bool,
59 head_dim: int,
60) -> tuple[torch.Tensor, torch.Tensor]:
61 """Apply the configured rotary convention to grouped-query attention."""
62 if position_embeddings is None:
63 return query, key
64 cos, sin = position_embeddings
65 if rotary_interleaved:
66 return apply_rotary_pos_emb_interleave(query, key, cos, sin, unsqueeze_dim=2)
67 if cos.shape[-1] >= head_dim:
68 return apply_rotary_pos_emb(query, key, cos, sin, unsqueeze_dim=2)
69 rotary_dim = cos.shape[-1]
70 query_rot, query_pass = query[..., :rotary_dim], query[..., rotary_dim:]
71 key_rot, key_pass = key[..., :rotary_dim], key[..., rotary_dim:]
72 cos = cos.unsqueeze(2)
73 sin = sin.unsqueeze(2)
74 query = torch.cat((query_rot * cos + rotate_half(query_rot) * sin, query_pass), dim=-1)
75 key = torch.cat((key_rot * cos + rotate_half(key_rot) * sin, key_pass), dim=-1)
76 return query, key
79@module_replacement
80class GQAAttention(nn.Module):
81 """Transformers-compatible GQA using a grouped ``linear_qkv`` layout.
83 The source module may expose a fused ``qkv_proj`` or separate
84 ``q_proj``/``k_proj``/``v_proj`` layers. Its checkpoint layout is converted
85 to the per-KV-head grouping consumed by the high-performance kernel.
86 Construction only creates the target structure; ``make_transforms``
87 declares the checkpoint conversion.
88 """
90 def _initialize_dimensions(self, module: nn.Module) -> None:
91 """Initialize head dimensions and grouped-QKV layout metadata."""
92 config = self.config
93 self.num_heads = getattr(module, "num_attention_heads", config.num_attention_heads)
94 self.num_key_value_heads = getattr(module, "num_key_value_heads", config.num_key_value_heads)
95 if self.num_heads % self.num_key_value_heads:
96 raise ValueError("num_attention_heads must be divisible by num_key_value_heads")
97 self.num_key_value_groups = self.num_heads // self.num_key_value_heads
98 self.qk_head_dim = getattr(
99 module, "head_dim", getattr(config, "head_dim", config.hidden_size // self.num_heads)
100 )
101 self.v_head_dim = (
102 getattr(module, "v_head_dim", None) or getattr(config, "v_head_dim", None) or self.qk_head_dim
103 )
104 self.qkv_split_sizes = (
105 self.num_key_value_groups * self.qk_head_dim,
106 self.qk_head_dim,
107 self.v_head_dim,
108 )
109 self.qkv_group_width = sum(self.qkv_split_sizes)
111 def _initialize_attention_options(self, module: nn.Module) -> None:
112 """Copy supported attention options and reject unsupported variants."""
113 config = self.config
114 self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5)
115 self.attention_dropout = getattr(module, "attention_dropout", 0.0)
116 if isinstance(self.attention_dropout, nn.Dropout):
117 self.attention_dropout = self.attention_dropout.p
118 self.is_causal = getattr(module, "is_causal", True)
119 self.sliding_window = getattr(module, "sliding_window", getattr(config, "sliding_window", None))
120 self.rotary_interleaved = getattr(
121 module,
122 "rotary_interleaved",
123 getattr(config, "rope_interleave", getattr(config, "rope_interleaved", False)),
124 )
125 if getattr(module, "param_sink_number", 0) > 0:
126 raise ValueError(
127 "GQAAttention only implements standard GQA; parameter-sink attention "
128 "must use the sink_attention high-performance interface."
129 )
130 self.attn_groupnorm = bool(getattr(module, "attn_groupnorm", False))
131 self.attn_elementwise_gate = bool(getattr(module, "attn_elementwise_gate", False))
132 if self.attn_groupnorm or self.attn_elementwise_gate:
133 raise ValueError(
134 "GQAAttention only implements standard GQA; group normalization and "
135 "elementwise attention gates require a dedicated replacement."
136 )
138 def _source_qkv(self, module: nn.Module) -> tuple[nn.Linear, tuple[str, ...]]:
139 """Validate and return the source QKV projection layout."""
140 self._source_is_fused = hasattr(module, "qkv_proj")
141 if self._source_is_fused:
142 source_qkv = module.qkv_proj
143 if not isinstance(source_qkv, nn.Linear):
144 raise TypeError("qkv_proj must be torch.nn.Linear")
145 expected_size = (
146 self.num_heads * self.qk_head_dim
147 + self.num_key_value_heads * self.qk_head_dim
148 + self.num_key_value_heads * self.v_head_dim
149 )
150 if source_qkv.out_features != expected_size:
151 raise ValueError("qkv_proj output size is incompatible with the configured GQA dimensions")
152 return source_qkv, ("qkv_proj",)
153 required = ("q_proj", "k_proj", "v_proj")
154 if any(not hasattr(module, name) for name in required):
155 raise TypeError("GQAAttention requires qkv_proj or q_proj/k_proj/v_proj on the source module")
156 projections = tuple(getattr(module, name) for name in required)
157 if any(not isinstance(projection, nn.Linear) for projection in projections):
158 raise TypeError("q_proj, k_proj, and v_proj must be torch.nn.Linear")
159 expected_sizes = (
160 self.num_heads * self.qk_head_dim,
161 self.num_key_value_heads * self.qk_head_dim,
162 self.num_key_value_heads * self.v_head_dim,
163 )
164 if tuple(projection.out_features for projection in projections) != expected_sizes:
165 raise ValueError("Q/K/V projection sizes are incompatible with the configured GQA dimensions")
166 if not _projections_can_fuse(projections):
167 raise ValueError("Q, K, and V projections cannot be represented by one fused projection")
168 return module.q_proj, required
170 def _initialize_projections(self, module: nn.Module, source_qkv: nn.Linear) -> None:
171 """Create the fused QKV projection and attach the source output projection."""
172 projection_size = self.num_key_value_heads * self.qkv_group_width
173 self.linear_qkv = nn.Linear(
174 source_qkv.in_features,
175 projection_size,
176 bias=source_qkv.bias is not None,
177 device=source_qkv.weight.device,
178 dtype=source_qkv.weight.dtype,
179 )
180 self.linear_qkv.weight.requires_grad_(source_qkv.weight.requires_grad)
181 if self.linear_qkv.bias is not None:
182 self.linear_qkv.bias.requires_grad_(source_qkv.bias.requires_grad)
183 if not hasattr(module, "o_proj") or not isinstance(module.o_proj, nn.Linear):
184 raise TypeError("GQAAttention requires o_proj on the source module")
185 if module.o_proj.in_features != self.num_heads * self.v_head_dim:
186 raise ValueError("o_proj input size is incompatible with the configured GQA dimensions")
187 self.o_proj = module.o_proj
188 self.q_norm = getattr(module, "q_norm", None)
189 self.k_norm = getattr(module, "k_norm", None)
191 def __init__(
192 self,
193 *,
194 module: nn.Module,
195 module_fqn: str = "",
196 context: Mapping[str, Any] | None = None,
197 attention_interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]] = (
198 npu_fusion_attention_forward
199 ),
200 ) -> None:
201 """Build the high-performance module from a Transformers attention module."""
202 super().__init__()
203 del module_fqn, context
204 self.attention_interface = attention_interface
205 config = module.config
206 self.config = config
207 self.layer_idx = getattr(module, "layer_idx", getattr(module, "layer_number", None))
208 self._initialize_dimensions(module)
209 self._initialize_attention_options(module)
210 source_qkv, source_projection_names = self._source_qkv(module)
211 self._source_projection_names = source_projection_names
212 self._initialize_projections(module, source_qkv)
213 self.train(module.training)
215 def make_transforms(self) -> list[WeightConverter]:
216 """Describe reversible source-checkpoint to high-performance conversion."""
217 if self._source_is_fused:
218 qkv_sources = "qkv_proj.weight"
219 bias_sources: str | list[str] = "qkv_proj.bias"
220 else:
221 qkv_sources = [f"{name}.weight" for name in self._source_projection_names]
222 bias_sources = [f"{name}.bias" for name in self._source_projection_names]
224 transforms: list[WeightConverter] = [
225 WeightConverter(
226 source_patterns=qkv_sources,
227 target_patterns="linear_qkv.weight",
228 operations=[
229 InterleaveQKV(
230 self.num_key_value_heads,
231 self.num_key_value_groups,
232 self.qk_head_dim,
233 self.v_head_dim,
234 self._source_is_fused,
235 )
236 ],
237 )
238 ]
239 if self.linear_qkv.bias is not None:
240 transforms.append(
241 WeightConverter(
242 source_patterns=bias_sources,
243 target_patterns="linear_qkv.bias",
244 operations=[
245 InterleaveQKV(
246 self.num_key_value_heads,
247 self.num_key_value_groups,
248 self.qk_head_dim,
249 self.v_head_dim,
250 self._source_is_fused,
251 )
252 ],
253 )
254 )
255 return transforms
257 def forward(
258 self,
259 hidden_states: torch.Tensor,
260 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
261 attention_mask: torch.Tensor | None = None,
262 past_key_values: Any | None = None,
263 actual_seq_len: torch.Tensor | Sequence[int] | None = None,
264 **kwargs: Any,
265 ) -> tuple[torch.Tensor, torch.Tensor | None]:
266 """Run GQA with the same external contract as Transformers attention."""
267 input_shape = hidden_states.shape[:-1]
268 qkv_states = self.linear_qkv(hidden_states).view(
269 *input_shape,
270 self.num_key_value_heads,
271 self.qkv_group_width,
272 )
273 query_states, key_states, value_states = torch.split(
274 qkv_states, self.qkv_split_sizes, dim=-1
275 )
276 query_states = query_states.reshape(
277 *input_shape, self.num_heads, self.qk_head_dim
278 )
280 if self.q_norm is not None:
281 query_states = self.q_norm(query_states)
282 if self.k_norm is not None:
283 key_states = self.k_norm(key_states)
285 query_states, key_states = _apply_gqa_rope(
286 query_states,
287 key_states,
288 position_embeddings,
289 self.rotary_interleaved,
290 self.qk_head_dim,
291 )
292 query_states = query_states.transpose(1, 2)
293 key_states = key_states.transpose(1, 2)
294 value_states = value_states.transpose(1, 2)
296 if past_key_values is not None:
297 key_states, value_states = past_key_values.update(
298 key_states, value_states, self.layer_idx
299 )
301 attn_output, attn_weights = self.attention_interface(
302 self,
303 query_states,
304 key_states,
305 value_states,
306 attention_mask,
307 dropout=self.attention_dropout if self.training else 0.0,
308 scaling=self.scaling,
309 sliding_window=self.sliding_window,
310 actual_seq_len=actual_seq_len,
311 **kwargs,
312 )
313 attn_output = attn_output.reshape(*input_shape, -1).contiguous()
314 attn_output = self.o_proj(attn_output)
315 return attn_output, attn_weights
318@module_replacement
319class GatedGQAAttention(nn.Module):
320 """GQA with a sigmoid output gate derived from each query head.
322 The source ``q_proj`` is expected to store ``[query, gate]`` rows for each
323 query head. This is the gated-query layout used by Qwen3.5 full attention.
324 """
326 def _initialize_dimensions(self, module: nn.Module) -> None:
327 """Initialize gated grouped-QKV dimensions and attention options."""
328 config = self.config
329 self.num_heads = getattr(module, "num_attention_heads", config.num_attention_heads)
330 self.num_key_value_heads = getattr(module, "num_key_value_heads", config.num_key_value_heads)
331 if self.num_heads % self.num_key_value_heads:
332 raise ValueError("num_attention_heads must be divisible by num_key_value_heads")
333 self.num_key_value_groups = self.num_heads // self.num_key_value_heads
334 self.qk_head_dim = getattr(
335 module, "head_dim", getattr(config, "head_dim", config.hidden_size // self.num_heads)
336 )
337 self.v_head_dim = (
338 getattr(module, "v_head_dim", None) or getattr(config, "v_head_dim", None) or self.qk_head_dim
339 )
340 self.qkv_split_sizes = (
341 self.num_key_value_groups * self.qk_head_dim,
342 self.qk_head_dim,
343 self.v_head_dim,
344 )
345 self.qkv_group_width = sum(self.qkv_split_sizes)
346 self.gated_qkv_group_width = self.qkv_group_width + self.num_key_value_groups * self.qk_head_dim
347 self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5)
348 self.attention_dropout = getattr(module, "attention_dropout", 0.0)
349 if isinstance(self.attention_dropout, nn.Dropout):
350 self.attention_dropout = self.attention_dropout.p
351 self.is_causal = getattr(module, "is_causal", True)
352 self.sliding_window = getattr(module, "sliding_window", getattr(config, "sliding_window", None))
353 self.rotary_interleaved = getattr(
354 module,
355 "rotary_interleaved",
356 getattr(config, "rope_interleave", getattr(config, "rope_interleaved", False)),
357 )
359 def _validate_source_projections(self, module: nn.Module) -> None:
360 """Validate separate gated-query, key, value, and output projections."""
361 projections = (module.q_proj, module.k_proj, module.v_proj)
362 if any(not isinstance(projection, nn.Linear) for projection in projections):
363 raise TypeError("q_proj, k_proj, and v_proj must be torch.nn.Linear")
364 expected_sizes = (
365 2 * self.num_heads * self.qk_head_dim,
366 self.num_key_value_heads * self.qk_head_dim,
367 self.num_key_value_heads * self.v_head_dim,
368 )
369 if tuple(projection.out_features for projection in projections) != expected_sizes:
370 raise ValueError(
371 "Q/gate, K, and V projection sizes are incompatible with the configured gated GQA dimensions"
372 )
373 if not _projections_can_fuse(projections):
374 raise ValueError("Q/gate, K, and V projections cannot be represented by one fused projection")
375 attention_output_size = self.num_heads * self.v_head_dim
376 if self.num_heads * self.qk_head_dim != attention_output_size:
377 raise ValueError("GatedGQAAttention requires the gate width to match the attention output width")
378 if not isinstance(module.o_proj, nn.Linear):
379 raise TypeError("o_proj must be torch.nn.Linear")
380 if module.o_proj.in_features != attention_output_size:
381 raise ValueError("o_proj input size is incompatible with the configured gated GQA dimensions")
383 def _initialize_projection(self, module: nn.Module) -> None:
384 """Create the fused gated-QKV projection and attach source output layers."""
385 self.linear_qkv = nn.Linear(
386 module.q_proj.in_features,
387 self.num_key_value_heads * self.gated_qkv_group_width,
388 bias=module.q_proj.bias is not None,
389 device=module.q_proj.weight.device,
390 dtype=module.q_proj.weight.dtype,
391 )
392 self.linear_qkv.weight.requires_grad_(module.q_proj.weight.requires_grad)
393 if self.linear_qkv.bias is not None:
394 self.linear_qkv.bias.requires_grad_(module.q_proj.bias.requires_grad)
395 self.q_norm = module.q_norm
396 self.k_norm = module.k_norm
397 self.o_proj = module.o_proj
399 def __init__(
400 self,
401 *,
402 module: nn.Module,
403 module_fqn: str = "",
404 context: Mapping[str, Any] | None = None,
405 attention_interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]] = (
406 npu_fusion_attention_forward
407 ),
408 ) -> None:
409 """Build gated GQA from a source with separate Q/gate, K, and V projections."""
410 super().__init__()
411 del module_fqn, context
412 if hasattr(module, "qkv_proj"):
413 raise ValueError(
414 "GatedGQAAttention requires separate q_proj/k_proj/v_proj projections"
415 )
416 required = ("q_proj", "k_proj", "v_proj", "q_norm", "k_norm", "o_proj")
417 if any(not hasattr(module, name) for name in required):
418 raise TypeError(
419 "GatedGQAAttention requires q_proj, k_proj, v_proj, q_norm, "
420 "k_norm, and o_proj on the source module"
421 )
423 self.attention_interface = attention_interface
424 self.config = module.config
425 self.layer_idx = getattr(module, "layer_idx", None)
426 self._initialize_dimensions(module)
427 self._validate_source_projections(module)
428 self._initialize_projection(module)
429 self.train(module.training)
431 def make_transforms(self) -> list[WeightConverter]:
432 """Describe reversible gated-query and grouped-QKV conversion."""
433 source_weights = ["q_proj.weight", "k_proj.weight", "v_proj.weight"]
434 operation = InterleaveGateQKV(
435 self.num_key_value_heads,
436 self.num_key_value_groups,
437 self.qk_head_dim,
438 self.v_head_dim,
439 )
440 transforms = [
441 WeightConverter(
442 source_patterns=source_weights,
443 target_patterns="linear_qkv.weight",
444 operations=[operation],
445 )
446 ]
447 if self.linear_qkv.bias is not None:
448 transforms.append(
449 WeightConverter(
450 source_patterns=["q_proj.bias", "k_proj.bias", "v_proj.bias"],
451 target_patterns="linear_qkv.bias",
452 operations=[
453 InterleaveGateQKV(
454 self.num_key_value_heads,
455 self.num_key_value_groups,
456 self.qk_head_dim,
457 self.v_head_dim,
458 )
459 ],
460 )
461 )
462 return transforms
464 def forward(
465 self,
466 hidden_states: torch.Tensor,
467 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,
468 attention_mask: torch.Tensor | None = None,
469 past_key_values: Any | None = None,
470 actual_seq_len: torch.Tensor | Sequence[int] | None = None,
471 **kwargs: Any,
472 ) -> tuple[torch.Tensor, torch.Tensor | None]:
473 """Run gated GQA with the same external contract as Transformers attention."""
474 input_shape = hidden_states.shape[:-1]
475 qkv_states = self.linear_qkv(hidden_states).view(
476 *input_shape,
477 self.num_key_value_heads,
478 self.gated_qkv_group_width,
479 )
480 query_states, key_states, value_states, gate = torch.split(
481 qkv_states,
482 (*self.qkv_split_sizes, self.num_key_value_groups * self.qk_head_dim),
483 dim=-1,
484 )
485 query_states = query_states.reshape(
486 *input_shape, self.num_heads, self.qk_head_dim
487 )
488 gate = gate.reshape(*input_shape, -1)
490 if self.q_norm is not None:
491 query_states = self.q_norm(query_states)
492 if self.k_norm is not None:
493 key_states = self.k_norm(key_states)
495 query_states, key_states = _apply_gqa_rope(
496 query_states,
497 key_states,
498 position_embeddings,
499 self.rotary_interleaved,
500 self.qk_head_dim,
501 )
502 query_states = query_states.transpose(1, 2)
503 key_states = key_states.transpose(1, 2)
504 value_states = value_states.transpose(1, 2)
506 if past_key_values is not None:
507 key_states, value_states = past_key_values.update(
508 key_states, value_states, self.layer_idx
509 )
511 attn_output, attn_weights = self.attention_interface(
512 self,
513 query_states,
514 key_states,
515 value_states,
516 attention_mask,
517 dropout=self.attention_dropout if self.training else 0.0,
518 scaling=self.scaling,
519 sliding_window=self.sliding_window,
520 actual_seq_len=actual_seq_len,
521 **kwargs,
522 )
523 attn_output = attn_output.reshape(*input_shape, -1).contiguous()
524 attn_output = attn_output * torch.sigmoid(gate)
525 attn_output = self.o_proj(attn_output)
526 return attn_output, attn_weights