Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / gqa_attention.py: 49%

225 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Grouped-query attention module using the Hyper high-performance layout.""" 

16 

17from __future__ import annotations 

18 

19from collections.abc import Callable, Mapping, Sequence 

20from typing import Any 

21 

22# This package provides PyTorch-specific high-performance modules. 

23import torch 

24from torch import nn 

25from hyper_parallel.components.checkpoint.weight_conversion import WeightConverter 

26 

27from hyper_parallel.components.checkpoint import InterleaveGateQKV, InterleaveQKV 

28from hyper_parallel.models.replacement import module_replacement 

29from hyper_parallel.components.functional import ( 

30 apply_rotary_pos_emb, 

31 apply_rotary_pos_emb_interleave, 

32 npu_fusion_attention_forward, 

33) 

34 

35 

36def rotate_half(x: torch.Tensor) -> torch.Tensor: 

37 """Rotate the two halves of the last dimension as Transformers does.""" 

38 first = x[..., : x.shape[-1] // 2] 

39 second = x[..., x.shape[-1] // 2 :] 

40 return torch.cat((-second, first), dim=-1) 

41 

42 

43def _projections_can_fuse(projections: tuple[nn.Linear, ...]) -> bool: 

44 """Return whether projections share the layout needed for fusion.""" 

45 biases = tuple(projection.bias for projection in projections) 

46 return ( 

47 len({projection.in_features for projection in projections}) == 1 

48 and len({projection.weight.requires_grad for projection in projections}) == 1 

49 and (all(bias is None for bias in biases) or all(bias is not None for bias in biases)) 

50 and (biases[0] is None or len({bias.requires_grad for bias in biases}) == 1) 

51 ) 

52 

53 

54def _apply_gqa_rope( 

55 query: torch.Tensor, 

56 key: torch.Tensor, 

57 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None, 

58 rotary_interleaved: bool, 

59 head_dim: int, 

60) -> tuple[torch.Tensor, torch.Tensor]: 

61 """Apply the configured rotary convention to grouped-query attention.""" 

62 if position_embeddings is None: 

63 return query, key 

64 cos, sin = position_embeddings 

65 if rotary_interleaved: 

66 return apply_rotary_pos_emb_interleave(query, key, cos, sin, unsqueeze_dim=2) 

67 if cos.shape[-1] >= head_dim: 

68 return apply_rotary_pos_emb(query, key, cos, sin, unsqueeze_dim=2) 

69 rotary_dim = cos.shape[-1] 

70 query_rot, query_pass = query[..., :rotary_dim], query[..., rotary_dim:] 

71 key_rot, key_pass = key[..., :rotary_dim], key[..., rotary_dim:] 

72 cos = cos.unsqueeze(2) 

73 sin = sin.unsqueeze(2) 

74 query = torch.cat((query_rot * cos + rotate_half(query_rot) * sin, query_pass), dim=-1) 

75 key = torch.cat((key_rot * cos + rotate_half(key_rot) * sin, key_pass), dim=-1) 

76 return query, key 

77 

78 

79@module_replacement 

80class GQAAttention(nn.Module): 

81 """Transformers-compatible GQA using a grouped ``linear_qkv`` layout. 

82 

83 The source module may expose a fused ``qkv_proj`` or separate 

84 ``q_proj``/``k_proj``/``v_proj`` layers. Its checkpoint layout is converted 

85 to the per-KV-head grouping consumed by the high-performance kernel. 

86 Construction only creates the target structure; ``make_transforms`` 

87 declares the checkpoint conversion. 

88 """ 

89 

90 def _initialize_dimensions(self, module: nn.Module) -> None: 

91 """Initialize head dimensions and grouped-QKV layout metadata.""" 

92 config = self.config 

93 self.num_heads = getattr(module, "num_attention_heads", config.num_attention_heads) 

94 self.num_key_value_heads = getattr(module, "num_key_value_heads", config.num_key_value_heads) 

95 if self.num_heads % self.num_key_value_heads: 

96 raise ValueError("num_attention_heads must be divisible by num_key_value_heads") 

97 self.num_key_value_groups = self.num_heads // self.num_key_value_heads 

98 self.qk_head_dim = getattr( 

99 module, "head_dim", getattr(config, "head_dim", config.hidden_size // self.num_heads) 

100 ) 

101 self.v_head_dim = ( 

102 getattr(module, "v_head_dim", None) or getattr(config, "v_head_dim", None) or self.qk_head_dim 

103 ) 

104 self.qkv_split_sizes = ( 

105 self.num_key_value_groups * self.qk_head_dim, 

106 self.qk_head_dim, 

107 self.v_head_dim, 

108 ) 

109 self.qkv_group_width = sum(self.qkv_split_sizes) 

110 

111 def _initialize_attention_options(self, module: nn.Module) -> None: 

112 """Copy supported attention options and reject unsupported variants.""" 

113 config = self.config 

114 self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5) 

115 self.attention_dropout = getattr(module, "attention_dropout", 0.0) 

116 if isinstance(self.attention_dropout, nn.Dropout): 

117 self.attention_dropout = self.attention_dropout.p 

118 self.is_causal = getattr(module, "is_causal", True) 

119 self.sliding_window = getattr(module, "sliding_window", getattr(config, "sliding_window", None)) 

120 self.rotary_interleaved = getattr( 

121 module, 

122 "rotary_interleaved", 

123 getattr(config, "rope_interleave", getattr(config, "rope_interleaved", False)), 

124 ) 

125 if getattr(module, "param_sink_number", 0) > 0: 

126 raise ValueError( 

127 "GQAAttention only implements standard GQA; parameter-sink attention " 

128 "must use the sink_attention high-performance interface." 

129 ) 

130 self.attn_groupnorm = bool(getattr(module, "attn_groupnorm", False)) 

131 self.attn_elementwise_gate = bool(getattr(module, "attn_elementwise_gate", False)) 

132 if self.attn_groupnorm or self.attn_elementwise_gate: 

133 raise ValueError( 

134 "GQAAttention only implements standard GQA; group normalization and " 

135 "elementwise attention gates require a dedicated replacement." 

136 ) 

137 

138 def _source_qkv(self, module: nn.Module) -> tuple[nn.Linear, tuple[str, ...]]: 

139 """Validate and return the source QKV projection layout.""" 

140 self._source_is_fused = hasattr(module, "qkv_proj") 

141 if self._source_is_fused: 

142 source_qkv = module.qkv_proj 

143 if not isinstance(source_qkv, nn.Linear): 

144 raise TypeError("qkv_proj must be torch.nn.Linear") 

145 expected_size = ( 

146 self.num_heads * self.qk_head_dim 

147 + self.num_key_value_heads * self.qk_head_dim 

148 + self.num_key_value_heads * self.v_head_dim 

149 ) 

150 if source_qkv.out_features != expected_size: 

151 raise ValueError("qkv_proj output size is incompatible with the configured GQA dimensions") 

152 return source_qkv, ("qkv_proj",) 

153 required = ("q_proj", "k_proj", "v_proj") 

154 if any(not hasattr(module, name) for name in required): 

155 raise TypeError("GQAAttention requires qkv_proj or q_proj/k_proj/v_proj on the source module") 

156 projections = tuple(getattr(module, name) for name in required) 

157 if any(not isinstance(projection, nn.Linear) for projection in projections): 

158 raise TypeError("q_proj, k_proj, and v_proj must be torch.nn.Linear") 

159 expected_sizes = ( 

160 self.num_heads * self.qk_head_dim, 

161 self.num_key_value_heads * self.qk_head_dim, 

162 self.num_key_value_heads * self.v_head_dim, 

163 ) 

164 if tuple(projection.out_features for projection in projections) != expected_sizes: 

165 raise ValueError("Q/K/V projection sizes are incompatible with the configured GQA dimensions") 

166 if not _projections_can_fuse(projections): 

167 raise ValueError("Q, K, and V projections cannot be represented by one fused projection") 

168 return module.q_proj, required 

169 

170 def _initialize_projections(self, module: nn.Module, source_qkv: nn.Linear) -> None: 

171 """Create the fused QKV projection and attach the source output projection.""" 

172 projection_size = self.num_key_value_heads * self.qkv_group_width 

173 self.linear_qkv = nn.Linear( 

174 source_qkv.in_features, 

175 projection_size, 

176 bias=source_qkv.bias is not None, 

177 device=source_qkv.weight.device, 

178 dtype=source_qkv.weight.dtype, 

179 ) 

180 self.linear_qkv.weight.requires_grad_(source_qkv.weight.requires_grad) 

181 if self.linear_qkv.bias is not None: 

182 self.linear_qkv.bias.requires_grad_(source_qkv.bias.requires_grad) 

183 if not hasattr(module, "o_proj") or not isinstance(module.o_proj, nn.Linear): 

184 raise TypeError("GQAAttention requires o_proj on the source module") 

185 if module.o_proj.in_features != self.num_heads * self.v_head_dim: 

186 raise ValueError("o_proj input size is incompatible with the configured GQA dimensions") 

187 self.o_proj = module.o_proj 

188 self.q_norm = getattr(module, "q_norm", None) 

189 self.k_norm = getattr(module, "k_norm", None) 

190 

191 def __init__( 

192 self, 

193 *, 

194 module: nn.Module, 

195 module_fqn: str = "", 

196 context: Mapping[str, Any] | None = None, 

197 attention_interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]] = ( 

198 npu_fusion_attention_forward 

199 ), 

200 ) -> None: 

201 """Build the high-performance module from a Transformers attention module.""" 

202 super().__init__() 

203 del module_fqn, context 

204 self.attention_interface = attention_interface 

205 config = module.config 

206 self.config = config 

207 self.layer_idx = getattr(module, "layer_idx", getattr(module, "layer_number", None)) 

208 self._initialize_dimensions(module) 

209 self._initialize_attention_options(module) 

210 source_qkv, source_projection_names = self._source_qkv(module) 

211 self._source_projection_names = source_projection_names 

212 self._initialize_projections(module, source_qkv) 

213 self.train(module.training) 

214 

215 def make_transforms(self) -> list[WeightConverter]: 

216 """Describe reversible source-checkpoint to high-performance conversion.""" 

217 if self._source_is_fused: 

218 qkv_sources = "qkv_proj.weight" 

219 bias_sources: str | list[str] = "qkv_proj.bias" 

220 else: 

221 qkv_sources = [f"{name}.weight" for name in self._source_projection_names] 

222 bias_sources = [f"{name}.bias" for name in self._source_projection_names] 

223 

224 transforms: list[WeightConverter] = [ 

225 WeightConverter( 

226 source_patterns=qkv_sources, 

227 target_patterns="linear_qkv.weight", 

228 operations=[ 

229 InterleaveQKV( 

230 self.num_key_value_heads, 

231 self.num_key_value_groups, 

232 self.qk_head_dim, 

233 self.v_head_dim, 

234 self._source_is_fused, 

235 ) 

236 ], 

237 ) 

238 ] 

239 if self.linear_qkv.bias is not None: 

240 transforms.append( 

241 WeightConverter( 

242 source_patterns=bias_sources, 

243 target_patterns="linear_qkv.bias", 

244 operations=[ 

245 InterleaveQKV( 

246 self.num_key_value_heads, 

247 self.num_key_value_groups, 

248 self.qk_head_dim, 

249 self.v_head_dim, 

250 self._source_is_fused, 

251 ) 

252 ], 

253 ) 

254 ) 

255 return transforms 

256 

257 def forward( 

258 self, 

259 hidden_states: torch.Tensor, 

260 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, 

261 attention_mask: torch.Tensor | None = None, 

262 past_key_values: Any | None = None, 

263 actual_seq_len: torch.Tensor | Sequence[int] | None = None, 

264 **kwargs: Any, 

265 ) -> tuple[torch.Tensor, torch.Tensor | None]: 

266 """Run GQA with the same external contract as Transformers attention.""" 

267 input_shape = hidden_states.shape[:-1] 

268 qkv_states = self.linear_qkv(hidden_states).view( 

269 *input_shape, 

270 self.num_key_value_heads, 

271 self.qkv_group_width, 

272 ) 

273 query_states, key_states, value_states = torch.split( 

274 qkv_states, self.qkv_split_sizes, dim=-1 

275 ) 

276 query_states = query_states.reshape( 

277 *input_shape, self.num_heads, self.qk_head_dim 

278 ) 

279 

280 if self.q_norm is not None: 

281 query_states = self.q_norm(query_states) 

282 if self.k_norm is not None: 

283 key_states = self.k_norm(key_states) 

284 

285 query_states, key_states = _apply_gqa_rope( 

286 query_states, 

287 key_states, 

288 position_embeddings, 

289 self.rotary_interleaved, 

290 self.qk_head_dim, 

291 ) 

292 query_states = query_states.transpose(1, 2) 

293 key_states = key_states.transpose(1, 2) 

294 value_states = value_states.transpose(1, 2) 

295 

296 if past_key_values is not None: 

297 key_states, value_states = past_key_values.update( 

298 key_states, value_states, self.layer_idx 

299 ) 

300 

301 attn_output, attn_weights = self.attention_interface( 

302 self, 

303 query_states, 

304 key_states, 

305 value_states, 

306 attention_mask, 

307 dropout=self.attention_dropout if self.training else 0.0, 

308 scaling=self.scaling, 

309 sliding_window=self.sliding_window, 

310 actual_seq_len=actual_seq_len, 

311 **kwargs, 

312 ) 

313 attn_output = attn_output.reshape(*input_shape, -1).contiguous() 

314 attn_output = self.o_proj(attn_output) 

315 return attn_output, attn_weights 

316 

317 

318@module_replacement 

319class GatedGQAAttention(nn.Module): 

320 """GQA with a sigmoid output gate derived from each query head. 

321 

322 The source ``q_proj`` is expected to store ``[query, gate]`` rows for each 

323 query head. This is the gated-query layout used by Qwen3.5 full attention. 

324 """ 

325 

326 def _initialize_dimensions(self, module: nn.Module) -> None: 

327 """Initialize gated grouped-QKV dimensions and attention options.""" 

328 config = self.config 

329 self.num_heads = getattr(module, "num_attention_heads", config.num_attention_heads) 

330 self.num_key_value_heads = getattr(module, "num_key_value_heads", config.num_key_value_heads) 

331 if self.num_heads % self.num_key_value_heads: 

332 raise ValueError("num_attention_heads must be divisible by num_key_value_heads") 

333 self.num_key_value_groups = self.num_heads // self.num_key_value_heads 

334 self.qk_head_dim = getattr( 

335 module, "head_dim", getattr(config, "head_dim", config.hidden_size // self.num_heads) 

336 ) 

337 self.v_head_dim = ( 

338 getattr(module, "v_head_dim", None) or getattr(config, "v_head_dim", None) or self.qk_head_dim 

339 ) 

340 self.qkv_split_sizes = ( 

341 self.num_key_value_groups * self.qk_head_dim, 

342 self.qk_head_dim, 

343 self.v_head_dim, 

344 ) 

345 self.qkv_group_width = sum(self.qkv_split_sizes) 

346 self.gated_qkv_group_width = self.qkv_group_width + self.num_key_value_groups * self.qk_head_dim 

347 self.scaling = getattr(module, "scaling", self.qk_head_dim**-0.5) 

348 self.attention_dropout = getattr(module, "attention_dropout", 0.0) 

349 if isinstance(self.attention_dropout, nn.Dropout): 

350 self.attention_dropout = self.attention_dropout.p 

351 self.is_causal = getattr(module, "is_causal", True) 

352 self.sliding_window = getattr(module, "sliding_window", getattr(config, "sliding_window", None)) 

353 self.rotary_interleaved = getattr( 

354 module, 

355 "rotary_interleaved", 

356 getattr(config, "rope_interleave", getattr(config, "rope_interleaved", False)), 

357 ) 

358 

359 def _validate_source_projections(self, module: nn.Module) -> None: 

360 """Validate separate gated-query, key, value, and output projections.""" 

361 projections = (module.q_proj, module.k_proj, module.v_proj) 

362 if any(not isinstance(projection, nn.Linear) for projection in projections): 

363 raise TypeError("q_proj, k_proj, and v_proj must be torch.nn.Linear") 

364 expected_sizes = ( 

365 2 * self.num_heads * self.qk_head_dim, 

366 self.num_key_value_heads * self.qk_head_dim, 

367 self.num_key_value_heads * self.v_head_dim, 

368 ) 

369 if tuple(projection.out_features for projection in projections) != expected_sizes: 

370 raise ValueError( 

371 "Q/gate, K, and V projection sizes are incompatible with the configured gated GQA dimensions" 

372 ) 

373 if not _projections_can_fuse(projections): 

374 raise ValueError("Q/gate, K, and V projections cannot be represented by one fused projection") 

375 attention_output_size = self.num_heads * self.v_head_dim 

376 if self.num_heads * self.qk_head_dim != attention_output_size: 

377 raise ValueError("GatedGQAAttention requires the gate width to match the attention output width") 

378 if not isinstance(module.o_proj, nn.Linear): 

379 raise TypeError("o_proj must be torch.nn.Linear") 

380 if module.o_proj.in_features != attention_output_size: 

381 raise ValueError("o_proj input size is incompatible with the configured gated GQA dimensions") 

382 

383 def _initialize_projection(self, module: nn.Module) -> None: 

384 """Create the fused gated-QKV projection and attach source output layers.""" 

385 self.linear_qkv = nn.Linear( 

386 module.q_proj.in_features, 

387 self.num_key_value_heads * self.gated_qkv_group_width, 

388 bias=module.q_proj.bias is not None, 

389 device=module.q_proj.weight.device, 

390 dtype=module.q_proj.weight.dtype, 

391 ) 

392 self.linear_qkv.weight.requires_grad_(module.q_proj.weight.requires_grad) 

393 if self.linear_qkv.bias is not None: 

394 self.linear_qkv.bias.requires_grad_(module.q_proj.bias.requires_grad) 

395 self.q_norm = module.q_norm 

396 self.k_norm = module.k_norm 

397 self.o_proj = module.o_proj 

398 

399 def __init__( 

400 self, 

401 *, 

402 module: nn.Module, 

403 module_fqn: str = "", 

404 context: Mapping[str, Any] | None = None, 

405 attention_interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]] = ( 

406 npu_fusion_attention_forward 

407 ), 

408 ) -> None: 

409 """Build gated GQA from a source with separate Q/gate, K, and V projections.""" 

410 super().__init__() 

411 del module_fqn, context 

412 if hasattr(module, "qkv_proj"): 

413 raise ValueError( 

414 "GatedGQAAttention requires separate q_proj/k_proj/v_proj projections" 

415 ) 

416 required = ("q_proj", "k_proj", "v_proj", "q_norm", "k_norm", "o_proj") 

417 if any(not hasattr(module, name) for name in required): 

418 raise TypeError( 

419 "GatedGQAAttention requires q_proj, k_proj, v_proj, q_norm, " 

420 "k_norm, and o_proj on the source module" 

421 ) 

422 

423 self.attention_interface = attention_interface 

424 self.config = module.config 

425 self.layer_idx = getattr(module, "layer_idx", None) 

426 self._initialize_dimensions(module) 

427 self._validate_source_projections(module) 

428 self._initialize_projection(module) 

429 self.train(module.training) 

430 

431 def make_transforms(self) -> list[WeightConverter]: 

432 """Describe reversible gated-query and grouped-QKV conversion.""" 

433 source_weights = ["q_proj.weight", "k_proj.weight", "v_proj.weight"] 

434 operation = InterleaveGateQKV( 

435 self.num_key_value_heads, 

436 self.num_key_value_groups, 

437 self.qk_head_dim, 

438 self.v_head_dim, 

439 ) 

440 transforms = [ 

441 WeightConverter( 

442 source_patterns=source_weights, 

443 target_patterns="linear_qkv.weight", 

444 operations=[operation], 

445 ) 

446 ] 

447 if self.linear_qkv.bias is not None: 

448 transforms.append( 

449 WeightConverter( 

450 source_patterns=["q_proj.bias", "k_proj.bias", "v_proj.bias"], 

451 target_patterns="linear_qkv.bias", 

452 operations=[ 

453 InterleaveGateQKV( 

454 self.num_key_value_heads, 

455 self.num_key_value_groups, 

456 self.qk_head_dim, 

457 self.v_head_dim, 

458 ) 

459 ], 

460 ) 

461 ) 

462 return transforms 

463 

464 def forward( 

465 self, 

466 hidden_states: torch.Tensor, 

467 position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, 

468 attention_mask: torch.Tensor | None = None, 

469 past_key_values: Any | None = None, 

470 actual_seq_len: torch.Tensor | Sequence[int] | None = None, 

471 **kwargs: Any, 

472 ) -> tuple[torch.Tensor, torch.Tensor | None]: 

473 """Run gated GQA with the same external contract as Transformers attention.""" 

474 input_shape = hidden_states.shape[:-1] 

475 qkv_states = self.linear_qkv(hidden_states).view( 

476 *input_shape, 

477 self.num_key_value_heads, 

478 self.gated_qkv_group_width, 

479 ) 

480 query_states, key_states, value_states, gate = torch.split( 

481 qkv_states, 

482 (*self.qkv_split_sizes, self.num_key_value_groups * self.qk_head_dim), 

483 dim=-1, 

484 ) 

485 query_states = query_states.reshape( 

486 *input_shape, self.num_heads, self.qk_head_dim 

487 ) 

488 gate = gate.reshape(*input_shape, -1) 

489 

490 if self.q_norm is not None: 

491 query_states = self.q_norm(query_states) 

492 if self.k_norm is not None: 

493 key_states = self.k_norm(key_states) 

494 

495 query_states, key_states = _apply_gqa_rope( 

496 query_states, 

497 key_states, 

498 position_embeddings, 

499 self.rotary_interleaved, 

500 self.qk_head_dim, 

501 ) 

502 query_states = query_states.transpose(1, 2) 

503 key_states = key_states.transpose(1, 2) 

504 value_states = value_states.transpose(1, 2) 

505 

506 if past_key_values is not None: 

507 key_states, value_states = past_key_values.update( 

508 key_states, value_states, self.layer_idx 

509 ) 

510 

511 attn_output, attn_weights = self.attention_interface( 

512 self, 

513 query_states, 

514 key_states, 

515 value_states, 

516 attention_mask, 

517 dropout=self.attention_dropout if self.training else 0.0, 

518 scaling=self.scaling, 

519 sliding_window=self.sliding_window, 

520 actual_seq_len=actual_seq_len, 

521 **kwargs, 

522 ) 

523 attn_output = attn_output.reshape(*input_shape, -1).contiguous() 

524 attn_output = attn_output * torch.sigmoid(gate) 

525 attn_output = self.o_proj(attn_output) 

526 return attn_output, attn_weights