Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / modules / swiglu_mlp.py: 0%
84 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Transformers-compatible SwiGLU MLP with a fused Gate/Up projection."""
17from __future__ import annotations
19from collections.abc import Mapping
20from typing import Any
22# This package provides PyTorch-specific high-performance modules.
23import torch
24import torch.nn.functional as F
25from torch import nn
26from hyper_parallel.components.checkpoint.weight_conversion import (
27 WeightConverter,
28 WeightRenaming,
29)
31from hyper_parallel.components.checkpoint import ConcatenateWithSections
32from hyper_parallel.models.replacement import module_replacement
33from hyper_parallel.components.functional import swiglu
36@module_replacement
37class SwiGLUMLP(nn.Module):
38 """Transformers-compatible SwiGLU MLP using one fused Gate/Up matmul."""
40 @staticmethod
41 def _source_projections(module: nn.Module) -> tuple[nn.Linear, nn.Linear, nn.Linear]:
42 """Validate and return the source Gate, Up, and Down projections."""
43 required = ("gate_proj", "up_proj", "down_proj")
44 missing = [name for name in required if not hasattr(module, name)]
45 if missing:
46 raise TypeError(f"SwiGLUMLP source module is missing: {missing}")
47 projections = (module.gate_proj, module.up_proj, module.down_proj)
48 if not all(isinstance(projection, nn.Linear) for projection in projections):
49 raise TypeError("SwiGLUMLP requires Gate, Up, and Down projections to be nn.Linear")
50 return projections
52 @staticmethod
53 def _validate_projection_contract(
54 gate_proj: nn.Linear,
55 up_proj: nn.Linear,
56 down_proj: nn.Linear,
57 ) -> None:
58 """Validate source projection shapes, placement, and training policy."""
59 if gate_proj.in_features != up_proj.in_features:
60 raise ValueError("Gate and Up projections must have the same input size")
61 if gate_proj.out_features != up_proj.out_features:
62 raise ValueError("Gate and Up projections must have the same output size")
63 if down_proj.in_features != gate_proj.out_features:
64 raise ValueError("Down projection input size must equal the SwiGLU intermediate size")
65 if down_proj.out_features != gate_proj.in_features:
66 raise ValueError("Down projection output size must equal the MLP hidden size")
67 if gate_proj.weight.device != up_proj.weight.device or gate_proj.weight.dtype != up_proj.weight.dtype:
68 raise ValueError("Gate and Up projection weights must share device and dtype")
69 if gate_proj.weight.device != down_proj.weight.device or gate_proj.weight.dtype != down_proj.weight.dtype:
70 raise ValueError("Gate, Up, and Down projection weights must share device and dtype")
71 if gate_proj.weight.requires_grad != up_proj.weight.requires_grad:
72 raise ValueError("Gate and Up projection weights must share a training policy")
73 if (gate_proj.bias is None) != (up_proj.bias is None):
74 raise ValueError("Gate and Up projections must use the same bias policy")
75 if gate_proj.bias is not None and gate_proj.bias.requires_grad != up_proj.bias.requires_grad:
76 raise ValueError("Gate and Up projection biases must share a training policy")
78 def _initialize_fc1(self, gate_proj: nn.Linear, up_proj: nn.Linear) -> None:
79 """Create the packed Gate/Up projection with source parameter values."""
80 pack = ConcatenateWithSections((self.intermediate_size, self.intermediate_size))
81 packed_weight = pack.convert(
82 {"gate": gate_proj.weight.detach(), "up": up_proj.weight.detach()},
83 ["gate", "up"],
84 ["linear_fc1.weight"],
85 )["linear_fc1.weight"]
86 self.linear_fc1 = nn.Linear(
87 self.hidden_size,
88 2 * self.intermediate_size,
89 bias=gate_proj.bias is not None,
90 device=gate_proj.weight.device,
91 dtype=gate_proj.weight.dtype,
92 )
93 self.linear_fc1.weight = nn.Parameter(packed_weight, requires_grad=gate_proj.weight.requires_grad)
94 if gate_proj.bias is not None:
95 packed_bias = pack.convert(
96 {"gate": gate_proj.bias.detach(), "up": up_proj.bias.detach()},
97 ["gate", "up"],
98 ["linear_fc1.bias"],
99 )["linear_fc1.bias"]
100 self.linear_fc1.bias = nn.Parameter(packed_bias, requires_grad=gate_proj.bias.requires_grad)
102 def _initialize_fc2(self, down_proj: nn.Linear) -> None:
103 """Create the Down projection and reuse its source parameters."""
104 self.linear_fc2 = nn.Linear(
105 down_proj.in_features,
106 down_proj.out_features,
107 bias=down_proj.bias is not None,
108 device=down_proj.weight.device,
109 dtype=down_proj.weight.dtype,
110 )
111 self.linear_fc2.weight = down_proj.weight
112 self.linear_fc2.bias = down_proj.bias
114 def __init__(
115 self,
116 *,
117 module: nn.Module,
118 module_fqn: str = "",
119 context: Mapping[str, Any] | None = None,
120 ) -> None:
121 """Build the high-performance MLP from separate source projections.
123 Args:
124 module: Source MLP exposing ``gate_proj``, ``up_proj``, and ``down_proj``.
125 module_fqn: Fully qualified source-module name supplied by replacement.
126 context: Replacement context supplied by Trainer.
128 Raises:
129 TypeError: If the source projection modules are missing or unsupported.
130 ValueError: If projection layouts, training policies, or activation differ.
131 """
132 super().__init__()
133 del module_fqn, context
134 gate_proj, up_proj, down_proj = self._source_projections(module)
135 self._validate_projection_contract(gate_proj, up_proj, down_proj)
137 config = getattr(module, "config", None)
138 hidden_act = getattr(config, "hidden_act", getattr(config, "hidden_activation", None))
139 if hidden_act is not None and hidden_act not in ("silu", "swiglu"):
140 raise ValueError(f"SwiGLUMLP requires a SiLU activation, but got {hidden_act}")
141 self.config = config
142 self.hidden_size = gate_proj.in_features
143 self.intermediate_size = gate_proj.out_features
145 self._initialize_fc1(gate_proj, up_proj)
146 self._initialize_fc2(down_proj)
147 self.train(module.training)
149 def make_transforms(self) -> list[WeightRenaming | WeightConverter]:
150 """Describe reversible source-checkpoint to high-performance conversion."""
151 transforms: list[WeightRenaming | WeightConverter] = [
152 WeightConverter(
153 source_patterns=["gate_proj.weight", "up_proj.weight"],
154 target_patterns="linear_fc1.weight",
155 operations=[
156 ConcatenateWithSections((self.intermediate_size, self.intermediate_size))
157 ],
158 )
159 ]
160 if self.linear_fc1.bias is not None:
161 transforms.append(
162 WeightConverter(
163 source_patterns=["gate_proj.bias", "up_proj.bias"],
164 target_patterns="linear_fc1.bias",
165 operations=[
166 ConcatenateWithSections((self.intermediate_size, self.intermediate_size))
167 ],
168 )
169 )
170 transforms.append(WeightRenaming("down_proj.weight", "linear_fc2.weight"))
171 if self.linear_fc2.bias is not None:
172 transforms.append(WeightRenaming("down_proj.bias", "linear_fc2.bias"))
173 return transforms
175 def forward(self, x: torch.Tensor) -> torch.Tensor:
176 """Apply the fused Gate/Up projection, SwiGLU, and Down projection."""
177 intermediate_parallel = self.linear_fc1(x)
178 if intermediate_parallel.device.type == "npu":
179 intermediate_parallel = swiglu( # pylint: disable=not-callable
180 intermediate_parallel
181 )
182 else:
183 gate, up = intermediate_parallel.chunk(2, dim=-1)
184 intermediate_parallel = F.silu(gate) * up
185 return self.linear_fc2(intermediate_parallel)