Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed / _builder / fsdp_adapter.py: 82%

398 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""fsdp_adapter: FSDP2 construction-time adapter (05 §15.2.4). 

16 

17Wrap policy, mesh/policy construction, ``fully_shard`` application and 

18prefetch configuration all happen at model-construction time (not in the 

19Trainer step runtime); ``FSDP2Manager`` / ``_instantiate_fsdp2`` and the 

20``parallelize`` signature are unchanged. The source-layout discovery logic 

21(FQN/DTensor source-shard resolution) merged into 

22``distributed/_builder/source_shard.py`` (05 row 406). 

23""" 

24 

25from __future__ import annotations 

26 

27import fnmatch 

28import logging 

29from collections import defaultdict 

30from collections.abc import Mapping, Sequence 

31from dataclasses import dataclass 

32from typing import TYPE_CHECKING, Any 

33 

34import torch 

35from torch.nn import Module as ModuleClass 

36from torch.nn import Parameter as ParameterClass 

37 

38from hyper_parallel.components.modules.module_family import collect_family_modules 

39import hyper_parallel.core.fully_shard.utils as fully_shard_utils 

40from hyper_parallel import DeviceMesh, HSDPModule, fully_shard 

41from hyper_parallel.core.fully_shard.hsdp_utils import get_managed_modules_parameters 

42from hyper_parallel.distributed._builder.source_shard import ( 

43 SourceShardInfoByFQN, # pylint: disable=unused-import 

44 SourceShardInfoByParam, # pylint: disable=unused-import 

45 _build_managed_source_shard_info, 

46 _build_source_shard_info_by_param, 

47 _source_infos_for_fully_shard, 

48) 

49from hyper_parallel.models.build_options import FSDP2Config 

50from hyper_parallel.models.registry import get_model_adapter 

51 

52if TYPE_CHECKING: 

53 from hyper_parallel.distributed.mesh import ( 

54 MeshContext, 

55 ) 

56 

57logger = logging.getLogger(__name__) 

58 

59 

60@dataclass(frozen=True) 

61class _WrapModuleInfo: 

62 """One FSDP child unit selected by automatic or adapter policy.""" 

63 

64 fqn: str 

65 module: ModuleClass 

66 

67 

68def _is_expert_source_mesh(mesh: DeviceMesh | None) -> bool: 

69 """Whether a source mesh carries the expert (EP) axis.""" 

70 return mesh is not None and "ep" in (getattr(mesh, "mesh_dim_names", None) or ()) 

71 

72 

73def _get_checkpoint_wrapped_module(module: ModuleClass) -> ModuleClass | None: 

74 """Return the module directly wrapped by a supported checkpoint wrapper.""" 

75 for attr_name in ("_wrapped_module", "_checkpoint_wrapped_module"): 

76 wrapped_module = getattr(module, attr_name, None) 

77 if isinstance(wrapped_module, ModuleClass): 

78 return wrapped_module 

79 return None 

80 

81 

82def _declared_expert_unit_fqns(model, expert_parameters): 

83 """Find family/native execution units by identity rather than an experts path suffix.""" 

84 expert_module_fqns = set() 

85 for boundary in collect_family_modules(model): 

86 parameters = set(boundary.module.parameters()) 

87 if parameters and parameters <= expert_parameters: 

88 expert_module_fqns.add(boundary.fqns[0]) 

89 for fqn, module in model.named_modules(): 

90 binding = getattr(module, "_hp_parallel_binding", None) 

91 if binding is not None and binding.ep_context is not None: 

92 expert_module_fqns.add(fqn) 

93 expert_module_fqns.discard("") 

94 return expert_module_fqns 

95 

96 

97class FSDP2Manager: 

98 """Configure and apply nested FSDP2 wrapping for a dual-mode model.""" 

99 

100 def __init__( 

101 self, 

102 config: FSDP2Config, 

103 mesh_context: MeshContext, 

104 fp32_main_params: bool = False, 

105 ) -> None: 

106 """Initialize the manager from resolved YAML config and runtime mesh. 

107 

108 Args: 

109 config: Resolved top-level ``fsdp_config``. 

110 mesh_context: Runtime topology containing the shared world mesh. 

111 fp32_main_params: Whether FSDP writes reduced gradients to 

112 ``main_grad`` for the fp32 main-parameter optimizer wrapper. 

113 """ 

114 self.config = config 

115 self.mesh_context = mesh_context 

116 self.fp32_main_params = fp32_main_params 

117 self._mp_policy: fully_shard_utils.MixedPrecisionPolicy | None = None 

118 

119 

120 def _resolve_replicate_params( 

121 self, 

122 model: ModuleClass, 

123 ) -> set[ParameterClass] | None: 

124 """Resolve configured replicate-parameter FQNs on the final model.""" 

125 if not self.config.replicate_params: 

126 return None 

127 

128 parameters_by_fqn = dict(model.named_parameters(remove_duplicate=False)) 

129 replicate_params = set() 

130 for parameter_fqn in self.config.replicate_params: 

131 parameter = parameters_by_fqn.get(parameter_fqn) 

132 if parameter is None: 

133 raise ValueError( 

134 "replicate_params contains unknown parameter FQN: " 

135 f"{parameter_fqn}" 

136 ) 

137 replicate_params.add(parameter) 

138 return replicate_params 

139 

140 def _build_expert_fsdp_mesh(self) -> DeviceMesh: 

141 """Select the expert FSDP child mesh.""" 

142 expert_mesh = self.mesh_context.fsdp_moe_mesh 

143 if expert_mesh is None: 

144 raise ValueError("Expert FSDP units require MeshContext.fsdp_moe_mesh") 

145 if "edp_replicate" in (expert_mesh.mesh_dim_names or ()): 

146 return expert_mesh[("edp_replicate", "edp_shard")] 

147 return expert_mesh["edp_shard"] 

148 

149 def _build_dense_fsdp_mesh(self) -> DeviceMesh | None: 

150 """Select the prebuilt dense FSDP child mesh when available.""" 

151 dense_mesh = self.mesh_context.fsdp_non_moe_mesh 

152 if dense_mesh is None: 

153 return None 

154 if self.mesh_context.dp_replicate_size > 1: 

155 return dense_mesh[("fsdp_replicate", "fsdp_shard")] 

156 return dense_mesh["fsdp_shard"] 

157 

158 def _build_compatibility_fsdp_mesh(self) -> DeviceMesh: 

159 """Build an FSDP mesh from the combined world mesh.""" 

160 world_mesh = self.mesh_context.device_mesh 

161 if world_mesh is None: 

162 raise ValueError("FSDP2 requires MeshContext.fsdp_non_moe_mesh") 

163 mesh_dim_names = world_mesh.mesh_dim_names 

164 if mesh_dim_names is None: 

165 raise ValueError("FSDP2 requires named DeviceMesh dimensions") 

166 shard_dim_names = tuple( 

167 dim_name for dim_name in ("dp_shard", "cp") if dim_name in mesh_dim_names 

168 ) 

169 if not shard_dim_names: 

170 raise ValueError("FSDP2 requires a dp_shard or cp dimension in the world mesh") 

171 shard_mesh = ( 

172 world_mesh[shard_dim_names[0]] 

173 if len(shard_dim_names) == 1 

174 else world_mesh[shard_dim_names].flatten("fsdp_shard") 

175 ) 

176 if "dp_replicate" not in mesh_dim_names: 

177 return shard_mesh 

178 return DeviceMesh.concatenate([world_mesh["dp_replicate"], shard_mesh]) 

179 

180 def _build_fsdp_actual_mesh(self, expert: bool = False) -> DeviceMesh: 

181 """Select the prebuilt dense or expert FSDP child mesh.""" 

182 if expert: 

183 return self._build_expert_fsdp_mesh() 

184 dense_mesh = self._build_dense_fsdp_mesh() 

185 if dense_mesh is not None: 

186 return dense_mesh 

187 return self._build_compatibility_fsdp_mesh() 

188 

189 def parameter_mesh_bindings( 

190 self, model: ModuleClass, source_shard_info: SourceShardInfoByFQN | None, 

191 ignored_params: set[ParameterClass] | None = None, 

192 ) -> dict[ParameterClass, DeviceMesh]: 

193 """Resolve unit-owned parameter meshes without installing FSDP hooks. 

194 

195 Args: 

196 model: Final model before graph capture or eager wrapping. 

197 source_shard_info: The planner's non-FSDP parameter placements. 

198 ignored_params: Parameters managed outside FSDP. 

199 

200 The same managed-parameter selector runs inside-out in both paths. 

201 Claimed identities model the initialized inner units, including tied 

202 aliases; nearest-parent ownership alone would not match fully_shard. 

203 """ 

204 metadata = _build_source_shard_info_by_param(self, model, source_shard_info) 

205 units = self._find_wrap_modules(model, metadata) 

206 units = [unit.module for unit in sorted(units, key=lambda unit: unit.fqn.count("."), reverse=True)] 

207 claimed = set(ignored_params or ()) 

208 bindings = {} 

209 for owner in [*units, model]: 

210 parameters = get_managed_modules_parameters((owner,), claimed) 

211 source = _build_managed_source_shard_info(self, owner, metadata, claimed) 

212 expert = self._uses_expert_mesh(source) 

213 if owner is model and expert: 

214 raise ValueError("Routed expert parameters must belong to a nested experts FSDP unit") 

215 mesh = self._build_fsdp_actual_mesh(expert) 

216 bindings.update(dict.fromkeys(parameters, mesh)) 

217 claimed.update(parameters) 

218 return bindings 

219 

220 def _resolve_custom_params( 

221 self, parameters_by_fqn: Mapping[str, ParameterClass] 

222 ) -> dict[ParameterClass, dict[str, Any]]: 

223 """Resolve configured names and patterns against the supplied parameter mapping.""" 

224 custom_params = {} 

225 for parameter_fqn, custom_dtypes in self.config.mix_precision.custom_params.items(): 

226 matched_fqns = [parameter_fqn] if parameter_fqn in parameters_by_fqn else [ 

227 fqn for fqn in parameters_by_fqn if fnmatch.fnmatchcase(fqn, parameter_fqn) 

228 ] 

229 if not matched_fqns: 

230 raise ValueError(f"mix_precision.custom_params matches no parameter: {parameter_fqn}") 

231 for matched_fqn in matched_fqns: 

232 parameter = parameters_by_fqn[matched_fqn] 

233 if parameter in custom_params and custom_params[parameter] != custom_dtypes: 

234 raise ValueError(f"mix_precision.custom_params has conflicting overrides for {matched_fqn}") 

235 custom_params[parameter] = dict(custom_dtypes) 

236 return custom_params 

237 

238 def _build_mixed_precision_policy( 

239 self, custom_params: Mapping[ParameterClass, Mapping[str, Any]] | None = None 

240 ) -> fully_shard_utils.MixedPrecisionPolicy: 

241 """Build or reuse the shared precision policy for the current model.""" 

242 if custom_params is None and self._mp_policy is not None: 

243 return self._mp_policy 

244 mix_precision = self.config.mix_precision 

245 dtype_by_name = { 

246 None: None, 

247 "bfloat16": torch.bfloat16, 

248 "float16": torch.float16, 

249 "float32": torch.float32, 

250 } 

251 custom_params = { 

252 parameter: {name: dtype_by_name[dtype] for name, dtype in custom_dtypes.items()} 

253 for parameter, custom_dtypes in (custom_params or {}).items() 

254 } 

255 self._mp_policy = fully_shard_utils.MixedPrecisionPolicy( 

256 param_dtype=dtype_by_name[mix_precision.param_dtype], 

257 reduce_dtype=dtype_by_name[mix_precision.reduce_dtype], 

258 output_dtype=dtype_by_name[mix_precision.output_dtype], 

259 cast_forward_inputs=mix_precision.cast_forward_inputs, 

260 apply_grad_on_fp32_main_grad=self.fp32_main_params, 

261 custom_params=custom_params if custom_params else None, 

262 ) 

263 return self._mp_policy 

264 

265 def _build_offload_policy(self) -> fully_shard_utils.OffloadPolicy: 

266 """Return CPU offload when enabled, otherwise the default no-offload policy.""" 

267 if not self.config.enable_offload: 

268 return fully_shard_utils.OffloadPolicy() 

269 return fully_shard_utils.CPUOffloadPolicy() 

270 

271 def _build_fully_shard_kwargs( 

272 self, 

273 fsdp_mesh: DeviceMesh, 

274 ) -> tuple[dict[str, Any], dict[str, Any]]: 

275 """Build child and root kwargs with the same configured reshard policy.""" 

276 mp_policy = self._build_mixed_precision_policy() 

277 offload_policy = self._build_offload_policy() 

278 

279 fsdp_sublayer_kwargs = { 

280 "mesh": fsdp_mesh, 

281 "reshard_after_forward": self.config.reshard_after_forward, 

282 "mp_policy": mp_policy, 

283 "offload_policy": offload_policy, 

284 "comm_fusion": self.config.comm_fusion, 

285 "comm_fusion_zero_copy": self.config.comm_fusion_zero_copy, 

286 } 

287 fsdp_root_kwargs = fsdp_sublayer_kwargs.copy() 

288 return fsdp_sublayer_kwargs, fsdp_root_kwargs 

289 

290 @staticmethod 

291 def _find_transformer_block_modules( 

292 model: ModuleClass, 

293 excluded_subtree_module_fqns: set[str] | None = None, 

294 ) -> tuple[list[_WrapModuleInfo], set[int]]: 

295 """Find transformer blocks under HF gradient-checkpointing containers. 

296 

297 Args: 

298 model: Root model to inspect. 

299 excluded_subtree_module_fqns: Roots declared by a model adapter as 

300 non-decoder branches or self-contained FSDP units. Automatic 

301 decoder discovery skips these roots and their descendants. 

302 """ 

303 wrap_modules = [] 

304 wrapped_module_ids = set() 

305 excluded_subtree_module_fqns = excluded_subtree_module_fqns or set() 

306 

307 def is_inside_declared_subtree(module_fqn: str) -> bool: 

308 """Whether one module is inside an automatic-discovery exclusion. 

309 

310 Args: 

311 module_fqn: Fully qualified name of the module. 

312 """ 

313 return any( 

314 module_fqn == declared_fqn 

315 or module_fqn.startswith(f"{declared_fqn}.") 

316 for declared_fqn in excluded_subtree_module_fqns 

317 ) 

318 

319 for container_fqn, container in model.named_modules(): 

320 if container_fqn and is_inside_declared_subtree(container_fqn): 

321 continue 

322 if id(container) in wrapped_module_ids: 

323 continue 

324 if not hasattr(container, "gradient_checkpointing"): 

325 continue 

326 for child_name, child in container.named_children(): 

327 child_fqn = f"{container_fqn}.{child_name}" if container_fqn else child_name 

328 if is_inside_declared_subtree(child_fqn): 

329 continue 

330 blocks = list(child.named_children()) 

331 if not blocks: 

332 continue 

333 for block_name, block in blocks: 

334 if id(block) in wrapped_module_ids: 

335 continue 

336 wrapped_module_ids.add(id(block)) 

337 wrapped_module = _get_checkpoint_wrapped_module(block) 

338 if wrapped_module is not None: 

339 # The wrapper and its direct child represent one logical 

340 # transformer block during module-tree traversal. 

341 wrapped_module_ids.add(id(wrapped_module)) 

342 wrap_modules.append( 

343 _WrapModuleInfo(f"{child_fqn}.{block_name}", block) 

344 ) 

345 return wrap_modules, wrapped_module_ids 

346 

347 def _find_expert_wrap_modules( 

348 self, 

349 model: ModuleClass, 

350 metadata_by_parameter: SourceShardInfoByParam | None, 

351 wrapped_module_ids: set[int], 

352 ) -> list[_WrapModuleInfo]: 

353 """Find nested expert containers whose parameters use the expert mesh.""" 

354 if metadata_by_parameter is None or self.mesh_context.fsdp_moe_mesh is None: 

355 return [] 

356 expert_parameters = { 

357 parameter 

358 for parameter, source_shard_info in metadata_by_parameter.items() 

359 if _is_expert_source_mesh(source_shard_info.mesh) 

360 } 

361 expert_module_fqns = { 

362 parameter_fqn.split(".experts.", 1)[0] + ".experts" 

363 for parameter_fqn, parameter in model.named_parameters(remove_duplicate=False) 

364 if parameter in expert_parameters and ".experts." in parameter_fqn 

365 } 

366 expert_module_fqns.update(_declared_expert_unit_fqns(model, expert_parameters)) 

367 expert_module_fqns.discard("") 

368 module_by_fqn = dict(model.named_modules()) 

369 expert_wrap_modules = [] 

370 for expert_module_fqn in sorted(expert_module_fqns): 

371 expert_module = module_by_fqn.get(expert_module_fqn) 

372 if expert_module is None or id(expert_module) in wrapped_module_ids: 

373 continue 

374 wrapped_module_ids.add(id(expert_module)) 

375 expert_wrap_modules.append( 

376 _WrapModuleInfo(expert_module_fqn, expert_module) 

377 ) 

378 return expert_wrap_modules 

379 

380 @staticmethod 

381 def _get_model_adapter_spec(model: ModuleClass) -> Any | None: 

382 """Resolve a model adapter from the final model configuration.""" 

383 if hasattr(model, "_hp_model_adapter_spec"): 

384 return model._hp_model_adapter_spec # pylint: disable=protected-access 

385 config = getattr(model, "config", None) 

386 if config is None: 

387 return None 

388 identities = ( 

389 getattr(config, "model_type", ""), 

390 *((getattr(config, "architectures", None) or [])[:1]), 

391 ) 

392 for identity in identities: 

393 if identity: 

394 adapter_spec = get_model_adapter(identity) 

395 if adapter_spec is not None: 

396 return adapter_spec 

397 return None 

398 

399 @classmethod 

400 def _find_adapter_declared_wrap_modules( 

401 cls, 

402 model: ModuleClass, 

403 ) -> list[_WrapModuleInfo]: 

404 """Resolve adapter-declared FSDP units outside HF decoder containers. 

405 

406 ``ModelAdapterSpec.fsdp_wrap_modules`` keeps architecture-specific 

407 execution boundaries on the family adapter side. It receives the 

408 finalized model and returns exact root-relative module FQNs. 

409 

410 Args: 

411 model: Root model being wrapped. 

412 

413 Returns: 

414 Additional FSDP child units selected by model declarations. 

415 

416 Raises: 

417 ValueError: If a declaration has an invalid type, targets no 

418 module in the model tree, or aliases the same module twice. 

419 """ 

420 adapter_spec = cls._get_model_adapter_spec(model) 

421 if adapter_spec is None or adapter_spec.fsdp_wrap_modules is None: 

422 return [] 

423 

424 declared_fqns = adapter_spec.fsdp_wrap_modules(model) 

425 if isinstance(declared_fqns, str) or not isinstance(declared_fqns, Sequence): 

426 raise ValueError( 

427 "ModelAdapterSpec.fsdp_wrap_modules must return a sequence of exact module FQNs" 

428 ) 

429 module_by_fqn = dict(model.named_modules()) 

430 declared_wrap_modules = [] 

431 wrapped_module_ids = set() 

432 for module_fqn in declared_fqns: 

433 if not isinstance(module_fqn, str) or not module_fqn: 

434 raise ValueError( 

435 "ModelAdapterSpec.fsdp_wrap_modules entries must be non-empty strings" 

436 ) 

437 wrap_module = module_by_fqn.get(module_fqn) 

438 if wrap_module is None: 

439 raise ValueError( 

440 "ModelAdapterSpec.fsdp_wrap_modules declared an unknown module: " 

441 f"{module_fqn}" 

442 ) 

443 if id(wrap_module) in wrapped_module_ids: 

444 raise ValueError( 

445 "ModelAdapterSpec.fsdp_wrap_modules aliases one module more than once: " 

446 f"{module_fqn}" 

447 ) 

448 wrapped_module_ids.add(id(wrap_module)) 

449 declared_wrap_modules.append(_WrapModuleInfo(module_fqn, wrap_module)) 

450 return declared_wrap_modules 

451 

452 @classmethod 

453 def _find_adapter_excluded_subtrees(cls, model: ModuleClass) -> set[str]: 

454 """Resolve model branches excluded from automatic decoder discovery.""" 

455 adapter_spec = cls._get_model_adapter_spec(model) 

456 excluded_subtrees_provider = getattr(adapter_spec, "fsdp_excluded_subtrees", None) 

457 if excluded_subtrees_provider is None: 

458 return set() 

459 if not callable(excluded_subtrees_provider): 

460 raise ValueError("ModelAdapterSpec.fsdp_excluded_subtrees must be callable") 

461 excluded_fqns = excluded_subtrees_provider(model) # pylint: disable=not-callable 

462 if isinstance(excluded_fqns, str) or not isinstance(excluded_fqns, Sequence): 

463 raise ValueError( 

464 "ModelAdapterSpec.fsdp_excluded_subtrees must return a sequence of exact module FQNs" 

465 ) 

466 module_by_fqn = dict(model.named_modules()) 

467 for module_fqn in excluded_fqns: 

468 if not isinstance(module_fqn, str) or not module_fqn: 

469 raise ValueError( 

470 "ModelAdapterSpec.fsdp_excluded_subtrees entries must be non-empty strings" 

471 ) 

472 if module_fqn not in module_by_fqn: 

473 raise ValueError( 

474 "ModelAdapterSpec.fsdp_excluded_subtrees declared an unknown module: " 

475 f"{module_fqn}" 

476 ) 

477 return set(excluded_fqns) 

478 

479 def _find_class_wrap_modules(self, model: ModuleClass) -> list[_WrapModuleInfo]: 

480 """Select explicit classes, retaining checkpoint wrappers as the units.""" 

481 requested = set(self.config.wrapper_class) 

482 matched = set() 

483 selected = [] 

484 seen = set() 

485 aliases = defaultdict(list) 

486 for name, parameter in model.named_parameters(remove_duplicate=False): 

487 aliases[parameter].append(name) 

488 for fqn, module in model.named_modules(): 

489 if not fqn or id(module) in seen: 

490 continue 

491 underlying = _get_checkpoint_wrapped_module(module) 

492 candidate = underlying if underlying is not None else module 

493 class_name = type(candidate).__name__ 

494 if class_name not in requested: 

495 continue 

496 matched.add(class_name) 

497 parameters = list(module.parameters()) 

498 if not parameters: 

499 raise ValueError(f"wrapper_class selected a parameterless module: {fqn}") 

500 if any(not name.startswith(f"{fqn}.") for param in parameters for name in aliases[param]): 

501 raise ValueError(f"wrapper_class module shares parameters outside its subtree: {fqn}") 

502 selected.append(_WrapModuleInfo(fqn, module)) 

503 seen.add(id(module)) 

504 seen.add(id(candidate)) 

505 if requested - matched: 

506 raise ValueError(f"wrapper_class matched no child module for: {sorted(requested - matched)}") 

507 return selected 

508 

509 def _find_wrap_modules( 

510 self, 

511 model: ModuleClass, 

512 metadata_by_parameter: SourceShardInfoByParam | None = None, 

513 ) -> list[_WrapModuleInfo]: 

514 """Find automatic decoder/expert units and adapter-declared units.""" 

515 if self.config.wrapper_class: 

516 wrap_modules = self._find_class_wrap_modules(model) 

517 wrapped_module_ids = {id(unit.module) for unit in wrap_modules} 

518 else: 

519 declared_wrap_modules = self._find_adapter_declared_wrap_modules(model) 

520 if declared_wrap_modules: 

521 logger.info( 

522 "Adapter-declared FSDP child units: %s", 

523 [wrap_module.fqn for wrap_module in declared_wrap_modules], 

524 ) 

525 excluded_subtree_module_fqns = { 

526 wrap_module.fqn for wrap_module in declared_wrap_modules 

527 } 

528 excluded_subtree_module_fqns.update( 

529 self._find_adapter_excluded_subtrees(model) 

530 ) 

531 wrap_modules, wrapped_module_ids = self._find_transformer_block_modules( 

532 model, 

533 excluded_subtree_module_fqns=excluded_subtree_module_fqns, 

534 ) 

535 wrapped_module_ids.update( 

536 id(wrap_module.module) for wrap_module in declared_wrap_modules 

537 ) 

538 wrap_modules.extend(declared_wrap_modules) 

539 wrap_modules.extend( 

540 self._find_expert_wrap_modules( 

541 model, 

542 metadata_by_parameter, 

543 wrapped_module_ids, 

544 ) 

545 ) 

546 if not wrap_modules: 

547 raise ValueError( 

548 "wrap_policy='transformer_block' found no decoder, expert, or " 

549 "adapter-declared FSDP child unit" 

550 ) 

551 return wrap_modules 

552 

553 @staticmethod 

554 def _resolve_parameter_owners( 

555 model: ModuleClass, 

556 wrap_modules: list[_WrapModuleInfo], 

557 ) -> dict[ParameterClass, ModuleClass]: 

558 """Assign each parameter to its deepest common FSDP wrap module.""" 

559 aliases_by_parameter: dict[ParameterClass, list[str]] = defaultdict(list) 

560 for parameter_fqn, parameter in model.named_parameters(remove_duplicate=False): 

561 aliases_by_parameter[parameter].append(parameter_fqn) 

562 

563 owner_by_parameter = {} 

564 for parameter, parameter_fqns in aliases_by_parameter.items(): 

565 common_wrap_modules = [ 

566 wrap_module 

567 for wrap_module in wrap_modules 

568 if all( 

569 parameter_fqn.startswith(f"{wrap_module.fqn}.") 

570 for parameter_fqn in parameter_fqns 

571 ) 

572 ] 

573 if common_wrap_modules: 

574 owner_by_parameter[parameter] = max( 

575 common_wrap_modules, 

576 key=lambda wrap_module: wrap_module.fqn.count("."), 

577 ).module 

578 else: 

579 owner_by_parameter[parameter] = model 

580 return owner_by_parameter 

581 

582 

583 def _uses_expert_mesh(self, metadata_by_parameter: SourceShardInfoByParam | None) -> bool: 

584 """Return whether one FSDP unit owns routed-expert source metadata.""" 

585 if metadata_by_parameter is None or self.mesh_context.fsdp_moe_mesh is None: 

586 return False 

587 return any( 

588 _is_expert_source_mesh(source_shard_info.mesh) 

589 for source_shard_info in metadata_by_parameter.values() 

590 ) 

591 

592 

593 @staticmethod 

594 def _build_managed_replicate_params( 

595 owner: ModuleClass, 

596 owner_by_parameter: Mapping[ParameterClass, ModuleClass], 

597 replicate_params: set[ParameterClass] | None, 

598 ignored_params: set[ParameterClass] | None = None, 

599 ) -> set[ParameterClass] | None: 

600 """Select configured replicate parameters owned by one FSDP module.""" 

601 if replicate_params is None: 

602 return None 

603 return { 

604 parameter 

605 for parameter, parameter_owner in owner_by_parameter.items() 

606 if parameter_owner is owner and parameter in replicate_params 

607 and parameter not in (ignored_params or ()) 

608 } 

609 

610 def _configure_source_layout_gradient_scaling( 

611 self, 

612 hsdp_module: HSDPModule, 

613 managed_source_shard_info: SourceShardInfoByParam | None, 

614 ) -> bool: 

615 """Restore global-mean gradients for one source-layout FSDP unit. 

616 

617 Master defaults every ``fully_shard`` unit to AVG reduction, so units 

618 with source-layout metadata explicitly select SUM. The token-weighted 

619 backward loss is scaled by the FSDP data domain (DP x CP), so a 

620 source-layout SUM-reduced unit must apply its inverse exactly once. 

621 With loss parallelism, TP ranks hold shards or partial contributions 

622 to one logical loss and TP is not another averaging axis. Without loss 

623 parallelism, logits and the loss are replicated over TP, so the SUM 

624 path also accumulates ``tp_size`` identical loss replicas and must 

625 divide by TP. EP is excluded because expert ranks own different 

626 parameters. 

627 

628 Returns: 

629 Whether a gradient scaling factor was configured for the unit. 

630 """ 

631 if not managed_source_shard_info: 

632 return False 

633 hsdp_module.set_reduce_op_type("sum", recurse=False) 

634 

635 tp_loss_replica_size = ( 

636 1 if self.mesh_context.loss_parallel else self.mesh_context.tp_size 

637 ) 

638 source_gradient_domain_size = ( 

639 self.mesh_context.dp_size 

640 * self.mesh_context.cp_size 

641 * tp_loss_replica_size 

642 ) 

643 if source_gradient_domain_size <= 1: 

644 return False 

645 hsdp_module.set_gradient_scaling_factor(1.0 / source_gradient_domain_size) 

646 return True 

647 

648 def _configure_prefetch(self, hsdp_modules: list[HSDPModule]) -> None: 

649 """Configure forward and backward prefetch targets by list depth.""" 

650 for module_index, hsdp_module in enumerate(hsdp_modules): 

651 forward_end = module_index + self.config.forward_prefetch_depth + 1 

652 hsdp_module.set_modules_to_forward_prefetch( 

653 hsdp_modules[module_index + 1:forward_end] 

654 ) 

655 

656 backward_start = max( 

657 0, module_index - self.config.backward_prefetch_depth 

658 ) 

659 hsdp_module.set_modules_to_backward_prefetch( 

660 list(reversed(hsdp_modules[backward_start:module_index])) 

661 ) 

662 

663 def _order_wrap_modules( 

664 self, 

665 model: ModuleClass, 

666 wrap_modules: list[_WrapModuleInfo], 

667 ) -> list[_WrapModuleInfo]: 

668 """Order FSDP units by adapter execution order or module traversal.""" 

669 module_order = { 

670 id(module): module_index 

671 for module_index, (_, module) in enumerate(model.named_modules()) 

672 } 

673 default_order = sorted( 

674 wrap_modules, 

675 key=lambda wrap_module: module_order.get(id(wrap_module.module), len(module_order)), 

676 ) 

677 adapter_spec = self._get_model_adapter_spec(model) 

678 execution_order_provider = getattr(adapter_spec, "fsdp_execution_order", None) 

679 if execution_order_provider is None: 

680 return default_order 

681 if not callable(execution_order_provider): 

682 raise ValueError("ModelAdapterSpec.fsdp_execution_order must be callable") 

683 

684 wrap_module_by_fqn = { 

685 wrap_module.fqn: wrap_module for wrap_module in wrap_modules 

686 } 

687 execution_fqns = execution_order_provider( # pylint: disable=not-callable 

688 model, 

689 tuple(wrap_module.fqn for wrap_module in default_order), 

690 ) 

691 if isinstance(execution_fqns, str) or not isinstance(execution_fqns, Sequence): 

692 raise ValueError( 

693 "ModelAdapterSpec.fsdp_execution_order must return a sequence of module FQNs" 

694 ) 

695 if any(not isinstance(module_fqn, str) for module_fqn in execution_fqns): 

696 raise ValueError( 

697 "ModelAdapterSpec.fsdp_execution_order entries must be module FQN strings" 

698 ) 

699 if ( 

700 len(execution_fqns) != len(wrap_module_by_fqn) 

701 or len(set(execution_fqns)) != len(execution_fqns) 

702 or set(execution_fqns) != set(wrap_module_by_fqn) 

703 ): 

704 raise ValueError( 

705 "ModelAdapterSpec.fsdp_execution_order must return every selected FSDP " 

706 "child-unit FQN exactly once" 

707 ) 

708 logger.info("Adapter-declared FSDP execution order: %s", list(execution_fqns)) 

709 return [wrap_module_by_fqn[module_fqn] for module_fqn in execution_fqns] 

710 

711 def _parallelize_child_units( 

712 self, 

713 wrap_modules: list[_WrapModuleInfo], 

714 owner_by_parameter, 

715 metadata_by_parameter, 

716 replicate_params, 

717 dense_fsdp_kwargs: dict[str, Any], 

718 ignored_params: set[ParameterClass] | None = None, 

719 ) -> int: 

720 """Apply FSDP to child units and return the gradient-scaled count.""" 

721 gradient_scaled_units = 0 

722 wrapping_order = sorted( 

723 wrap_modules, key=lambda wrap_module: wrap_module.fqn.count("."), reverse=True 

724 ) 

725 for wrap_module in wrapping_order: 

726 managed_source_info = _build_managed_source_shard_info( 

727 self, 

728 wrap_module.module, 

729 metadata_by_parameter, 

730 ignored_params, 

731 ) 

732 fsdp_sublayer_kwargs = dense_fsdp_kwargs 

733 if self._uses_expert_mesh(managed_source_info): 

734 if self.mesh_context.fsdp_moe_mesh is None: 

735 raise ValueError("Expert TP metadata requires MeshContext.fsdp_moe_mesh") 

736 fsdp_sublayer_kwargs, _ = self._build_fully_shard_kwargs( 

737 self._build_fsdp_actual_mesh(expert=True) 

738 ) 

739 fully_shard( # pylint: disable=unexpected-keyword-arg 

740 wrap_module.module, 

741 source_shard_infos=_source_infos_for_fully_shard(managed_source_info), 

742 replicate_params=self._build_managed_replicate_params( 

743 wrap_module.module, owner_by_parameter, replicate_params, ignored_params 

744 ), 

745 ignored_params=ignored_params, 

746 **fsdp_sublayer_kwargs, 

747 ) 

748 if self._configure_source_layout_gradient_scaling(wrap_module.module, managed_source_info): 

749 gradient_scaled_units += 1 

750 return gradient_scaled_units 

751 

752 def _parallelize_root( 

753 self, 

754 model: ModuleClass, 

755 owner_by_parameter, 

756 metadata_by_parameter, 

757 replicate_params, 

758 dense_root_kwargs: dict[str, Any], 

759 ignored_params: set[ParameterClass] | None = None, 

760 ) -> int: 

761 """Apply root FSDP and report whether gradient scaling was configured.""" 

762 root_source_info = _build_managed_source_shard_info( 

763 self, 

764 model, 

765 metadata_by_parameter, 

766 ignored_params, 

767 ) 

768 if self._uses_expert_mesh(root_source_info): 

769 raise ValueError("Routed expert parameters must belong to a nested experts FSDP unit") 

770 fully_shard( # pylint: disable=unexpected-keyword-arg 

771 model, 

772 source_shard_infos=_source_infos_for_fully_shard(root_source_info), 

773 replicate_params=self._build_managed_replicate_params( 

774 model, owner_by_parameter, replicate_params, ignored_params 

775 ), 

776 ignored_params=ignored_params, 

777 **dense_root_kwargs, 

778 ) 

779 return int(self._configure_source_layout_gradient_scaling(model, root_source_info)) 

780 

781 def _log_gradient_scaling(self, gradient_scaled_units: int) -> None: 

782 """Log the source-layout gradient scaling applied to FSDP units.""" 

783 if not gradient_scaled_units: 

784 return 

785 domain_size = ( 

786 self.mesh_context.dp_size 

787 * self.mesh_context.cp_size 

788 * (1 if self.mesh_context.loss_parallel else self.mesh_context.tp_size) 

789 ) 

790 logger.info( 

791 "Applied source-layout SUM gradient scaling: dp_size=%d, cp_size=%d, tp_size=%d, " 

792 "source_gradient_domain_size=%d, factor=%s, units=%d", 

793 self.mesh_context.dp_size, 

794 self.mesh_context.cp_size, 

795 self.mesh_context.tp_size, 

796 domain_size, 

797 1.0 / domain_size, 

798 gradient_scaled_units, 

799 ) 

800 

801 def parallelize( 

802 self, 

803 model: ModuleClass, 

804 source_shard_info: SourceShardInfoByFQN | None = None, 

805 ignored_params: set[ParameterClass] | None = None, 

806 ) -> ModuleClass: 

807 """Apply configured child FSDP units, root FSDP, and prefetch links. 

808 

809 Args: 

810 model: Model to wrap in place. 

811 source_shard_info: FQN-keyed source-layout metadata returned by 

812 ``apply_sharding_plan``. The manager resolves parameter 

813 identities before applying nested FSDP units. 

814 

815 Returns: 

816 The model enhanced with the HyperParallel HSDPModule interface. 

817 

818 Raises: 

819 ValueError: If TP is enabled without FQN metadata or configured 

820 parameter FQNs cannot be resolved. 

821 """ 

822 self._mp_policy = None 

823 metadata_by_parameter = _build_source_shard_info_by_param( 

824 self, 

825 model, 

826 source_shard_info, 

827 ) 

828 replicate_params = self._resolve_replicate_params(model) 

829 custom_params = None 

830 if self.config.mix_precision.custom_params: 

831 custom_params = self._resolve_custom_params(dict(model.named_parameters(remove_duplicate=False))) 

832 self._build_mixed_precision_policy(custom_params) 

833 

834 wrap_modules = self._find_wrap_modules(model, metadata_by_parameter) 

835 owner_by_parameter = self._resolve_parameter_owners(model, wrap_modules) 

836 dense_fsdp_kwargs, dense_root_kwargs = self._build_fully_shard_kwargs( 

837 self._build_fsdp_actual_mesh() 

838 ) 

839 

840 gradient_scaled_units = self._parallelize_child_units( 

841 wrap_modules, 

842 owner_by_parameter, 

843 metadata_by_parameter, 

844 replicate_params, 

845 dense_fsdp_kwargs, 

846 ignored_params, 

847 ) 

848 gradient_scaled_units += self._parallelize_root( 

849 model, 

850 owner_by_parameter, 

851 metadata_by_parameter, 

852 replicate_params, 

853 dense_root_kwargs, 

854 ignored_params, 

855 ) 

856 self._log_gradient_scaling(gradient_scaled_units) 

857 ordered_wrap_modules = self._order_wrap_modules(model, wrap_modules) 

858 self._configure_prefetch( 

859 [wrap_module.module for wrap_module in ordered_wrap_modules] 

860 ) 

861 logger.info( 

862 "Applied FSDP2 to %d child units and the root module", 

863 len(wrap_modules), 

864 ) 

865 return model 

866 

867 

868def _instantiate_fsdp2( 

869 *, 

870 config: Any, 

871 mesh_context: MeshContext, 

872 fp32_main_params: bool = False, 

873) -> FSDP2Manager | None: 

874 """Create an FSDP2 manager for the resolved strategy config.""" 

875 if config is None: 

876 return None 

877 if isinstance(config, FSDP2Config): 

878 return FSDP2Manager( 

879 config=config, 

880 mesh_context=mesh_context, 

881 fp32_main_params=fp32_main_params, 

882 ) 

883 raise ValueError(f"FSDP2Manager requires FSDP2Config, got {type(config).__name__}") 

884 

885 

886def _apply_fsdp2( 

887 model: ModuleClass, 

888 fsdp2_manager: Any, 

889 source_shard_info: Any, 

890 ignored_params: set[ParameterClass] | None = None, 

891) -> ModuleClass: 

892 """Apply FSDP2; Torch scheduler hooks remain outside Dynamo by design. 

893 

894 Moved from ``_transformers/infrastructure.py`` (05 §15.2.1) so the FSDP 

895 application step lives with ``FSDP2Manager.parallelize``. 

896 """ 

897 if fsdp2_manager is None: 

898 return model 

899 if not isinstance(fsdp2_manager, FSDP2Manager): 

900 logger.warning("fsdp2_manager is not an FSDP2Manager instance") 

901 return model 

902 model = fsdp2_manager.parallelize( 

903 model, 

904 source_shard_info=source_shard_info, 

905 ignored_params=ignored_params, 

906 ) 

907 logger.info("FSDP2 wrap applied") 

908 return model