Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / trainer / runtime / fsdp.py: 84%

215 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2025-2026 Bytedance Ltd. and/or its affiliates 

2# Copyright 2026 Huawei Technologies Co., Ltd 

3# 

4# Licensed under the Apache License, Version 2.0 (the "License"); 

5# you may not use this file except in compliance with the License. 

6# You may obtain a copy of the License at 

7# 

8# http://www.apache.org/licenses/LICENSE-2.0 

9# 

10# Unless required by applicable law or agreed to in writing, software 

11# distributed under the License is distributed on an "AS IS" BASIS, 

12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

13# See the License for the specific language governing permissions and 

14# limitations under the License. 

15 

16"""Trainer-side FSDP lifecycle policies and runtime diagnostics. 

17 

18Split out of the former ``auto_models/trainer/base.py`` in stage 7 

19(05 §15.11 step 3). ``BaseTrainer`` keeps the same-named methods as thin 

20delegating subclass hooks; the policy itself lives here. The FSDP config is 

21duck-typed (``reshard_after_backward`` / ``dp_shard_size`` / 

22``requires_grad_sync`` attributes) so this module does not import Trainer DTOs. 

23""" 

24 

25import logging 

26from collections import defaultdict 

27from dataclasses import dataclass, field 

28from typing import Any, List, Optional 

29 

30import torch 

31from torch.utils._pytree import tree_flatten 

32 

33 

34logger = logging.getLogger(__name__) 

35 

36 

37@dataclass 

38class _FSDPMicroStepTrace: 

39 """Observed FSDP unit order for one forward/backward micro-step.""" 

40 

41 global_step: int 

42 micro_step: int 

43 forward: list[str] = field(default_factory=list) 

44 backward: list[str] = field(default_factory=list) 

45 

46 

47class FSDPRuntimeDiagnostics: 

48 """Validate Torch FSDP prefetch order at the optimizer-step boundary. 

49 

50 This observer does not launch, wait, release, or otherwise mutate FSDP 

51 communication. It records the actual module execution order and inspects 

52 outstanding all-gather state immediately before the optimizer may update 

53 parameter shards. 

54 """ 

55 

56 def __init__(self, hsdp_model_parts: List[Any]) -> None: 

57 """Attach diagnostic hooks to all HSDP units in the model parts.""" 

58 self._module_by_name: dict[str, Any] = {} 

59 self._name_by_module_id: dict[int, str] = {} 

60 self._scheduler_by_name: dict[str, Any] = {} 

61 self._hook_handles: list[Any] = [] 

62 self._traces: list[_FSDPMicroStepTrace] = [] 

63 self._active_trace: Optional[_FSDPMicroStepTrace] = None 

64 self._discover_units(hsdp_model_parts) 

65 self._register_hooks() 

66 

67 @property 

68 def unit_count(self) -> int: 

69 """Return the number of observed HSDP modules.""" 

70 return len(self._module_by_name) 

71 

72 def _discover_units(self, hsdp_model_parts: List[Any]) -> None: 

73 """Build stable names for HSDP modules without importing backend internals.""" 

74 multiple_parts = len(hsdp_model_parts) > 1 

75 for part_index, model_part in enumerate(hsdp_model_parts): 

76 part_prefix = f"part[{part_index}]" if multiple_parts else "" 

77 for module_fqn, module in model_part.named_modules(): 

78 scheduler = getattr(module, "hsdp_scheduler", None) 

79 if scheduler is None or id(module) in self._name_by_module_id: 

80 continue 

81 if part_prefix and module_fqn: 

82 module_name = f"{part_prefix}.{module_fqn}" 

83 elif part_prefix: 

84 module_name = part_prefix 

85 else: 

86 module_name = module_fqn or "<root>" 

87 self._module_by_name[module_name] = module 

88 self._name_by_module_id[id(module)] = module_name 

89 self._scheduler_by_name[module_name] = scheduler 

90 

91 def _register_hooks(self) -> None: 

92 """Register hooks on every unit for order and child-bypass diagnostics.""" 

93 for module_name, module in self._module_by_name.items(): 

94 self._hook_handles.append( 

95 module.register_forward_pre_hook( 

96 self._make_forward_pre_hook(module_name) 

97 ) 

98 ) 

99 self._hook_handles.append( 

100 module.register_forward_hook( 

101 self._make_backward_registration_hook(module_name) 

102 ) 

103 ) 

104 

105 def _make_forward_pre_hook(self, module_name: str): 

106 """Create a hook recording non-recompute forward execution.""" 

107 

108 def record_forward(module: Any, inputs: Any) -> None: 

109 """Record one observed module forward entry.""" 

110 del inputs 

111 if self._active_trace is None: 

112 return 

113 scheduler = getattr(module, "hsdp_scheduler", None) 

114 scheduler_ctx = getattr(scheduler, "scheduler_ctx", None) 

115 if getattr(scheduler_ctx, "root_bp_state", False): 

116 return 

117 self._active_trace.forward.append(module_name) 

118 

119 return record_forward 

120 

121 def _make_backward_registration_hook(self, module_name: str): 

122 """Create a forward hook attaching a one-shot output-gradient observer.""" 

123 

124 def register_backward_observer(module: Any, inputs: Any, outputs: Any) -> None: 

125 """Attach a one-shot gradient observer to differentiable outputs.""" 

126 del inputs 

127 scheduler = getattr(module, "hsdp_scheduler", None) 

128 scheduler_ctx = getattr(scheduler, "scheduler_ctx", None) 

129 if getattr(scheduler_ctx, "root_bp_state", False): 

130 return 

131 output_tensors = [ 

132 output 

133 for output in tree_flatten(outputs)[0] 

134 if torch.is_tensor(output) and output.requires_grad 

135 ] 

136 if not output_tensors: 

137 return 

138 

139 fired = [False] 

140 hook_handles = [] 

141 

142 def record_backward(gradient: Any) -> Any: 

143 """Record the module on its first output-gradient callback.""" 

144 if not fired[0]: 

145 fired[0] = True 

146 if self._active_trace is not None: 

147 self._active_trace.backward.append(module_name) 

148 for hook_handle in hook_handles: 

149 hook_handle.remove() 

150 return gradient 

151 

152 for output_tensor in output_tensors: 

153 hook_handles.append(output_tensor.register_hook(record_backward)) 

154 

155 return register_backward_observer 

156 

157 def begin_micro_step(self, global_step: int, micro_step: int) -> None: 

158 """Start tracing one micro-step in the current optimizer step.""" 

159 if micro_step == 0: 

160 self._traces.clear() 

161 trace = _FSDPMicroStepTrace(global_step, micro_step) 

162 self._traces.append(trace) 

163 self._active_trace = trace 

164 

165 @staticmethod 

166 def _first_positions(execution_order: list[str]) -> dict[str, int]: 

167 """Return first-observed positions while preserving repeated calls in logs.""" 

168 positions = {} 

169 for position, module_name in enumerate(execution_order): 

170 positions.setdefault(module_name, position) 

171 return positions 

172 

173 def _prefetch_target_names(self, module_name: str, phase: str) -> list[str]: 

174 """Resolve configured prefetch targets to diagnostic module names.""" 

175 scheduler = self._scheduler_by_name[module_name] 

176 attribute = ( 

177 "forward_prefetch_cells" 

178 if phase == "forward" 

179 else "backward_prefetch_cells" 

180 ) 

181 target_names = [] 

182 for target in getattr(scheduler, attribute, ()): 

183 target_names.append( 

184 self._name_by_module_id.get( 

185 id(target), 

186 f"<untracked:{type(target).__name__}>", 

187 ) 

188 ) 

189 return target_names 

190 

191 def _find_order_violations(self) -> list[str]: 

192 """Compare configured prefetch edges with each observed execution trace.""" 

193 violations = [] 

194 for trace in self._traces: 

195 for phase in ("forward", "backward"): 

196 execution_order = getattr(trace, phase) 

197 positions = self._first_positions(execution_order) 

198 for source_name, source_position in positions.items(): 

199 for target_name in self._prefetch_target_names(source_name, phase): 

200 target_position = positions.get(target_name) 

201 if target_position is None: 

202 violations.append( 

203 f"step={trace.global_step} micro_step={trace.micro_step} " 

204 f"phase={phase}: {source_name} prefetched {target_name}, " 

205 "but the target did not execute" 

206 ) 

207 elif target_position <= source_position: 

208 violations.append( 

209 f"step={trace.global_step} micro_step={trace.micro_step} " 

210 f"phase={phase}: {source_name} prefetched {target_name} " 

211 f"at positions {source_position}->{target_position}" 

212 ) 

213 return violations 

214 

215 @staticmethod 

216 def _parameter_name(hsdp_param: Any, param_index: int) -> str: 

217 """Return a useful parameter name for diagnostics.""" 

218 return getattr(hsdp_param, "_param_fqn", None) or f"param[{param_index}]" 

219 

220 def _find_pending_all_gathers(self) -> list[str]: 

221 """Find unconsumed fused and non-fused all-gathers without mutating them.""" 

222 pending = [] 

223 seen_schedulers = set() 

224 for module_name, scheduler in self._scheduler_by_name.items(): 

225 if id(scheduler) in seen_schedulers: 

226 continue 

227 seen_schedulers.add(id(scheduler)) 

228 state = getattr(scheduler, "hsdp_state", None) 

229 if state is None: 

230 continue 

231 

232 param_group = getattr(state, "param_group", None) 

233 for bucket_index, bucket in enumerate( 

234 getattr(param_group, "all_gather_buckets", ()) 

235 ): 

236 result = getattr(bucket, "all_gather_result", None) 

237 if result is not None: 

238 handle = getattr(result, "handle", None) 

239 pending.append( 

240 f"{module_name}: fused_bucket[{bucket_index}] " 

241 f"handle={'set' if handle is not None else 'completed-not-consumed'}" 

242 ) 

243 

244 for param_index, hsdp_param in enumerate(getattr(state, "hsdp_params", ())): 

245 comm_ctx = getattr(hsdp_param, "allgather_comm_ctx", None) 

246 handle = getattr(comm_ctx, "allgather_handle", None) 

247 output = getattr(comm_ctx, "allgather_output", None) 

248 if handle is not None or output is not None: 

249 state_details = [] 

250 if handle is not None: 

251 state_details.append("handle=set") 

252 if output is not None: 

253 state_details.append("output=not-consumed") 

254 pending.append( 

255 f"{module_name}: " 

256 f"{self._parameter_name(hsdp_param, param_index)} " 

257 + ", ".join(state_details) 

258 ) 

259 return pending 

260 

261 @staticmethod 

262 def _hsdp_parameter_has_gradient(hsdp_param: Any) -> bool: 

263 """Return whether a managed parameter participated in backward.""" 

264 sharded_param = getattr(hsdp_param, "sharded_param", None) 

265 candidates = ( 

266 getattr(sharded_param, "grad", None), 

267 getattr(sharded_param, "main_grad", None), 

268 getattr(hsdp_param, "_grad", None), 

269 getattr(hsdp_param, "unsharded_accumulated_grad", None), 

270 getattr(hsdp_param, "unsharded_accumulated_grad_data", None), 

271 ) 

272 return any(value is not None for value in candidates) 

273 

274 def _find_bypassed_child_units(self) -> list[str]: 

275 """Find child FSDP parameters used without executing their owner module.""" 

276 if not self._traces: 

277 return [] 

278 forward_counts = defaultdict(int) 

279 for trace in self._traces: 

280 for module_name in trace.forward: 

281 forward_counts[module_name] += 1 

282 bypassed = [] 

283 root_names = {"<root>"} 

284 root_names.update( 

285 name for name in self._module_by_name 

286 if name.startswith("part[") and "." not in name 

287 ) 

288 for module_name, scheduler in self._scheduler_by_name.items(): 

289 if module_name in root_names or forward_counts[module_name] > 0: 

290 continue 

291 state = getattr(scheduler, "hsdp_state", None) 

292 gradient_parameters = [ 

293 self._parameter_name(hsdp_param, param_index) 

294 for param_index, hsdp_param in enumerate(getattr(state, "hsdp_params", ())) 

295 if self._hsdp_parameter_has_gradient(hsdp_param) 

296 ] 

297 if gradient_parameters: 

298 bypassed.append( 

299 f"{module_name}: forward_count=0, gradient_parameters={gradient_parameters}" 

300 ) 

301 return bypassed 

302 

303 def validate_before_optimizer_step(self) -> None: 

304 """Fail before parameter updates if the traced FSDP lifecycle is unsafe.""" 

305 self._active_trace = None 

306 order_violations = self._find_order_violations() 

307 pending_all_gathers = self._find_pending_all_gathers() 

308 bypassed_child_units = self._find_bypassed_child_units() 

309 error_sections = [] 

310 if bypassed_child_units: 

311 error_sections.append( 

312 "child FSDP module was bypassed while its parameter " 

313 "participated in backward:\n - " + "\n - ".join(bypassed_child_units) 

314 ) 

315 if order_violations: 

316 error_sections.append( 

317 "configured prefetch targets do not follow " 

318 "the observed runtime order:\n - " + "\n - ".join(order_violations) 

319 ) 

320 if pending_all_gathers: 

321 error_sections.append( 

322 "unconsumed parameter all-gather exists at " 

323 "the optimizer boundary:\n - " + "\n - ".join(pending_all_gathers) 

324 ) 

325 if error_sections: 

326 raise RuntimeError( 

327 "\n".join(error_sections) 

328 + "\nThe optimizer was not executed. Configure FSDP prefetch from the " 

329 "model's actual forward/backward execution order; do not carry an " 

330 "all-gather across optimizer.step()." 

331 ) 

332 

333 def trace_evidence(self) -> dict[str, Any]: 

334 """Return stable execution-order evidence for structured reports.""" 

335 return { 

336 "units": sorted(self._module_by_name), 

337 "micro_steps": [ 

338 { 

339 "global_step": trace.global_step, 

340 "micro_step": trace.micro_step, 

341 "forward": list(trace.forward), 

342 "backward": list(trace.backward), 

343 } 

344 for trace in self._traces 

345 ], 

346 } 

347 

348 def close(self) -> None: 

349 """Remove diagnostic hooks when a caller no longer needs the observer.""" 

350 for hook_handle in self._hook_handles: 

351 hook_handle.remove() 

352 self._hook_handles.clear() 

353 

354 

355def build_fsdp_runtime_diagnostics( 

356 hsdp_model_parts: List[Any], 

357) -> Optional[FSDPRuntimeDiagnostics]: 

358 """Create diagnostics for Torch Trainer model parts, or return ``None``.""" 

359 if not hsdp_model_parts: 

360 return None 

361 diagnostics = FSDPRuntimeDiagnostics(hsdp_model_parts) 

362 if diagnostics.unit_count == 0: 

363 diagnostics.close() 

364 return None 

365 logger.info( 

366 "Enabled Trainer FSDP runtime diagnostics for %d modules", 

367 diagnostics.unit_count, 

368 ) 

369 return diagnostics 

370 

371 

372def model_reshard( 

373 hsdp_model_parts: List[Any], 

374 fsdp_config: Any, 

375 micro_step: int, 

376 num_micro_steps: int, 

377) -> None: 

378 """Reshard model after backward pass.""" 

379 if ( 

380 fsdp_config.reshard_after_backward is False 

381 and num_micro_steps > 1 

382 ): 

383 if micro_step == 0: 

384 for model_part in hsdp_model_parts: 

385 model_part.set_reshard_after_backward(False) 

386 elif micro_step == num_micro_steps - 1: 

387 for model_part in hsdp_model_parts: 

388 model_part.set_reshard_after_backward(True) 

389 

390 

391def configure_fsdp_gradient_sync( 

392 hsdp_model_parts: List[Any], 

393 fsdp_config: Any, 

394 dp_replicate_size: int, 

395 micro_step: int, 

396 num_micro_steps: int, 

397) -> None: 

398 """Configure FSDP gradient synchronization for one micro step.""" 

399 if ( 

400 fsdp_config.dp_shard_size > 1 

401 and num_micro_steps > 1 

402 ): 

403 is_last_micro_batch = micro_step == num_micro_steps - 1 

404 requires_gradient_sync = ( 

405 fsdp_config.requires_grad_sync 

406 or is_last_micro_batch 

407 ) 

408 is_hsdp = dp_replicate_size > 1 

409 for model_part in hsdp_model_parts: 

410 model_part.set_requires_gradient_sync(requires_gradient_sync) 

411 model_part.set_is_last_backward(is_last_micro_batch) 

412 if is_hsdp: 

413 model_part.set_requires_all_reduce(is_last_micro_batch) 

414 

415 

416__all__ = [ 

417 "FSDPRuntimeDiagnostics", 

418 "build_fsdp_runtime_diagnostics", 

419 "configure_fsdp_gradient_sync", 

420 "model_reshard", 

421]