Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / trainer / runtime / fsdp.py: 84%
215 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2025-2026 Bytedance Ltd. and/or its affiliates
2# Copyright 2026 Huawei Technologies Co., Ltd
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
16"""Trainer-side FSDP lifecycle policies and runtime diagnostics.
18Split out of the former ``auto_models/trainer/base.py`` in stage 7
19(05 §15.11 step 3). ``BaseTrainer`` keeps the same-named methods as thin
20delegating subclass hooks; the policy itself lives here. The FSDP config is
21duck-typed (``reshard_after_backward`` / ``dp_shard_size`` /
22``requires_grad_sync`` attributes) so this module does not import Trainer DTOs.
23"""
25import logging
26from collections import defaultdict
27from dataclasses import dataclass, field
28from typing import Any, List, Optional
30import torch
31from torch.utils._pytree import tree_flatten
34logger = logging.getLogger(__name__)
37@dataclass
38class _FSDPMicroStepTrace:
39 """Observed FSDP unit order for one forward/backward micro-step."""
41 global_step: int
42 micro_step: int
43 forward: list[str] = field(default_factory=list)
44 backward: list[str] = field(default_factory=list)
47class FSDPRuntimeDiagnostics:
48 """Validate Torch FSDP prefetch order at the optimizer-step boundary.
50 This observer does not launch, wait, release, or otherwise mutate FSDP
51 communication. It records the actual module execution order and inspects
52 outstanding all-gather state immediately before the optimizer may update
53 parameter shards.
54 """
56 def __init__(self, hsdp_model_parts: List[Any]) -> None:
57 """Attach diagnostic hooks to all HSDP units in the model parts."""
58 self._module_by_name: dict[str, Any] = {}
59 self._name_by_module_id: dict[int, str] = {}
60 self._scheduler_by_name: dict[str, Any] = {}
61 self._hook_handles: list[Any] = []
62 self._traces: list[_FSDPMicroStepTrace] = []
63 self._active_trace: Optional[_FSDPMicroStepTrace] = None
64 self._discover_units(hsdp_model_parts)
65 self._register_hooks()
67 @property
68 def unit_count(self) -> int:
69 """Return the number of observed HSDP modules."""
70 return len(self._module_by_name)
72 def _discover_units(self, hsdp_model_parts: List[Any]) -> None:
73 """Build stable names for HSDP modules without importing backend internals."""
74 multiple_parts = len(hsdp_model_parts) > 1
75 for part_index, model_part in enumerate(hsdp_model_parts):
76 part_prefix = f"part[{part_index}]" if multiple_parts else ""
77 for module_fqn, module in model_part.named_modules():
78 scheduler = getattr(module, "hsdp_scheduler", None)
79 if scheduler is None or id(module) in self._name_by_module_id:
80 continue
81 if part_prefix and module_fqn:
82 module_name = f"{part_prefix}.{module_fqn}"
83 elif part_prefix:
84 module_name = part_prefix
85 else:
86 module_name = module_fqn or "<root>"
87 self._module_by_name[module_name] = module
88 self._name_by_module_id[id(module)] = module_name
89 self._scheduler_by_name[module_name] = scheduler
91 def _register_hooks(self) -> None:
92 """Register hooks on every unit for order and child-bypass diagnostics."""
93 for module_name, module in self._module_by_name.items():
94 self._hook_handles.append(
95 module.register_forward_pre_hook(
96 self._make_forward_pre_hook(module_name)
97 )
98 )
99 self._hook_handles.append(
100 module.register_forward_hook(
101 self._make_backward_registration_hook(module_name)
102 )
103 )
105 def _make_forward_pre_hook(self, module_name: str):
106 """Create a hook recording non-recompute forward execution."""
108 def record_forward(module: Any, inputs: Any) -> None:
109 """Record one observed module forward entry."""
110 del inputs
111 if self._active_trace is None:
112 return
113 scheduler = getattr(module, "hsdp_scheduler", None)
114 scheduler_ctx = getattr(scheduler, "scheduler_ctx", None)
115 if getattr(scheduler_ctx, "root_bp_state", False):
116 return
117 self._active_trace.forward.append(module_name)
119 return record_forward
121 def _make_backward_registration_hook(self, module_name: str):
122 """Create a forward hook attaching a one-shot output-gradient observer."""
124 def register_backward_observer(module: Any, inputs: Any, outputs: Any) -> None:
125 """Attach a one-shot gradient observer to differentiable outputs."""
126 del inputs
127 scheduler = getattr(module, "hsdp_scheduler", None)
128 scheduler_ctx = getattr(scheduler, "scheduler_ctx", None)
129 if getattr(scheduler_ctx, "root_bp_state", False):
130 return
131 output_tensors = [
132 output
133 for output in tree_flatten(outputs)[0]
134 if torch.is_tensor(output) and output.requires_grad
135 ]
136 if not output_tensors:
137 return
139 fired = [False]
140 hook_handles = []
142 def record_backward(gradient: Any) -> Any:
143 """Record the module on its first output-gradient callback."""
144 if not fired[0]:
145 fired[0] = True
146 if self._active_trace is not None:
147 self._active_trace.backward.append(module_name)
148 for hook_handle in hook_handles:
149 hook_handle.remove()
150 return gradient
152 for output_tensor in output_tensors:
153 hook_handles.append(output_tensor.register_hook(record_backward))
155 return register_backward_observer
157 def begin_micro_step(self, global_step: int, micro_step: int) -> None:
158 """Start tracing one micro-step in the current optimizer step."""
159 if micro_step == 0:
160 self._traces.clear()
161 trace = _FSDPMicroStepTrace(global_step, micro_step)
162 self._traces.append(trace)
163 self._active_trace = trace
165 @staticmethod
166 def _first_positions(execution_order: list[str]) -> dict[str, int]:
167 """Return first-observed positions while preserving repeated calls in logs."""
168 positions = {}
169 for position, module_name in enumerate(execution_order):
170 positions.setdefault(module_name, position)
171 return positions
173 def _prefetch_target_names(self, module_name: str, phase: str) -> list[str]:
174 """Resolve configured prefetch targets to diagnostic module names."""
175 scheduler = self._scheduler_by_name[module_name]
176 attribute = (
177 "forward_prefetch_cells"
178 if phase == "forward"
179 else "backward_prefetch_cells"
180 )
181 target_names = []
182 for target in getattr(scheduler, attribute, ()):
183 target_names.append(
184 self._name_by_module_id.get(
185 id(target),
186 f"<untracked:{type(target).__name__}>",
187 )
188 )
189 return target_names
191 def _find_order_violations(self) -> list[str]:
192 """Compare configured prefetch edges with each observed execution trace."""
193 violations = []
194 for trace in self._traces:
195 for phase in ("forward", "backward"):
196 execution_order = getattr(trace, phase)
197 positions = self._first_positions(execution_order)
198 for source_name, source_position in positions.items():
199 for target_name in self._prefetch_target_names(source_name, phase):
200 target_position = positions.get(target_name)
201 if target_position is None:
202 violations.append(
203 f"step={trace.global_step} micro_step={trace.micro_step} "
204 f"phase={phase}: {source_name} prefetched {target_name}, "
205 "but the target did not execute"
206 )
207 elif target_position <= source_position:
208 violations.append(
209 f"step={trace.global_step} micro_step={trace.micro_step} "
210 f"phase={phase}: {source_name} prefetched {target_name} "
211 f"at positions {source_position}->{target_position}"
212 )
213 return violations
215 @staticmethod
216 def _parameter_name(hsdp_param: Any, param_index: int) -> str:
217 """Return a useful parameter name for diagnostics."""
218 return getattr(hsdp_param, "_param_fqn", None) or f"param[{param_index}]"
220 def _find_pending_all_gathers(self) -> list[str]:
221 """Find unconsumed fused and non-fused all-gathers without mutating them."""
222 pending = []
223 seen_schedulers = set()
224 for module_name, scheduler in self._scheduler_by_name.items():
225 if id(scheduler) in seen_schedulers:
226 continue
227 seen_schedulers.add(id(scheduler))
228 state = getattr(scheduler, "hsdp_state", None)
229 if state is None:
230 continue
232 param_group = getattr(state, "param_group", None)
233 for bucket_index, bucket in enumerate(
234 getattr(param_group, "all_gather_buckets", ())
235 ):
236 result = getattr(bucket, "all_gather_result", None)
237 if result is not None:
238 handle = getattr(result, "handle", None)
239 pending.append(
240 f"{module_name}: fused_bucket[{bucket_index}] "
241 f"handle={'set' if handle is not None else 'completed-not-consumed'}"
242 )
244 for param_index, hsdp_param in enumerate(getattr(state, "hsdp_params", ())):
245 comm_ctx = getattr(hsdp_param, "allgather_comm_ctx", None)
246 handle = getattr(comm_ctx, "allgather_handle", None)
247 output = getattr(comm_ctx, "allgather_output", None)
248 if handle is not None or output is not None:
249 state_details = []
250 if handle is not None:
251 state_details.append("handle=set")
252 if output is not None:
253 state_details.append("output=not-consumed")
254 pending.append(
255 f"{module_name}: "
256 f"{self._parameter_name(hsdp_param, param_index)} "
257 + ", ".join(state_details)
258 )
259 return pending
261 @staticmethod
262 def _hsdp_parameter_has_gradient(hsdp_param: Any) -> bool:
263 """Return whether a managed parameter participated in backward."""
264 sharded_param = getattr(hsdp_param, "sharded_param", None)
265 candidates = (
266 getattr(sharded_param, "grad", None),
267 getattr(sharded_param, "main_grad", None),
268 getattr(hsdp_param, "_grad", None),
269 getattr(hsdp_param, "unsharded_accumulated_grad", None),
270 getattr(hsdp_param, "unsharded_accumulated_grad_data", None),
271 )
272 return any(value is not None for value in candidates)
274 def _find_bypassed_child_units(self) -> list[str]:
275 """Find child FSDP parameters used without executing their owner module."""
276 if not self._traces:
277 return []
278 forward_counts = defaultdict(int)
279 for trace in self._traces:
280 for module_name in trace.forward:
281 forward_counts[module_name] += 1
282 bypassed = []
283 root_names = {"<root>"}
284 root_names.update(
285 name for name in self._module_by_name
286 if name.startswith("part[") and "." not in name
287 )
288 for module_name, scheduler in self._scheduler_by_name.items():
289 if module_name in root_names or forward_counts[module_name] > 0:
290 continue
291 state = getattr(scheduler, "hsdp_state", None)
292 gradient_parameters = [
293 self._parameter_name(hsdp_param, param_index)
294 for param_index, hsdp_param in enumerate(getattr(state, "hsdp_params", ()))
295 if self._hsdp_parameter_has_gradient(hsdp_param)
296 ]
297 if gradient_parameters:
298 bypassed.append(
299 f"{module_name}: forward_count=0, gradient_parameters={gradient_parameters}"
300 )
301 return bypassed
303 def validate_before_optimizer_step(self) -> None:
304 """Fail before parameter updates if the traced FSDP lifecycle is unsafe."""
305 self._active_trace = None
306 order_violations = self._find_order_violations()
307 pending_all_gathers = self._find_pending_all_gathers()
308 bypassed_child_units = self._find_bypassed_child_units()
309 error_sections = []
310 if bypassed_child_units:
311 error_sections.append(
312 "child FSDP module was bypassed while its parameter "
313 "participated in backward:\n - " + "\n - ".join(bypassed_child_units)
314 )
315 if order_violations:
316 error_sections.append(
317 "configured prefetch targets do not follow "
318 "the observed runtime order:\n - " + "\n - ".join(order_violations)
319 )
320 if pending_all_gathers:
321 error_sections.append(
322 "unconsumed parameter all-gather exists at "
323 "the optimizer boundary:\n - " + "\n - ".join(pending_all_gathers)
324 )
325 if error_sections:
326 raise RuntimeError(
327 "\n".join(error_sections)
328 + "\nThe optimizer was not executed. Configure FSDP prefetch from the "
329 "model's actual forward/backward execution order; do not carry an "
330 "all-gather across optimizer.step()."
331 )
333 def trace_evidence(self) -> dict[str, Any]:
334 """Return stable execution-order evidence for structured reports."""
335 return {
336 "units": sorted(self._module_by_name),
337 "micro_steps": [
338 {
339 "global_step": trace.global_step,
340 "micro_step": trace.micro_step,
341 "forward": list(trace.forward),
342 "backward": list(trace.backward),
343 }
344 for trace in self._traces
345 ],
346 }
348 def close(self) -> None:
349 """Remove diagnostic hooks when a caller no longer needs the observer."""
350 for hook_handle in self._hook_handles:
351 hook_handle.remove()
352 self._hook_handles.clear()
355def build_fsdp_runtime_diagnostics(
356 hsdp_model_parts: List[Any],
357) -> Optional[FSDPRuntimeDiagnostics]:
358 """Create diagnostics for Torch Trainer model parts, or return ``None``."""
359 if not hsdp_model_parts:
360 return None
361 diagnostics = FSDPRuntimeDiagnostics(hsdp_model_parts)
362 if diagnostics.unit_count == 0:
363 diagnostics.close()
364 return None
365 logger.info(
366 "Enabled Trainer FSDP runtime diagnostics for %d modules",
367 diagnostics.unit_count,
368 )
369 return diagnostics
372def model_reshard(
373 hsdp_model_parts: List[Any],
374 fsdp_config: Any,
375 micro_step: int,
376 num_micro_steps: int,
377) -> None:
378 """Reshard model after backward pass."""
379 if (
380 fsdp_config.reshard_after_backward is False
381 and num_micro_steps > 1
382 ):
383 if micro_step == 0:
384 for model_part in hsdp_model_parts:
385 model_part.set_reshard_after_backward(False)
386 elif micro_step == num_micro_steps - 1:
387 for model_part in hsdp_model_parts:
388 model_part.set_reshard_after_backward(True)
391def configure_fsdp_gradient_sync(
392 hsdp_model_parts: List[Any],
393 fsdp_config: Any,
394 dp_replicate_size: int,
395 micro_step: int,
396 num_micro_steps: int,
397) -> None:
398 """Configure FSDP gradient synchronization for one micro step."""
399 if (
400 fsdp_config.dp_shard_size > 1
401 and num_micro_steps > 1
402 ):
403 is_last_micro_batch = micro_step == num_micro_steps - 1
404 requires_gradient_sync = (
405 fsdp_config.requires_grad_sync
406 or is_last_micro_batch
407 )
408 is_hsdp = dp_replicate_size > 1
409 for model_part in hsdp_model_parts:
410 model_part.set_requires_gradient_sync(requires_gradient_sync)
411 model_part.set_is_last_backward(is_last_micro_batch)
412 if is_hsdp:
413 model_part.set_requires_all_reduce(is_last_micro_batch)
416__all__ = [
417 "FSDPRuntimeDiagnostics",
418 "build_fsdp_runtime_diagnostics",
419 "configure_fsdp_gradient_sync",
420 "model_reshard",
421]