Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / fully_shard / hsdp_scheduler.py: 78%
426 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""HSDP scheduler"""
16import functools
17import inspect
18from collections import deque
19from typing import Any, Callable, List, Mapping, Optional, ParamSpec, Tuple, TypeVar, Union
21import torch
22from torch import nn
23from torch.autograd import Variable
24from torch.utils._pytree import tree_flatten, tree_unflatten
26from hyper_parallel.core.dtensor.device_mesh import DeviceMesh
27from hyper_parallel.core.fully_shard.hook_function import PostBackwardFunction
28from hyper_parallel.core.fully_shard.hsdp_utils import (
29 FSDPSchedulerState,
30 get_managed_modules_parameters,
31)
32from hyper_parallel.core.fully_shard.utils import (
33 CommFusionPolicy,
34 SourceShardMetaInfo,
35 apply_to_tensors,
36 cast_fp_tensor,
37 parameters_dict,
38 profiler_record,
39)
40from hyper_parallel.tools.logging import get_logger
42logger = get_logger("FSDP")
44nn.Module = nn.Module
46_P = ParamSpec("_P")
47_R = TypeVar("_R")
50def _dynamo_disable(func: Callable[_P, _R]) -> Callable[_P, _R]:
51 """Disable Dynamo tracing while an FSDP runtime hook executes."""
53 @functools.wraps(func)
54 def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
55 return torch._dynamo.disable(
56 func,
57 recursive=True,
58 )(*args, **kwargs)
60 return wrapper
63class ParamGroupCommCtx:
64 """Track the in-flight parameter-group communication for one module tree."""
66 def __init__(self) -> None:
67 self.pre_param_group = None
68 self.all_reduce_param_group = None
69 self.comm_handle = None
72class HSDPSchedulerContext:
73 """Share scheduler and backward-pipeline state within one HSDP module tree."""
75 def __init__(self) -> None:
76 self.is_last_backward: bool = True
77 self.root_module = None
78 # Compile tracing may enter the root more than once; initialize shared
79 # parameter state only on the first real forward.
80 self.lazy_init_done: bool = False
81 self.root_bp_state = False
82 self.post_backward_final_callback_queued = False
83 # all_hsdp_schedulers (for one module tree structure), deduplicated at
84 # registration time because ``fully_shard`` may be given a module list
85 # whose modules share one scheduler.
86 self.all_hsdp_schedulers = []
87 # Parameter FQNs are initialized once after all schedulers share this context.
88 self._param_fqn_initialized = False
89 # Each queue entry belongs to one non-fused unit, including empty units.
90 # Configuration and queues are shared by the entire module tree.
91 self.reduce_interval: int = 1
92 self.pre_reduce_scatter_params = deque()
93 self.pre_all_reduce_params = []
94 self.pre_direct_all_reduce_grads = []
95 self.pre_all_reduce_groups = deque()
96 self.pending_all_reduce_groups = []
97 self.param_group_comm_ctx = ParamGroupCommCtx()
100class HSDPSchedulerV2:
101 """HSDPScheduler is used to scheduler hsdp"""
103 def __init__(
104 self,
105 cell: Union[nn.Module, Tuple[nn.Module, ...]],
106 mesh,
107 reshard_after_forward,
108 shard_placement_fn,
109 mp_policy,
110 offload_policy,
111 ignored_params,
112 replicate_params,
113 device,
114 comm_fusion,
115 comm_fusion_zero_copy=False,
116 source_shard_infos: Optional[Mapping[nn.Parameter, SourceShardMetaInfo]] = None,
117 ):
118 """init hsdp scheduler.
120 Args:
121 cell: A single module or tuple of modules to manage as one FSDP unit.
122 mesh: Explicit data-parallel device mesh.
123 reshard_after_forward: Whether to reshard parameters after forward.
124 shard_placement_fn: Optional function selecting the parameter shard dimension.
125 mp_policy: Mixed-precision policy.
126 offload_policy: Parameter offload policy.
127 ignored_params: Parameters excluded from fully_shard management.
128 replicate_params: Managed parameters that remain replicated.
129 device: Target device for parameters.
130 comm_fusion: Communication fusion configuration.
131 comm_fusion_zero_copy: Whether fused communication writes into views.
132 source_shard_infos: Optional per-parameter TP/EP source metadata.
133 """
134 self.modules = (cell,) if isinstance(cell, nn.Module) else tuple(cell)
135 self.cell = self.modules[0]
136 self.mesh: DeviceMesh = mesh
137 self.shard_placement_fn = shard_placement_fn
138 self.mp_policy = mp_policy
139 self.offload_policy = offload_policy
140 self.comm_fusion_policy = CommFusionPolicy(comm_fusion, comm_fusion_zero_copy)
141 self.ignored_params = ignored_params
142 self.replicate_params = replicate_params
143 self.device = device
144 self.reshard_after_forward = reshard_after_forward
145 self.source_shard_infos = source_shard_infos
146 self.scheduler_state = None
147 self.forward_prefetch_cells = []
148 self.backward_prefetch_cells = []
149 self._backup_forward_fetch = None
150 # Flag to identify root module.
151 self._is_root = True
152 # module and its all sub-modules share one same 'HSDPSchedulerContext'
153 self.scheduler_ctx = HSDPSchedulerContext()
154 # When ``fully_shard`` is given multiple root modules, forward pre/post hooks coordinate
155 # so unshard / PostBackward / reshard run once per forward (aligned with PyTorch FSDP2).
156 self._fsdp_group_post_pending: Optional[set] = set() if len(self.modules) > 1 else None
157 self._new_cell_state()
158 self._register_hooks()
160 def _new_cell_state(self):
161 """Create the HSDP state that owns this scheduler's managed parameters."""
162 # Imported lazily: ``state`` pulls in ``param_group``, which imports this
163 # module for ``ParamGroupCommCtx``.
164 from hyper_parallel.core.fully_shard.hsdp_state import ( # pylint: disable=C0415
165 HSDPStateV2,
166 )
168 self.hsdp_state = HSDPStateV2(
169 self.modules,
170 self.mesh,
171 self.shard_placement_fn,
172 self.comm_fusion_policy,
173 self.mp_policy,
174 self.offload_policy,
175 self.ignored_params,
176 self.replicate_params,
177 self.scheduler_ctx,
178 self.device,
179 source_shard_infos=self.source_shard_infos,
180 )
182 def _register_hooks(self):
183 """Register the forward and backward hooks driving shard/unshard."""
184 self._register_forward_backward_hooks()
186 def _get_managed_params(self):
187 """Return deduplicated parameters from all managed modules."""
188 return get_managed_modules_parameters(self.modules, self.ignored_params)
190 def set_reshard_after_forward(self, reshard_after_forward: bool) -> None:
191 """Set reshard_after_forward flag.
193 Args:
194 reshard_after_forward: Whether to reshard parameters after forward.
195 """
196 if not isinstance(reshard_after_forward, bool):
197 raise ValueError(f"reshard_after_forward should be a bool, got {type(reshard_after_forward)}")
198 self.reshard_after_forward = reshard_after_forward
200 def set_reshard_after_backward(self, reshard_after_backward: bool) -> None:
201 """Set reshard_after_backward flag.
203 Args:
204 reshard_after_backward: Whether to reshard after backward completes.
205 """
206 if not isinstance(reshard_after_backward, bool):
207 raise ValueError(f"reshard_after_backward should be a bool, got {type(reshard_after_backward)}")
208 if self.hsdp_state is not None:
209 self.hsdp_state.reshard_after_backward = reshard_after_backward
211 def set_requires_all_reduce(self, requires_all_reduce: bool) -> None:
212 """Set requires_all_reduce flag.
214 Args:
215 requires_all_reduce: Whether this unit participates in all-reduce.
216 """
217 if not isinstance(requires_all_reduce, bool):
218 raise ValueError(f"requires_all_reduce should be a bool, got {type(requires_all_reduce)}")
219 if self.hsdp_state is not None:
220 self.hsdp_state.set_requires_all_reduce(requires_all_reduce)
222 @_dynamo_disable
223 def reset_iter_state(self) -> None:
224 """Reset scheduler bookkeeping after a completed iteration."""
225 self.scheduler_ctx.root_bp_state = False
226 self.scheduler_ctx.post_backward_final_callback_queued = False
227 self.scheduler_ctx.pre_reduce_scatter_params.clear()
228 self.scheduler_ctx.pre_all_reduce_params.clear()
229 self.scheduler_ctx.pre_direct_all_reduce_grads.clear()
230 self.scheduler_ctx.pre_all_reduce_groups.clear()
231 self.scheduler_ctx.pending_all_reduce_groups.clear()
232 self.scheduler_ctx.param_group_comm_ctx.pre_param_group = None
233 self.scheduler_ctx.param_group_comm_ctx.all_reduce_param_group = None
234 self.scheduler_ctx.param_group_comm_ctx.comm_handle = None
235 self.scheduler_state = None
236 if self._fsdp_group_post_pending is not None:
237 self._fsdp_group_post_pending.clear()
238 self._restore_forward_prefetch_after_recompute()
239 self.hsdp_state.reset_iter_state()
241 def set_requires_grad_sync(self, requires_grad_sync: bool) -> None:
242 """Set flag controlling whether gradients are synchronized.
244 Args:
245 requires_grad_sync: When True, enable grad sync for this scheduler.
246 """
247 if not isinstance(requires_grad_sync, bool):
248 raise ValueError(f"requires_grad_sync should be a bool, got {type(requires_grad_sync)}")
249 self.hsdp_state.set_requires_grad_sync(requires_grad_sync)
251 # pylint: disable=W0613
252 def _hsdp_forward_pre_hook(self, cell, args, kwargs):
253 """Forward pre hook to unsharded parameter for forward process."""
254 logger.debug("hook=forward_pre enter module=%s", self.hsdp_state)
255 if self.scheduler_state == FSDPSchedulerState.PRE_BACKWARD:
256 logger.debug("hook=forward_pre skip module=%s reason=pre_backward", self.hsdp_state)
257 return args, kwargs
258 if self.scheduler_ctx.root_bp_state:
259 self._disable_forward_prefetch_for_recompute()
260 if self.scheduler_ctx.root_module is None:
261 tree_ctx = self.scheduler_ctx
262 tree_ctx.root_module = self.cell
263 registered_schedulers = set()
264 for module_name, module in tree_ctx.root_module.named_modules():
265 from hyper_parallel.core.fully_shard.api import HSDPModule # pylint: disable=C0415
266 if isinstance(module, HSDPModule):
267 submod_scheduler = module.hsdp_scheduler
268 if submod_scheduler is None or id(submod_scheduler) in registered_schedulers:
269 continue
270 registered_schedulers.add(id(submod_scheduler))
271 if submod_scheduler.scheduler_ctx is not tree_ctx:
272 if submod_scheduler.scheduler_ctx.root_module is not None:
273 raise ValueError(
274 "HSDP scheduler already belongs to another initialized module tree"
275 )
276 submod_scheduler.scheduler_ctx = tree_ctx
277 submod_scheduler.hsdp_state.scheduler_ctx = tree_ctx
278 if submod_scheduler.hsdp_state.param_group is not None:
279 submod_scheduler.hsdp_state.param_group.comm_ctx = tree_ctx.param_group_comm_ctx
280 submod_scheduler._is_root = submod_scheduler is self # pylint: disable=protected-access
281 submod_scheduler.hsdp_state.module_name = module_name
282 tree_ctx.all_hsdp_schedulers.append(submod_scheduler)
284 self.scheduler_state = FSDPSchedulerState.PRE_FORWARD
285 if self._is_root and not self.scheduler_ctx.lazy_init_done:
286 self._init_params_fqn()
287 self._lazy_init_all_states()
288 self.scheduler_ctx.lazy_init_done = True
289 if self.mp_policy.cast_forward_inputs and self.mp_policy.param_dtype:
290 cast_fn = functools.partial(cast_fp_tensor, self.mp_policy.param_dtype)
291 args = apply_to_tensors(cast_fn, args)
292 kwargs = apply_to_tensors(cast_fn, kwargs)
293 with profiler_record(f"pre_forward unshard:{self.hsdp_state.module_name}"):
294 logger.debug("hook=forward_pre action=unshard module=%s", self.hsdp_state)
295 self.hsdp_state.unshard()
296 for prefetch_cell in self.forward_prefetch_cells:
297 prefetch_state = prefetch_cell.hsdp_scheduler.hsdp_state
298 with profiler_record(f"pre_forward prefetch:"
299 f"{prefetch_state.module_name}"):
300 logger.debug(
301 "hook=forward_pre action=prefetch module=%s target=%s",
302 self.hsdp_state,
303 prefetch_state,
304 )
305 prefetch_state.prefetch()
306 return args, kwargs
308 def _lazy_init_all_states(self):
309 if self._is_root and self.scheduler_ctx.root_module is not None:
310 for submod_scheduler in self.scheduler_ctx.all_hsdp_schedulers:
311 hsdp_state = submod_scheduler.hsdp_state
312 if hsdp_state:
313 hsdp_state.lazy_init()
315 def _init_params_fqn(self): # pylint: disable=W0212
316 if not self._is_root or self.scheduler_ctx.root_module is None:
317 return
318 if self.scheduler_ctx._param_fqn_initialized: # pylint: disable=protected-access
319 return
320 # Build a map from original (sharded) parameter tensor to its HSDPParam wrapper.
321 param_to_hsdp_param = {}
322 for submod_scheduler in self.scheduler_ctx.all_hsdp_schedulers:
323 hsdp_state = submod_scheduler.hsdp_state
324 if hsdp_state is None:
325 continue
326 for hsdp_param in hsdp_state.hsdp_params:
327 orig_param = hsdp_param.sharded_param
328 # Shared parameters: keep only the first mapping to preserve the
329 # first-seen FQN (consistent with the deduplication in _init_hsdp_params).
330 if orig_param not in param_to_hsdp_param:
331 param_to_hsdp_param[orig_param] = hsdp_param
333 # Walk the full parameter tree and assign FQNs; skip params already seen
334 # (shared-parameter deduplication: first name wins).
335 visited_params = set()
336 for param_name, parameter in parameters_dict(self.scheduler_ctx.root_module):
337 if parameter in visited_params:
338 continue
339 visited_params.add(parameter)
340 hsdp_param = param_to_hsdp_param.get(parameter)
341 if hsdp_param is not None:
342 hsdp_param._param_fqn = param_name # pylint: disable=W0212
343 self.scheduler_ctx._param_fqn_initialized = True # pylint: disable=protected-access
345 # pylint: disable=W0613, R1710
346 def _hsdp_forward_hook(self, cell, inputs, outputs):
347 """Forward hook to shard parameter for saving memory."""
348 logger.debug("hook=forward enter module=%s", self.hsdp_state)
349 if self.scheduler_state == FSDPSchedulerState.PRE_BACKWARD:
350 logger.debug("hook=forward skip module=%s reason=pre_backward", self.hsdp_state)
351 return
352 self.scheduler_state = FSDPSchedulerState.FORWARD
353 if self.reshard_after_forward:
354 with profiler_record(f"forward reshard:{self.hsdp_state.module_name}"):
355 logger.debug("hook=forward action=reshard module=%s", self.hsdp_state)
356 self.hsdp_state.shard()
357 if self.mp_policy.output_dtype is not None:
358 outputs = apply_to_tensors(
359 functools.partial(cast_fp_tensor, self.mp_policy.output_dtype),
360 outputs,
361 )
362 return outputs
364 # pylint: disable=W0613
365 def _hsdp_backward_pre_hook(self, cell, grad_outputs):
366 """Backward pre hook to unsharded parameter for backward process."""
367 logger.debug("hook=backward_pre enter module=%s", self.hsdp_state)
368 self.scheduler_state = FSDPSchedulerState.PRE_BACKWARD
369 if self.reshard_after_forward:
370 with profiler_record(f"pre_backward unshard:{self.hsdp_state.module_name}"):
371 logger.debug("hook=backward_pre action=unshard module=%s", self.hsdp_state)
372 self.hsdp_state.unshard()
373 for prefetch_cell in self.backward_prefetch_cells:
374 prefetch_state = prefetch_cell.hsdp_scheduler.hsdp_state
375 with profiler_record(f"pre_backward prefetch:"
376 f"{prefetch_state.module_name}"):
377 logger.debug(
378 "hook=backward_pre action=prefetch module=%s target=%s",
379 self.hsdp_state,
380 prefetch_state,
381 )
382 prefetch_state.prefetch()
384 # pylint: disable=W0613
385 def _hsdp_backward_hook(self, cell, grad_inputs, grad_outputs):
386 """Backward hook to shard parameter for optimizer process or saving memory."""
387 logger.debug("hook=backward_hook enter module=%s", self.hsdp_state)
388 self.scheduler_state = FSDPSchedulerState.BACKWARD
389 with profiler_record(f"post_backward:{self.hsdp_state.module_name}"):
390 logger.debug("hook=backward_hook action=post_backward module=%s", self.hsdp_state)
391 self.hsdp_state.post_backward()
392 if self._fsdp_group_post_pending is not None:
393 self._fsdp_group_post_pending.clear()
395 # pylint: disable=W0613
396 @staticmethod
397 def _grouped_forward_pre_hook_skip(cell, args, kwargs) -> None:
398 """Return value when grouped pre-forward should not run (first module already did).
400 ``nn.Module`` pre-hooks use ``None`` for a no-op.
401 """
402 return None
404 @staticmethod
405 def _grouped_forward_post_hook_skip(outputs) -> None:
406 """Return value when grouped post-forward is deferred to a later module.
408 ``nn.Module`` forward hooks use ``None`` for a no-op.
409 """
410 return None
412 @_dynamo_disable
413 def _grouped_forward_pre_hook(self, cell, args, kwargs):
414 """Run FSDP pre-forward only for the first module in the group (PyTorch FSDP2-aligned)."""
415 pending = self._fsdp_group_post_pending
416 if pending is None:
417 return self._forward_pre_hook(cell, args, kwargs)
418 if len(pending) == 0:
419 pending.update(self.modules)
420 return self._forward_pre_hook(cell, args, kwargs)
421 return self._grouped_forward_pre_hook_skip(cell, args, kwargs)
423 def _make_grouped_forward_post_hook(self, mod):
424 """Build post-forward hook: last module in the group runs reshard + output backward hooks."""
426 def grouped_post_hook(cell, inputs, outputs):
427 pending = self._fsdp_group_post_pending
428 if pending is None:
429 return self._forward_hook(cell, inputs, outputs)
430 pending.discard(mod)
431 if len(pending) == 0:
432 return self._forward_hook(cell, inputs, outputs)
433 return self._grouped_forward_post_hook_skip(outputs)
435 return grouped_post_hook
437 def set_forward_prefetch_cells(self, hsdp_cell_list: List[Any]) -> None:
438 """Set cells prefetched during forward.
440 Args:
441 hsdp_cell_list: HSDP cells to prefetch ahead of forward.
442 """
443 self.forward_prefetch_cells = hsdp_cell_list
445 def set_backward_prefetch_cells(self, hsdp_cell_list: List[Any]) -> None:
446 """Set cells prefetched during backward.
448 Args:
449 hsdp_cell_list: HSDP cells to prefetch ahead of backward.
450 """
451 self.backward_prefetch_cells = hsdp_cell_list
453 def _disable_forward_prefetch_for_recompute(self) -> None:
454 """Temporarily disable forward prefetch during activation recompute."""
455 self._backup_forward_fetch = self.forward_prefetch_cells
456 self.forward_prefetch_cells = []
458 def _restore_forward_prefetch_after_recompute(self) -> None:
459 """Restore the saved forward prefetch list after activation recompute."""
460 if self._backup_forward_fetch is None:
461 return
462 self.forward_prefetch_cells = self._backup_forward_fetch
463 self._backup_forward_fetch = None
465 def _register_post_backward_hook(self, args, kwargs):
466 """Wrap forward args/kwargs through PostBackwardFunction to register backward hook."""
467 if not torch.is_grad_enabled():
468 return args, kwargs
469 args_list, args_spec = tree_flatten(args)
470 kwargs_list, kwargs_spec = tree_flatten(kwargs)
471 args_kwargs_list = list(args_list) + list(kwargs_list)
472 inp_tensor_indices: List[int] = []
473 inp_tensors: List[torch.Tensor] = []
474 for i, obj in enumerate(args_kwargs_list):
475 if torch.is_tensor(obj) and obj.requires_grad:
476 inp_tensor_indices.append(i)
477 inp_tensors.append(obj)
478 if len(inp_tensors) == 0:
479 return args, kwargs # no tensors that require gradients
480 processed_tensors = PostBackwardFunction.apply(self, *inp_tensors)
481 for inp_tensor_idx, processed_tensor in zip(inp_tensor_indices, processed_tensors):
482 args_kwargs_list[inp_tensor_idx] = processed_tensor
483 args_list = args_kwargs_list[: len(args_list)]
484 kwargs_list = args_kwargs_list[len(args_list) :]
485 args = tree_unflatten(args_list, args_spec)
486 kwargs = tree_unflatten(kwargs_list, kwargs_spec)
487 return args, kwargs
489 @_dynamo_disable
490 def _forward_pre_hook(self, cell, args, kwargs):
491 """Execute forward pre hook and set up backward hook."""
492 args, kwargs = self._hsdp_forward_pre_hook(cell, args, kwargs)
493 return self._register_post_backward_hook(args, kwargs)
495 def _register_backward_pre_hook(self, outputs) -> None:
496 """Register gradient hooks on all requires-grad outputs to trigger backward pre hook."""
497 flat_outputs, _ = tree_flatten(outputs)
498 for output in flat_outputs:
499 if isinstance(output, torch.Tensor) and output.requires_grad:
500 handle_ref = [None]
501 # pylint: disable=C0103, W0102
503 def wrapper_for_backward_pre_hook(grad, _handle_ref=handle_ref):
504 """Remove this hook after it fires to prevent accmulation"""
505 handle = _handle_ref[0]
506 if handle is not None:
507 handle.remove()
508 return self._backward_pre_hook(grad)
509 # pylint: enable=C0103, W0102
510 handle = output.register_hook(wrapper_for_backward_pre_hook)
511 handle_ref[0] = handle
513 @_dynamo_disable
514 def _forward_hook(self, cell, inputs, outputs): # pylint: disable=R1710
515 """Execute forward hook."""
516 if self.scheduler_state == FSDPSchedulerState.PRE_BACKWARD:
517 return
518 self._register_backward_pre_hook(outputs)
519 if self.scheduler_ctx.root_bp_state:
520 self._restore_forward_prefetch_after_recompute()
521 return
522 return self._hsdp_forward_hook(cell, inputs, outputs)
524 # pylint: disable=W0212
525 @_dynamo_disable
526 def _backward_pre_hook(self, grad):
527 """Execute backward pre hook."""
528 if self.scheduler_state == FSDPSchedulerState.PRE_BACKWARD:
529 return grad
530 if not self.scheduler_ctx.post_backward_final_callback_queued:
531 self.scheduler_ctx.post_backward_final_callback_queued = True
532 root_scheduler = self.scheduler_ctx.root_module.hsdp_scheduler
533 Variable._execution_engine.queue_callback(root_scheduler._root_backward_hook)
534 self.scheduler_ctx.root_bp_state = True
535 self._hsdp_backward_pre_hook(self.cell, None)
536 return grad
538 @_dynamo_disable
539 def _root_backward_hook(self):
540 """Drain gradient communication and finish backward for all states."""
541 logger.debug("hook=root_backward_hook enter module=%s", self.hsdp_state)
542 for hsdp_scheduler in self.scheduler_ctx.all_hsdp_schedulers:
543 # let modules which are not triggered backward_hook launch backward communication.
544 hsdp_scheduler._backward_hook()
545 hsdp_scheduler._finalize_unused_prefetches()
546 hsdp_scheduler.scheduler_state = None
547 self.scheduler_ctx.root_bp_state = False
548 with torch.profiler.record_function(f"root_backward reduce:{self.hsdp_state.module_name}"):
549 logger.debug(
550 "hook=root_backward_hook action=final_reduce module=%s",
551 self.hsdp_state,
552 )
553 self._finalize_comm_fusion_reductions()
554 self._finalize_per_param_reductions()
555 self._apply_reduced_gradients()
556 self.scheduler_ctx.post_backward_final_callback_queued = False
558 def _finalize_unused_prefetches(self) -> None:
559 """Wait and release this scheduler's unused all-gathers after backward.
561 A module used in forward need not participate in this backward. Its
562 speculative prefetch may therefore have no pre-backward consumer.
563 """
564 state = self.hsdp_state
565 if state is None:
566 return
567 for param in state.hsdp_params:
568 context = param.allgather_comm_ctx
569 if context.allgather_handle is None and context.allgather_output is None:
570 continue
571 with profiler_record(f"finalize_unused_prefetch param:{param}"):
572 if context.allgather_handle is not None:
573 context.allgather_handle.wait()
574 context.allgather_handle = None
575 context.allgather_output = None
576 if state.is_shard and param.shard_world_size > 1:
577 param.free_unsharded_param()
578 if state.param_group is not None:
579 for bucket in state.param_group.all_gather_buckets:
580 result = bucket.all_gather_result
581 if result is None:
582 continue
583 param_fqns = ", ".join(str(param) for param in bucket.hsdp_params)
584 with profiler_record(f"finalize_unused_prefetch bucket:{state.module_name}:{param_fqns}"):
585 if result.handle is not None:
586 result.handle.wait()
587 result.handle = None
588 result.all_gather_input = None
589 result.all_gather_output = None
590 bucket.all_gather_result = None
592 def check_state_prefetch_handles_consumed(self) -> None:
593 """Check that all parameters in the module tree consumed their prefetch work.
595 Raises:
596 RuntimeError: If a parameter or fused bucket retains an all-gather
597 handle because the configured prefetch order is incorrect.
598 """
599 unconsumed_param_fqns = []
600 for hsdp_scheduler in self.scheduler_ctx.all_hsdp_schedulers:
601 hsdp_state = hsdp_scheduler.hsdp_state
602 if hsdp_state is None:
603 continue
604 unconsumed_param_fqns.extend(
605 str(param) for param in hsdp_state.hsdp_params
606 if param.allgather_comm_ctx.allgather_handle is not None
607 )
608 if hsdp_state.param_group is not None:
609 for bucket in hsdp_state.param_group.all_gather_buckets:
610 result = bucket.all_gather_result
611 if result is not None and result.handle is not None:
612 unconsumed_param_fqns.extend(str(param) for param in bucket.hsdp_params)
613 if unconsumed_param_fqns:
614 param_fqns = ", ".join(dict.fromkeys(unconsumed_param_fqns))
615 raise RuntimeError(
616 f"Unconsumed prefetch handles for parameters: {param_fqns}. "
617 "The prefetch order is incorrect. Set forward/backward prefetch order "
618 "according to the model's actual execution order."
619 )
621 def _finalize_comm_fusion_reductions(self) -> None:
622 """Drain the comm_fusion=True RS/AR pipeline."""
623 comm_ctx = self.scheduler_ctx.param_group_comm_ctx
624 if comm_ctx.all_reduce_param_group is not None:
625 logger.debug(
626 "hook=root_backward_hook wait=comm_fusion_all_reduce module=%s",
627 self.hsdp_state,
628 )
629 comm_ctx.all_reduce_param_group.wait_all_reduce_and_save_grad()
630 comm_ctx.all_reduce_param_group = None
631 if comm_ctx.pre_param_group is not None:
632 logger.debug(
633 "hook=root_backward_hook wait=comm_fusion_reduce_scatter module=%s",
634 self.hsdp_state,
635 )
636 comm_ctx.pre_param_group.wait_reduce_scatter_and_issue_all_reduce()
637 comm_ctx.pre_param_group = None
638 if comm_ctx.all_reduce_param_group is not None:
639 comm_ctx.all_reduce_param_group.wait_all_reduce_and_save_grad()
640 comm_ctx.all_reduce_param_group = None
642 def _finalize_per_param_reductions(self) -> None:
643 """Drain the module-tree-local comm_fusion=False RS/AR queues."""
644 # A fused root may own non-fused children, so always drain the tree queues.
645 while self.scheduler_ctx.pre_reduce_scatter_params:
646 last_all_reduce_groups = self.hsdp_state._wait_prev_reduce_scatter()
647 self.hsdp_state._wait_prev_reduce_scatter_without_all_reduce()
648 self.hsdp_state._issue_prev_fused_all_reduce(last_all_reduce_groups)
649 self.hsdp_state.wait_and_split_all_reduce_work_groups()
651 def _apply_reduced_gradients(self) -> None:
652 """Apply pending reduced gradients after communication completes."""
653 for hsdp_scheduler in self.scheduler_ctx.all_hsdp_schedulers:
654 hsdp_state = hsdp_scheduler.hsdp_state
655 if hsdp_state is None:
656 continue
657 need_synchronize = False
658 for hsdp_param in hsdp_state.hsdp_params:
659 reduced_grad = hsdp_param.all_reduce_comm_ctx.all_reduce_output
660 if reduced_grad is None:
661 reduced_grad = hsdp_param.reduce_scatter_comm_ctx.reduce_scatter_output
662 if reduced_grad is None:
663 continue
664 if hsdp_state.comm_fusion_policy.enable_comm_fusion:
665 hsdp_param.all_reduce_source_replicate_grad_inplace(
666 reduced_grad,
667 hsdp_state.reduce_op_type,
668 )
669 need_synchronize = hsdp_param.apply_reduced_grad(reduced_grad) or need_synchronize
670 hsdp_param.clear_all_reduce_output()
671 hsdp_param.clear_reduce_scatter_output()
672 hsdp_state._sync_current_stream_if_needed(need_synchronize)
674 @_dynamo_disable
675 def _backward_hook(self):
676 """Execute backward hook."""
677 if self.scheduler_state == FSDPSchedulerState.BACKWARD:
678 return
679 self._hsdp_backward_hook(self.cell, None, None)
681 @staticmethod
682 def _register_forward_module_hook(mod, hook) -> None:
683 """Register forward hook; use ``always_call=True`` when supported (matches PyTorch FSDP)."""
684 sig = inspect.signature(mod.register_forward_hook)
685 if "always_call" in sig.parameters:
686 mod.register_forward_hook(hook, prepend=False, always_call=True)
687 else:
688 mod.register_forward_hook(hook, prepend=False)
690 def _register_forward_backward_hooks(self):
691 """Register module forward and backward hook on all managed modules."""
692 if self._fsdp_group_post_pending is None:
693 for mod in self.modules:
694 mod.register_forward_pre_hook(self._forward_pre_hook, with_kwargs=True)
695 mod.register_forward_hook(self._forward_hook)
696 return
697 for mod in self.modules:
698 mod.register_forward_pre_hook(self._grouped_forward_pre_hook, with_kwargs=True)
699 grouped_forward_hook = _dynamo_disable(self._make_grouped_forward_post_hook(mod))
700 self._register_forward_module_hook(mod, grouped_forward_hook)