Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / utils / clip_grad.py: 78%
443 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Distributed-aware gradient clipping for parallel training.
17Communication is driven by each parameter's DTensorSpec (device_mesh +
18placements) rather than any specific parallelism strategy, so a single
19implementation covers FSDP, HSDP, TP+FSDP, and other DTensor-expressed
20parallelisms.
22Collective safety aligned with FSDP1; numerical precision aligned with
23FSDP2's ``_NormPartial`` norm computation path:
25* Gradient norms from sharded parameters are all-reduced across the
26 corresponding shard process group.
27* Non-sharded / replicated norms contribute locally without communication.
28* **All ranks participate in the same collectives** regardless of local
29 gradient availability, preventing collective-misalignment deadlocks.
31The finite p-norm is bit-exact with upstream only for the pure-FSDP,
32single-dtype case; mixed sharded + replicated is mathematically correct
33but intentionally diverges (see ``_total_norm_fsdp2_aligned``).
35Note: PP does not use DTensor layout for gradients today. Cross-stage
36norm aggregation will require an additional manual all-reduce and is
37left for future work.
38"""
39# pylint: disable=not-callable
40import functools
41import math
42import warnings
43from collections import defaultdict, namedtuple
44from collections.abc import Mapping
45from dataclasses import dataclass
46from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union
48import torch
49import torch.distributed as dist
51from hyper_parallel.core.dtensor.placement_types import Partial
53if TYPE_CHECKING:
54 from hyper_parallel.core.dtensor.layout import Layout
56try:
57 from torch.utils._foreach_utils import (
58 _device_has_foreach_support,
59 _group_tensors_by_device_and_dtype,
60 _has_foreach_support,
61 )
62except ImportError:
63 _device_has_foreach_support = None # type: ignore[assignment]
64 _group_tensors_by_device_and_dtype = None # type: ignore[assignment]
65 _has_foreach_support = None # type: ignore[assignment]
67__all__: list[str] = ["clip_grad_norm_"]
70@dataclass(frozen=True)
71class NormShardPlan:
72 """One norm contributor and its replica ranks for each dense parameter."""
74 representatives: dict[str, bool]
75 replica_ranks: dict[str, tuple[int, ...]]
78def _norm_coverage(param: torch.nn.Parameter) -> tuple[tuple[int, ...], tuple[tuple[int, int], ...]]:
79 """Describe the exact global rectangle represented by a local parameter."""
80 global_shape = tuple(param.shape)
81 layout = getattr(param, "layout", None) if _is_dtensor(param) else None
82 if layout is None:
83 area = tuple((0, size) for size in global_shape)
84 else:
85 from hyper_parallel.core.dtensor.layout import infer_slice_area_by_layout # pylint: disable=C0415
87 rank = dist.get_rank() if dist.is_initialized() else 0
88 if rank not in layout.rank_list:
89 raise ValueError("Norm layout does not contain the current global rank")
90 area = infer_slice_area_by_layout(
91 layout, layout.rank_list.index(rank), global_shape,
92 )
93 local = param._local_tensor if _is_dtensor(param) else param # pylint: disable=W0212
94 if tuple(end - start for start, end in area) != tuple(local.shape):
95 raise ValueError("Norm layout cannot describe the exact local parameter coverage")
96 return global_shape, tuple((int(start), int(end)) for start, end in area)
99def _norm_representative(fqn: str, shape: tuple[int, ...],
100 stage_data: dict[int, dict], rank: int,
101 local_area: tuple) -> tuple[bool, tuple[int, ...]]:
102 """Validate disjoint shard coverage and identify copies of this area."""
103 representatives = {}
104 for member, data in stage_data.items():
105 if data[fqn][0] != shape:
106 raise ValueError(f"Norm shard global shape mismatch: {fqn}")
107 area = data[fqn][1]
108 representatives.setdefault(area, []).append(member)
109 nonempty = [area for area in representatives if all(end > start for start, end in area)]
110 volume = sum(math.prod(end - start for start, end in area) for area in nonempty)
111 if volume != math.prod(shape):
112 raise ValueError(f"Norm shard coverage is incomplete or duplicated: {fqn}")
113 for index, left in enumerate(nonempty):
114 for right in nonempty[index + 1:]:
115 if all(max(a[0], b[0]) < min(a[1], b[1]) for a, b in zip(left, right)):
116 raise ValueError(f"Norm shard rectangles overlap: {fqn}")
117 copies = tuple(representatives[local_area])
118 return rank == min(copies), copies
121def build_norm_shard_plan(model: torch.nn.Module, excluded_params: Iterable[torch.nn.Parameter],
122 mesh_context: Any) -> NormShardPlan:
123 """Choose one rank per identical global shard and verify full coverage.
125 Args:
126 model: Model with final FSDP parameter layouts.
127 excluded_params: Parameters managed by an external optimizer.
128 mesh_context: Stage-local mesh and topology metadata.
129 """
130 excluded_ids = {id(param) for param in excluded_params}
131 local = {
132 fqn: _norm_coverage(param)
133 for fqn, param in model.named_parameters()
134 if param.requires_grad and id(param) not in excluded_ids
135 }
136 if dist.is_initialized():
137 gathered = [None] * dist.get_world_size()
138 dist.all_gather_object(gathered, local)
139 rank = dist.get_rank()
140 else:
141 gathered = [local]
142 rank = 0
143 mesh = getattr(mesh_context, "device_mesh", None)
144 stage_ranks = (tuple(mesh.rank_list) if mesh is not None and hasattr(mesh, "rank_list")
145 else tuple(range(len(gathered))))
146 stage_data = {member: gathered[member] for member in stage_ranks}
147 reference_names = set(local)
148 if any(set(data) != reference_names for data in stage_data.values()):
149 raise ValueError("Norm shard plan has different dense FQNs within one PP stage")
150 representatives = {}
151 replica_ranks = {}
152 for fqn in sorted(reference_names):
153 representatives[fqn], replica_ranks[fqn] = _norm_representative(
154 fqn, local[fqn][0], stage_data, rank, local[fqn][1],
155 )
156 return NormShardPlan(representatives, replica_ranks)
159def _effective_dense_grads(parameters: list[tuple[str, torch.nn.Parameter]]) -> tuple[list, dict]:
160 """Collect local gradients and reduce partial placements once."""
161 infos = []
162 mesh_cache = {}
163 for _, param in parameters:
164 mesh, shard_dims, partial_info = _get_param_mesh_info(param)
165 if mesh is not None:
166 mesh_cache[id(mesh)] = mesh
167 infos.append((param, _get_local_grad(param), mesh, partial_info,
168 (id(mesh) if mesh is not None else None, shard_dims)))
169 return infos, _coalesce_partial_reduce(infos, mesh_cache)
172def _check_dense_grad_presence(present: dict[str, bool], replica_ranks: dict[str, tuple[int, ...]]) -> None:
173 """Reject mismatched gradient presence on copies of the same shard."""
174 if not dist.is_initialized():
175 return
176 gathered = [None] * dist.get_world_size()
177 dist.all_gather_object(gathered, present)
178 for fqn, ranks in replica_ranks.items():
179 copies = [gathered[member][fqn] for member in ranks]
180 if any(value != copies[0] for value in copies[1:]):
181 raise ValueError(f"Norm shard gradient presence differs across replicas: {fqn}")
184def dense_grad_norm_sq_and_refs(model: torch.nn.Module,
185 excluded_params: Iterable[torch.nn.Parameter],
186 norm_shard_plan: NormShardPlan,
187 mesh_context: Any) -> tuple[torch.Tensor, list[torch.Tensor]]:
188 """Sum each dense global shard once and return mutable effective gradients.
190 Args:
191 model: Model with final FSDP parameter layouts.
192 excluded_params: Parameters managed by an external optimizer.
193 norm_shard_plan: Validated coverage and one representative per shard.
194 mesh_context: Stage-local mesh and topology metadata.
195 """
196 excluded_ids = {id(param) for param in excluded_params}
197 parameters = [
198 (fqn, param) for fqn, param in model.named_parameters()
199 if param.requires_grad and id(param) not in excluded_ids
200 ]
201 device = next(((_param_device(param)) for _, param in parameters), None)
202 if device is None:
203 mesh = getattr(mesh_context, "device_mesh", None)
204 device = torch.device(getattr(mesh, "device_type", "cpu"))
205 infos, reduced = _effective_dense_grads(parameters)
206 refs = []
207 local_sq = torch.zeros((), device=device, dtype=torch.float32)
208 present = {}
209 for index, (fqn, _) in enumerate(parameters):
210 grad = infos[index][1]
211 present[fqn] = grad is not None
212 if grad is None:
213 continue
214 refs.append(grad)
215 if norm_shard_plan.representatives[fqn]:
216 view = reduced.get(index, grad)
217 local_sq.add_(view.float().square().sum().to(device))
218 _check_dense_grad_presence(present, norm_shard_plan.replica_ranks)
219 for name in getattr(getattr(mesh_context, "device_mesh", None), "mesh_dim_names", ()) or ():
220 axis = mesh_context.device_mesh[name]
221 if axis.size() > 1:
222 dist.all_reduce(local_sq, op=dist.ReduceOp.SUM, group=axis.get_group())
223 return local_sq, refs
226def scale_grad_refs_(refs: Iterable[torch.Tensor], coefficient: torch.Tensor) -> None:
227 """Scale the exact dense gradients that the optimizer will consume.
229 Args:
230 refs: Mutable dense effective gradients.
231 coefficient: Shared clipping coefficient.
232 """
233 for grad in refs:
234 grad.mul_(coefficient)
237# (id(mesh) or None, shard_dims) -> list of local grads for norm computation
238_GradGroupKey = Tuple[Optional[int], Tuple[int, ...]]
240# (mesh_dim_index, dist.ReduceOp, needs_manual_avg)
241_PartialReduceInfo = Tuple[int, "dist.ReduceOp", bool]
243# Result of _build_grad_groups; tuple-unpacking compatible with prior 7-tuple.
244_GradGroups = namedtuple(
245 "_GradGroups",
246 "grad_groups all_grads norm_grads key_per_grad mesh_cache device "
247 "has_dtensor_grad",
248)
251def _is_dtensor(value: object) -> bool:
252 """Return whether *value* is a HyperParallel DTensor.
254 The DTensor module imports ``core.utils`` during initialization, so keep
255 this import lazy to avoid a package initialization cycle.
256 """
257 from hyper_parallel.core.dtensor.dtensor import DTensor # pylint: disable=C0415
259 return isinstance(value, DTensor)
262# ---------------------------------------------------------------------------
263# Reduce-op mapping
264# ---------------------------------------------------------------------------
266_REDUCE_OP_AVG_SUPPORTED = hasattr(dist.ReduceOp, "AVG")
268_STR_TO_REDUCE_OP: Dict[str, "dist.ReduceOp"] = {
269 "sum": dist.ReduceOp.SUM,
270 "max": dist.ReduceOp.MAX,
271 "min": dist.ReduceOp.MIN,
272}
273if _REDUCE_OP_AVG_SUPPORTED:
274 _STR_TO_REDUCE_OP["avg"] = dist.ReduceOp.AVG
277def _str_to_reduce_op(op_str: str) -> Tuple["dist.ReduceOp", bool]:
278 """Map a ``Partial`` placement's *reduce_op* string to ``dist.ReduceOp``.
280 Returns ``(reduce_op, needs_manual_avg)`` where *needs_manual_avg*
281 is ``True`` when ``"avg"`` is requested but the backend does not
282 support ``dist.ReduceOp.AVG`` — the caller should use SUM and
283 manually divide by the group size.
284 """
285 lower = op_str.lower()
286 if lower == "avg" and not _REDUCE_OP_AVG_SUPPORTED:
287 return dist.ReduceOp.SUM, True
288 op = _STR_TO_REDUCE_OP.get(lower)
289 if op is None:
290 raise ValueError(
291 f"Unsupported Partial reduce_op: {op_str!r}. "
292 f"Supported: {sorted(set(list(_STR_TO_REDUCE_OP) + ['avg']))}"
293 )
294 return op, False
297# ---------------------------------------------------------------------------
298# Helpers
299# ---------------------------------------------------------------------------
301def _normalize_parameters(
302 parameters: Union["torch.nn.Module", torch.Tensor, Iterable[torch.Tensor]],
303) -> List[torch.Tensor]:
304 """Normalize *parameters* to a flat list of tensors.
306 * ``torch.nn.Module`` -> ``list(module.parameters())``
307 * single ``torch.Tensor`` -> ``[tensor]``
308 * iterable of tensors -> ``list(iterable)``
309 """
310 if isinstance(parameters, torch.nn.Module):
311 return list(parameters.parameters())
312 if isinstance(parameters, torch.Tensor):
313 return [parameters]
314 return list(parameters)
317def _param_device(param: torch.Tensor) -> torch.device:
318 """Return the local device of *param* (unwrap DTensor if needed)."""
319 if _is_dtensor(param):
320 return param._local_tensor.device # pylint: disable=protected-access
321 return param.device
324def _all_reduce_offloaded(
325 tensor: torch.Tensor, op: dist.ReduceOp, group: object,
326) -> torch.Tensor:
327 """Reduce CPU-offloaded values on the backend's communication device."""
328 original_device = tensor.device
329 if original_device.type == "cpu":
330 device_type = {"nccl": "cuda", "hccl": "npu"}.get(dist.get_backend(group))
331 if device_type is not None:
332 device_module = torch.get_device_module(device_type)
333 tensor = tensor.to(torch.device(device_type, device_module.current_device()))
334 dist.all_reduce(tensor, op=op, group=group)
335 return tensor.to(original_device)
338def _get_grad_obj(param: torch.nn.Parameter) -> Optional[torch.Tensor]:
339 """Return the gradient object for *param*.
341 Checks ``param.main_grad`` first (used when
342 ``MixedPrecisionPolicy.apply_grad_on_fp32_main_grad=True``),
343 falling back to ``param.grad``.
344 """
345 grad = getattr(param, "main_grad", None)
346 if grad is not None:
347 return grad
348 return param.grad
351def _get_local_grad(param: torch.nn.Parameter) -> Optional[torch.Tensor]:
352 """Return the local gradient tensor, or ``None`` if absent.
354 Supports ``main_grad`` for fp32 mixed-precision training.
355 """
356 if not param.requires_grad:
357 return None
358 grad = _get_grad_obj(param)
359 if grad is None:
360 return None
361 if _is_dtensor(grad):
362 return grad._local_tensor # pylint: disable=protected-access
363 return grad
366def _get_param_mesh_info(
367 param: torch.nn.Parameter,
368 parameter_layout: Optional["Layout"] = None,
369) -> Tuple[
370 Optional[object],
371 Tuple[int, ...],
372 Tuple[_PartialReduceInfo, ...],
373]:
374 """Derive DeviceMesh, Shard dims and Partial info from DTensorSpec.
376 Checks the *gradient's* spec first; falls back to the *parameter's*
377 spec when the gradient is a plain tensor on a DTensor parameter
378 (common after FSDP/HSDP backward where ``param.grad`` is stored as
379 the local shard tensor).
381 Returns ``(mesh, shard_dims, partial_info)`` where *partial_info*
382 is a tuple of ``(mesh_dim, dist.ReduceOp, needs_manual_avg)``
383 triples that respect the ``Partial`` placement's ``reduce_op``
384 attribute. *needs_manual_avg* is ``True`` when ``"avg"`` was
385 requested but the backend lacks ``dist.ReduceOp.AVG`` support.
386 """
387 grad = _get_grad_obj(param)
388 # Prefer grad's spec (most accurate); fall back to param's.
389 spec_source = grad if _is_dtensor(grad) else param
390 if parameter_layout is None:
391 if not _is_dtensor(spec_source):
392 return None, (), ()
393 placements = spec_source.placements
394 mesh = spec_source.device_mesh
395 else:
396 layout = spec_source.layout if _is_dtensor(spec_source) else None
397 if layout is not None and (
398 layout.mesh_shape, layout.rank_list, layout.alias_name, layout.placements, layout.tensor_shape
399 ) != (
400 parameter_layout.mesh_shape, parameter_layout.rank_list, parameter_layout.alias_name,
401 parameter_layout.placements, parameter_layout.tensor_shape
402 ):
403 raise ValueError("Explicit parameter layout conflicts with its DTensor layout")
404 layout = layout if layout is not None else parameter_layout
405 placements = layout.placements
406 mesh = layout.mesh
408 shard_dims = tuple(
409 i for i, p in enumerate(placements)
410 if p.is_shard()
411 )
412 partial_info = tuple(
413 (i, *_str_to_reduce_op(p.reduce_op))
414 for i, p in enumerate(placements)
415 if isinstance(p, Partial)
416 )
417 return mesh, shard_dims, partial_info
420def _sum_p_norms(
421 dev_grads: List[torch.Tensor],
422 norm_type: float,
423 device: torch.device,
424 total: torch.Tensor,
425) -> None:
426 """Accumulate sum-of-p-th-powers for *dev_grads* into *total*."""
427 for g in dev_grads:
428 n = torch.linalg.vector_norm(g, norm_type)
429 total.add_(n.to(device=device) ** norm_type)
432def _foreach_p_norms(
433 grads: List[torch.Tensor],
434 norm_type: float,
435 device: torch.device,
436) -> torch.Tensor:
437 """Fast path: fuse per-tensor norms via ``_foreach_norm``.
439 Restricted to float32 tensors to preserve the same numerical
440 precision as ``vector_norm(dtype=float32)``. Non-float32 tensors
441 and backends that raise ``RuntimeError`` fall back to per-tensor
442 ``vector_norm``.
443 """
444 total = torch.tensor(0.0, device=device, dtype=torch.float32)
445 grouped = _group_tensors_by_device_and_dtype(
446 [[g.detach() for g in grads]],
447 )
448 for (dev, _), ([dev_grads], _) in grouped.items():
449 if (
450 dev_grads[0].dtype == torch.float32
451 and _has_foreach_support(dev_grads, dev)
452 ):
453 try:
454 per_norms = torch._foreach_norm( # pylint: disable=W0212
455 dev_grads, norm_type,
456 )
457 except RuntimeError:
458 per_norms = None
459 if per_norms is not None:
460 total.add_(
461 torch.stack([
462 n.to(device=device) ** norm_type
463 for n in per_norms
464 ]).sum(),
465 )
466 else:
467 _sum_p_norms(dev_grads, norm_type, device, total)
468 else:
469 _sum_p_norms(dev_grads, norm_type, device, total)
470 return total
473def _per_tensor_norms(
474 grads: List[torch.Tensor],
475 norm_type: float,
476 device: torch.device,
477) -> List[torch.Tensor]:
478 """Return per-tensor norms as a list of scalar tensors on *device*."""
479 if not grads:
480 return []
482 if _group_tensors_by_device_and_dtype is None or not hasattr(torch, "_foreach_norm"):
483 return [
484 torch.linalg.vector_norm(g.detach(), norm_type).to(device=device)
485 for g in grads
486 ]
488 norms: List[torch.Tensor] = []
489 grouped = _group_tensors_by_device_and_dtype(
490 [[g.detach() for g in grads]],
491 )
492 for (dev, _), ([dev_grads], _) in grouped.items():
493 if dev_grads and _has_foreach_support(dev_grads, dev):
494 try:
495 per_norms = torch._foreach_norm( # pylint: disable=W0212
496 dev_grads, norm_type,
497 )
498 except RuntimeError:
499 per_norms = None
500 if per_norms is not None:
501 norms.extend(
502 [n.to(device=device) for n in per_norms],
503 )
504 continue
505 norms.extend([
506 torch.linalg.vector_norm(g, norm_type).to(device=device)
507 for g in dev_grads
508 ])
509 return norms
512def _compute_local_norm( # pylint: disable=R0911
513 grads: List[torch.Tensor],
514 norm_type: float,
515 device: torch.device,
516) -> torch.Tensor:
517 """Compute the combined norm of *grads* locally in FP32.
519 When *grads* is empty, returns the **identity element** for the
520 subsequent all-reduce so that this rank contributes a neutral value
521 (aligned with FSDP1's ``_zero_scalar`` approach):
523 * ``inf`` -> 0 (neutral for MAX; norms are non-negative)
524 * ``-inf`` -> +inf (neutral for MIN)
525 * ``0`` -> 0 (neutral for SUM)
526 * finite -> 0 (neutral for SUM)
527 """
528 if not grads:
529 if norm_type == -math.inf:
530 return torch.tensor(
531 float("inf"), device=device, dtype=torch.float32,
532 )
533 return torch.tensor(0.0, device=device, dtype=torch.float32)
535 if norm_type == math.inf:
536 norms = [
537 torch.linalg.vector_norm(g.detach(), math.inf)
538 for g in grads
539 ]
540 return torch.stack(norms).max().to(device)
542 if norm_type == -math.inf:
543 norms = [
544 torch.linalg.vector_norm(g.detach(), -math.inf)
545 for g in grads
546 ]
547 return torch.stack(norms).min().to(device)
549 if norm_type == 0:
550 norms = [
551 torch.linalg.vector_norm(g.detach(), 0)
552 for g in grads
553 ]
554 return torch.stack(norms).sum().to(device)
556 # Finite p-norm: return sum of p-th powers.
557 if (
558 len(grads) > 1
559 and _group_tensors_by_device_and_dtype is not None
560 and hasattr(torch, "_foreach_norm")
561 ):
562 return _foreach_p_norms(grads, norm_type, device)
564 # Scalar fallback when foreach utilities are unavailable.
565 norms = [
566 torch.linalg.vector_norm(g.detach(), norm_type)
567 for g in grads
568 ]
569 norm_powers = [n.to(device=device) ** norm_type for n in norms]
570 return torch.stack(norm_powers).sum()
573# ---------------------------------------------------------------------------
574# Total norm aggregation with collectives
575# ---------------------------------------------------------------------------
577def _get_total_norm(
578 grad_groups: Dict[_GradGroupKey, List[torch.Tensor]],
579 norm_type: float,
580 mesh_cache: Dict[int, object],
581 device: torch.device,
582 norm_grads: List[torch.Tensor],
583 key_per_grad: List[_GradGroupKey],
584) -> torch.Tensor:
585 """Compute total gradient norm with per-group all-reduce.
587 ``norm_grads`` (parallel to ``key_per_grad``) holds the tensor whose
588 norm to take per parameter; only the finite p-norm path consumes it.
589 """
590 if norm_type == math.inf:
591 return _total_norm_inf(
592 grad_groups, norm_type, mesh_cache, device,
593 dist.ReduceOp.MAX,
594 )
596 if norm_type == -math.inf:
597 return _total_norm_inf(
598 grad_groups, norm_type, mesh_cache, device,
599 dist.ReduceOp.MIN,
600 )
602 if norm_type == 0:
603 return _total_norm_sum(
604 grad_groups, norm_type, mesh_cache, device,
605 )
607 # Finite p-norm: FSDP2-aligned sequence.
608 total_p = _total_norm_fsdp2_aligned(
609 grad_groups, norm_type, mesh_cache, device,
610 norm_grads, key_per_grad,
611 )
612 return total_p ** (1.0 / norm_type)
615def _total_norm_inf( # pylint: disable=R0913,R0917
616 grad_groups, norm_type, mesh_cache, device, reduce_op,
617):
618 """Shared logic for inf / -inf norms."""
619 group_norms: List[torch.Tensor] = []
620 for (mesh_id, shard_dims), grads in grad_groups.items():
621 local_norm = _compute_local_norm(grads, norm_type, device)
622 if mesh_id is not None:
623 mesh = mesh_cache[mesh_id]
624 for dim in shard_dims:
625 local_norm = _all_reduce_offloaded(local_norm, reduce_op, mesh.get_group(dim))
626 group_norms.append(local_norm)
627 if not group_norms:
628 if norm_type == -math.inf:
629 return torch.tensor(float("inf"), device=device)
630 return torch.tensor(0.0, device=device)
631 stacked = torch.stack(group_norms)
632 return stacked.max() if reduce_op == dist.ReduceOp.MAX else stacked.min()
635def _total_norm_sum(grad_groups, norm_type, mesh_cache, device):
636 """Shared logic for finite norms and L0 (all use SUM all-reduce)."""
637 total = torch.tensor(0.0, device=device)
638 for (mesh_id, shard_dims), grads in grad_groups.items():
639 local_val = _compute_local_norm(grads, norm_type, device)
640 if mesh_id is not None:
641 mesh = mesh_cache[mesh_id]
642 for dim in shard_dims:
643 local_val = _all_reduce_offloaded(local_val, dist.ReduceOp.SUM, mesh.get_group(dim))
644 total.add_(local_val)
645 return total
648def _reduction_signature(grad_groups, mesh_cache):
649 """Bucket grad-group keys by the *process group(s)* they reduce over.
651 Two keys that reduce over the same set of process groups (same global
652 ranks per shard dim) must be **pooled** so their norms accumulate in
653 one stack -- matching FSDP2's single foreach-norm + single reduce and
654 keeping the loss bit-exact even when params live on several distinct
655 ``DeviceMesh`` objects that share the same DP process group (common
656 for multi-component models). Keys that reduce over *different*
657 process groups (TP+FSDP heterogeneous sharding, expert parallel) get
658 separate buckets, each reduced over its own group.
660 Returns ``(key_to_sig, sig_groups, sig_order)``:
662 * ``key_to_sig`` -- ``key -> signature`` (hashable; ``()`` = replicate
663 / no communication).
664 * ``sig_groups`` -- ``signature -> list[ProcessGroup]`` to all-reduce.
665 * ``sig_order`` -- signatures in first-seen (parameter) order, so all
666 ranks issue the same collectives in the same order.
667 """
668 key_to_sig: Dict[_GradGroupKey, Tuple] = {}
669 sig_groups: Dict[Tuple, List[object]] = {}
670 sig_order: List[Tuple] = []
671 for mesh_id, shard_dims in grad_groups:
672 if shard_dims and mesh_id is not None:
673 mesh = mesh_cache[mesh_id]
674 groups = [mesh.get_group(dim) for dim in shard_dims]
675 sig = tuple(
676 tuple(dist.get_process_group_ranks(group)) for group in groups
677 )
678 else:
679 groups = []
680 sig = ()
681 key_to_sig[(mesh_id, shard_dims)] = sig
682 if sig not in sig_groups:
683 sig_groups[sig] = groups
684 sig_order.append(sig)
685 return key_to_sig, sig_groups, sig_order
688def _total_norm_fsdp2_aligned(grad_groups, norm_type, mesh_cache, device,
689 norm_grads, key_per_grad):
690 """FSDP2-aligned norm for finite p-norms.
692 Grads are bucketed by the *process group* they reduce over (see
693 :func:`_reduction_signature`); ``norm_grads`` supplies, in global
694 parameter order, the Partial-reduced (already-global) view for Partial
695 grads and the raw local grad otherwise. Each bucket does ONE ``stack``
696 → ONE ``vector_norm`` → ``^p`` → ONE ``all_reduce SUM`` per shard dim,
697 and the buckets are summed locally.
699 Reduction rules:
701 * Sharded bucket -- reduce over its own group(s). Same-group params
702 across distinct ``DeviceMesh`` objects pool into one reduce; distinct
703 groups (TP+FSDP heterogeneous, expert parallel) stay separate -- the
704 per-group convention also used by :func:`_total_norm_inf` /
705 :func:`_total_norm_sum` and by VeOmni / torchtitan.
706 * Replicate bucket (signature ``()``) -- contribute norm² locally with
707 NO communication; reducing it over the shard group would over-count
708 it ``shard_world_size`` times (the FSDP1 convention).
710 Empty-but-present buckets still issue their all_reduce (identity ``0``
711 for SUM) so all ranks run the same collectives.
713 Bit-exact with upstream ``torch.nn.utils.clip_grad_norm_`` only for the
714 pure-FSDP, single-bucket, single-dtype case. Mixed shard + replicate is
715 mathematically correct but intentionally diverges (upstream folds the
716 replicate norm into one ``_NormPartial`` reduce and over-counts it);
717 mixed-dtype follows upstream's :func:`_per_tensor_norms` device/dtype
718 regrouping rather than strict global order.
720 Returns the global sum of p-th powers (caller takes p-th root).
721 """
722 key_to_sig, sig_groups, sig_order = _reduction_signature(
723 grad_groups, mesh_cache,
724 )
726 # Bucket norms by reduction signature, preserving global parameter order.
727 sig_grads: Dict[Tuple, List[torch.Tensor]] = {sig: [] for sig in sig_order}
728 for grad, key in zip(norm_grads, key_per_grad):
729 sig_grads[key_to_sig[key]].append(grad)
731 total_p = torch.tensor(0.0, device=device, dtype=torch.float32)
732 for sig in sig_order:
733 grads = sig_grads[sig]
734 if grads:
735 norms = _per_tensor_norms(grads, norm_type, device)
736 local_p = torch.linalg.vector_norm(
737 torch.stack(norms).to(torch.float32), norm_type,
738 ) ** norm_type
739 else:
740 local_p = torch.tensor(0.0, device=device, dtype=torch.float32)
742 for group in sig_groups[sig]:
743 local_p = _all_reduce_offloaded(local_p, dist.ReduceOp.SUM, group)
745 total_p = total_p + local_p
747 return total_p
750def _build_coalesce_buffer(
751 param_infos: List[Tuple],
752 indices: List[int],
753) -> Tuple[List[torch.Tensor], List[int], List[bool], List[int]]:
754 """Build flat fp32 chunks for one coalesce group.
756 Returns ``(chunks, chunk_sizes, has_grad, active_indices)``.
757 Frozen params are skipped; trainable grad-free params contribute
758 zeros so the collective matches ranks that have a grad.
759 """
760 chunks: List[torch.Tensor] = []
761 chunk_sizes: List[int] = []
762 has_grad: List[bool] = []
763 active_indices: List[int] = []
765 for idx in indices:
766 param = param_infos[idx][0]
767 local_grad = param_infos[idx][1]
768 if local_grad is not None:
769 chunks.append(
770 local_grad.detach().reshape(-1).to(torch.float32),
771 )
772 chunk_sizes.append(local_grad.numel())
773 has_grad.append(True)
774 active_indices.append(idx)
775 elif param.requires_grad:
776 local_p = (
777 param._local_tensor # pylint: disable=W0212
778 if _is_dtensor(param) else param.data
779 )
780 numel = local_p.numel()
781 chunks.append(
782 torch.zeros(
783 numel, device=local_p.device,
784 dtype=torch.float32,
785 ),
786 )
787 chunk_sizes.append(numel)
788 has_grad.append(False)
789 active_indices.append(idx)
791 return chunks, chunk_sizes, has_grad, active_indices
794def _coalesce_partial_reduce( # pylint: disable=R0914
795 param_infos: List[Tuple],
796 mesh_cache: Dict[int, object],
797) -> Dict[int, torch.Tensor]:
798 """Coalesce Partial all-reduces: O(N) collectives → O(G).
800 Groups parameters sharing the same ``(mesh, partial_info)`` and
801 flattens their gradients (or zeros for trainable grad-free params)
802 into a single fp32 buffer. **One** ``all_reduce`` per buffer
803 replaces the previous per-parameter collective calls.
805 For TP+FSDP (all params share the same mesh / placements), this
806 turns ~200 individual all-reduces into 1 — saving 10-20 ms per
807 training step at typical HCCS/NCCL latencies.
809 Frozen params (``requires_grad=False``) are consistently grad-free
810 across all ranks and are excluded from the buffer to avoid wasting
811 bandwidth.
813 All buffers use float32 to guarantee dtype consistency across ranks
814 in mixed-precision training (grad may be fp16/bf16 while param is
815 fp32).
817 Returns a dict mapping *param_infos* index → reduced gradient view
818 (1-D fp32 slice of the coalesced buffer). Only entries for params
819 with actual gradients are included.
820 """
821 # Group by Partial coalesce key: (mesh_id, partial_info)
822 coalesce_groups: Dict[
823 Tuple, List[int],
824 ] = defaultdict(list)
825 for idx, info in enumerate(param_infos):
826 mesh, partial_info = info[2], info[3]
827 if partial_info:
828 if mesh is None:
829 raise RuntimeError(
830 "clip_grad_norm_: parameter has Partial placements "
831 "but no DeviceMesh. This is a DTensor invariant "
832 "violation."
833 )
834 pck = (id(mesh), partial_info)
835 coalesce_groups[pck].append(idx)
837 reduced: Dict[int, torch.Tensor] = {}
839 for (mesh_id, partial_info), indices in coalesce_groups.items():
840 mesh = mesh_cache[mesh_id]
841 chunks, chunk_sizes, has_grad, active_indices = (
842 _build_coalesce_buffer(param_infos, indices)
843 )
845 if not chunks:
846 continue # all params frozen, no collective needed
848 # Sanity check: same mesh → same device. Fail fast on
849 # misconfigured inputs rather than silent NCCL errors.
850 buf_device = chunks[0].device
851 for chunk in chunks[1:]:
852 if chunk.device != buf_device:
853 raise RuntimeError(
854 f"clip_grad_norm_: parameters in the same Partial "
855 f"coalesce group are on different devices "
856 f"({buf_device} vs {chunk.device}). All parameters "
857 f"sharing the same DeviceMesh must reside on the "
858 f"same local device."
859 )
861 buf = torch.cat(chunks)
863 for pdim, reduce_op, needs_avg in partial_info:
864 group = mesh.get_group(pdim)
865 buf = _all_reduce_offloaded(buf, reduce_op, group)
866 if needs_avg:
867 buf /= dist.get_world_size(group=group)
869 # Extract views for params with actual gradients.
870 offset = 0
871 for i, idx in enumerate(active_indices):
872 numel = chunk_sizes[i]
873 if has_grad[i]:
874 reduced[idx] = buf[offset:offset + numel]
875 offset += numel
877 return reduced
880def _build_grad_groups( # pylint: disable=R0914
881 params: List[torch.Tensor],
882 parameter_layouts: Optional[Mapping[torch.nn.Parameter, "Layout"]] = None,
883) -> Tuple[
884 Dict[_GradGroupKey, List[torch.Tensor]],
885 List[torch.Tensor],
886 List[torch.Tensor],
887 List[_GradGroupKey],
888 Dict[int, object],
889 torch.device,
890 bool,
891]:
892 """Classify parameters into grad groups and pre-reduce Partial grads.
894 Group structure is derived from *parameter* DTensorSpecs (always
895 present on every rank) rather than gradients (which may be ``None``
896 on some ranks). This ensures every rank enters the same set of
897 collectives, preventing deadlocks (aligned with FSDP1 where all
898 ranks unconditionally execute the same all-reduce path).
900 Partial gradients are reduced via a **coalesced** all-reduce
901 (see ``_coalesce_partial_reduce``), turning O(N) per-parameter
902 collectives into O(G) where G is the number of distinct
903 ``(mesh, partial_info)`` groups (typically 1 for TP+FSDP).
905 Returns
906 ``(grad_groups, all_grads, norm_grads, key_per_grad, mesh_cache,
907 device, has_dtensor_grad)``. ``grad_groups`` maps each
908 ``(mesh_id, shard_dims)`` key to its grads; ``all_grads`` is the
909 flat list of raw local grads (global parameter order) scaled
910 in-place by the clip step; ``norm_grads`` is parallel to
911 ``all_grads`` but holds the Partial-reduced view for Partial grads
912 (the value whose norm is taken on the finite-p path);
913 ``key_per_grad`` is parallel to both and maps each grad back to its
914 group key.
915 """
916 # --- Phase 1: classify all parameters ---
917 param_infos: List[Tuple] = []
918 mesh_cache: Dict[int, object] = {}
919 device: Optional[torch.device] = None
921 for param in params:
922 if parameter_layouts is None:
923 mesh, shard_dims, partial_info = _get_param_mesh_info(param)
924 else:
925 layout = parameter_layouts.get(param)
926 if layout is None and not _is_dtensor(param) and not _is_dtensor(_get_grad_obj(param)):
927 raise ValueError("Explicit layouts must cover every local parameter participating in clipping")
928 mesh, shard_dims, partial_info = _get_param_mesh_info(param, layout)
929 key: _GradGroupKey = (
930 id(mesh) if mesh is not None else None, shard_dims,
931 )
932 if mesh is not None:
933 mesh_cache[id(mesh)] = mesh
934 if device is None:
935 device = _param_device(param)
936 local_grad = _get_local_grad(param)
937 param_infos.append(
938 (param, local_grad, mesh, partial_info, key),
939 )
941 if device is None:
942 device = torch.device("cpu")
944 # --- Phase 2: coalesced Partial reduction (O(N) → O(G)) ---
945 reduced = _coalesce_partial_reduce(param_infos, mesh_cache)
947 # --- Phase 3: build grad_groups ---
948 grad_groups: Dict[_GradGroupKey, List[torch.Tensor]] = defaultdict(
949 list,
950 )
951 all_grads: List[torch.Tensor] = []
952 # norm_grads / key_per_grad are parallel to all_grads (see Returns): the
953 # norm-input view (Partial-reduced where coalesced, else raw local) and
954 # the group key per grad, kept in global parameter order.
955 norm_grads: List[torch.Tensor] = []
956 key_per_grad: List[_GradGroupKey] = []
957 has_dtensor_grad = False
959 for idx, info in enumerate(param_infos):
960 param, local_grad, key = info[0], info[1], info[4]
961 if local_grad is None:
962 # Ensure the key exists so the Shard norm all-reduce is
963 # entered even when this rank has no grads for the group.
964 if key not in grad_groups:
965 grad_groups[key] = []
966 continue
968 grad_obj = _get_grad_obj(param)
969 if _is_dtensor(grad_obj):
970 has_dtensor_grad = True
971 all_grads.append(local_grad)
972 key_per_grad.append(key)
973 if idx in reduced:
974 grad_groups[key].append(reduced[idx])
975 norm_grads.append(reduced[idx])
976 else:
977 grad_groups[key].append(local_grad)
978 norm_grads.append(local_grad)
980 return _GradGroups(
981 grad_groups, all_grads, norm_grads, key_per_grad,
982 mesh_cache, device, has_dtensor_grad,
983 )
986def _clip_grads_with_norm_(
987 all_grads: List[torch.Tensor],
988 max_norm: float,
989 total_norm: torch.Tensor,
990 foreach: Optional[bool] = None,
991) -> None:
992 """Scale gradients in-place so the total norm <= *max_norm*."""
993 clip_coef = max_norm / (total_norm + 1e-6)
994 clip_coef_clamped = torch.clamp(clip_coef, max=1.0)
996 if _group_tensors_by_device_and_dtype is not None:
997 grouped_grads = _group_tensors_by_device_and_dtype(
998 [all_grads],
999 )
1000 for (device, dtype), ([device_grads], _) in grouped_grads.items():
1001 use_foreach = (
1002 foreach is None and _has_foreach_support(device_grads, device)
1003 ) or (foreach and _device_has_foreach_support(device))
1004 if use_foreach:
1005 torch._foreach_mul_( # pylint: disable=W0212
1006 device_grads,
1007 clip_coef_clamped.to(device=device, dtype=dtype),
1008 )
1009 elif foreach:
1010 raise RuntimeError(
1011 f"foreach=True was passed, but can't use the "
1012 f"foreach API on {device.type} tensors"
1013 )
1014 else:
1015 clip_coef_clamped_cast = clip_coef_clamped.to(device=device, dtype=dtype)
1016 for g in device_grads:
1017 g.mul_(clip_coef_clamped_cast)
1018 else:
1019 # Fallback when _foreach_utils is unavailable.
1020 if foreach:
1021 raise RuntimeError(
1022 "foreach=True was passed, but "
1023 "torch.utils._foreach_utils is not available"
1024 )
1025 for grad in all_grads:
1026 grad.mul_(clip_coef_clamped.to(grad.device, grad.dtype))
1029# ---------------------------------------------------------------------------
1030# Public API
1031# ---------------------------------------------------------------------------
1033@torch.no_grad()
1034def clip_grad_norm_(
1035 parameters: Union[
1036 "torch.nn.Module", torch.Tensor, Iterable[torch.Tensor],
1037 ],
1038 max_norm: float,
1039 norm_type: float = 2.0,
1040 error_if_nonfinite: bool = False,
1041 foreach: Optional[bool] = None,
1042 *,
1043 parameter_layouts: Optional[Mapping[torch.nn.Parameter, "Layout"]] = None,
1044) -> torch.Tensor:
1045 """Compute and clip gradient norm for distributed models.
1047 Drop-in replacement for the standard ``clip_grad_norm_`` that
1048 correctly handles DTensor-sharded parameters by deriving
1049 communication from each parameter's DTensorSpec.
1051 .. warning:: This function uses collective communications. It
1052 **must be called on all ranks** to avoid deadlocks. Aligned
1053 with FSDP1: every rank participates in the same collectives
1054 regardless of local gradient availability.
1056 Communication is derived from each parameter's DTensorSpec:
1058 * ``Shard`` on mesh dim *d* -- all-reduce norm statistics
1059 across ``device_mesh.get_group(d)``
1060 * ``Partial`` on mesh dim *d* -- all-reduce gradient values
1061 using the placement's ``reduce_op`` before norm computation
1062 * ``Replicate`` / plain tensor -- no communication
1064 This covers FSDP, HSDP, TP+FSDP, and any combination expressible
1065 via DTensor placements. PP cross-stage norm aggregation is not
1066 yet handled (requires manual all-reduce across stages).
1068 Args:
1069 parameters: An ``nn.Module``, a single ``Tensor``, or an
1070 iterable of ``Tensor`` s whose gradients to clip.
1071 max_norm: Maximum allowed gradient norm.
1072 norm_type: Type of the norm (default ``2.0``).
1073 error_if_nonfinite: If ``True``, raise a ``RuntimeError``
1074 when the total norm is non-finite. Default ``False``.
1075 foreach: Use the faster foreach-based implementation for the
1076 gradient clipping step. If ``None``, use the foreach
1077 implementation for devices that support it and silently
1078 fall back to the per-tensor implementation for others.
1079 Default ``None``.
1080 parameter_layouts: Optional parameter-identity mapping to final HyperParallel
1081 Layouts for local tensors. DTensor gradient/parameter layouts take
1082 precedence and must agree with explicit metadata. Local trainable
1083 parameters must be fully covered; frozen parameters are ignored.
1085 Returns:
1086 The total (unclipped) gradient norm as a scalar tensor,
1087 cast to the promoted dtype of all gradient tensors.
1088 """
1089 max_norm = float(max_norm)
1090 norm_type = float(norm_type)
1092 params = _normalize_parameters(parameters)
1093 if parameter_layouts is not None:
1094 params = [param for param in dict.fromkeys(params) if param.requires_grad]
1095 if foreach is None:
1096 foreach = False
1098 (
1099 grad_groups, all_grads, norm_grads, key_per_grad,
1100 mesh_cache, device, has_dtensor_grad,
1101 ) = (_build_grad_groups(params) if parameter_layouts is None
1102 else _build_grad_groups(params, parameter_layouts))
1104 # -- Norm + clip (all ranks participate) --------------------------------
1105 # _compute_local_norm returns identity elements for empty groups,
1106 # so the subsequent all-reduce is safe and semantically neutral.
1107 total_norm = _get_total_norm(
1108 grad_groups, norm_type, mesh_cache, device,
1109 norm_grads, key_per_grad,
1110 )
1112 if error_if_nonfinite and torch.logical_or(
1113 total_norm.isnan(), total_norm.isinf()
1114 ):
1115 raise RuntimeError(
1116 f"The total norm of order {norm_type} for gradients from "
1117 "`parameters` is non-finite, so it cannot be clipped. To "
1118 "disable this error and scale the gradients by the "
1119 "non-finite norm anyway, set "
1120 "`error_if_nonfinite=False`"
1121 )
1123 if all_grads:
1124 # Disable foreach for dtensor-backed grads to avoid dispatch issues.
1125 effective_foreach = False if has_dtensor_grad and foreach is None else foreach
1126 _clip_grads_with_norm_(
1127 all_grads, max_norm, total_norm, effective_foreach,
1128 )
1130 # Promote return dtype to match gradient dtypes (FSDP1 convention).
1131 # When this rank has no gradients, return in the default FP32 dtype
1132 # (same as FSDP1's behavior to avoid extra communication).
1133 if not all_grads:
1134 warnings.warn(
1135 "clip_grad_norm_ called on this rank with no gradients -- "
1136 "returning the local norm in the default dtype "
1137 f"{total_norm.dtype}",
1138 stacklevel=2,
1139 )
1140 return total_norm
1142 total_norm_dtype = functools.reduce(
1143 torch.promote_types,
1144 [g.dtype for g in all_grads],
1145 )
1146 # Return global all-reduced norm, consistent with torchtitan's
1147 # full_tensor() approach — .item() returns the correct global value.
1148 return total_norm.to(total_norm_dtype)