Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / utils / clip_grad.py: 78%

443 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Distributed-aware gradient clipping for parallel training. 

16 

17Communication is driven by each parameter's DTensorSpec (device_mesh + 

18placements) rather than any specific parallelism strategy, so a single 

19implementation covers FSDP, HSDP, TP+FSDP, and other DTensor-expressed 

20parallelisms. 

21 

22Collective safety aligned with FSDP1; numerical precision aligned with 

23FSDP2's ``_NormPartial`` norm computation path: 

24 

25* Gradient norms from sharded parameters are all-reduced across the 

26 corresponding shard process group. 

27* Non-sharded / replicated norms contribute locally without communication. 

28* **All ranks participate in the same collectives** regardless of local 

29 gradient availability, preventing collective-misalignment deadlocks. 

30 

31The finite p-norm is bit-exact with upstream only for the pure-FSDP, 

32single-dtype case; mixed sharded + replicated is mathematically correct 

33but intentionally diverges (see ``_total_norm_fsdp2_aligned``). 

34 

35Note: PP does not use DTensor layout for gradients today. Cross-stage 

36norm aggregation will require an additional manual all-reduce and is 

37left for future work. 

38""" 

39# pylint: disable=not-callable 

40import functools 

41import math 

42import warnings 

43from collections import defaultdict, namedtuple 

44from collections.abc import Mapping 

45from dataclasses import dataclass 

46from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union 

47 

48import torch 

49import torch.distributed as dist 

50 

51from hyper_parallel.core.dtensor.placement_types import Partial 

52 

53if TYPE_CHECKING: 

54 from hyper_parallel.core.dtensor.layout import Layout 

55 

56try: 

57 from torch.utils._foreach_utils import ( 

58 _device_has_foreach_support, 

59 _group_tensors_by_device_and_dtype, 

60 _has_foreach_support, 

61 ) 

62except ImportError: 

63 _device_has_foreach_support = None # type: ignore[assignment] 

64 _group_tensors_by_device_and_dtype = None # type: ignore[assignment] 

65 _has_foreach_support = None # type: ignore[assignment] 

66 

67__all__: list[str] = ["clip_grad_norm_"] 

68 

69 

70@dataclass(frozen=True) 

71class NormShardPlan: 

72 """One norm contributor and its replica ranks for each dense parameter.""" 

73 

74 representatives: dict[str, bool] 

75 replica_ranks: dict[str, tuple[int, ...]] 

76 

77 

78def _norm_coverage(param: torch.nn.Parameter) -> tuple[tuple[int, ...], tuple[tuple[int, int], ...]]: 

79 """Describe the exact global rectangle represented by a local parameter.""" 

80 global_shape = tuple(param.shape) 

81 layout = getattr(param, "layout", None) if _is_dtensor(param) else None 

82 if layout is None: 

83 area = tuple((0, size) for size in global_shape) 

84 else: 

85 from hyper_parallel.core.dtensor.layout import infer_slice_area_by_layout # pylint: disable=C0415 

86 

87 rank = dist.get_rank() if dist.is_initialized() else 0 

88 if rank not in layout.rank_list: 

89 raise ValueError("Norm layout does not contain the current global rank") 

90 area = infer_slice_area_by_layout( 

91 layout, layout.rank_list.index(rank), global_shape, 

92 ) 

93 local = param._local_tensor if _is_dtensor(param) else param # pylint: disable=W0212 

94 if tuple(end - start for start, end in area) != tuple(local.shape): 

95 raise ValueError("Norm layout cannot describe the exact local parameter coverage") 

96 return global_shape, tuple((int(start), int(end)) for start, end in area) 

97 

98 

99def _norm_representative(fqn: str, shape: tuple[int, ...], 

100 stage_data: dict[int, dict], rank: int, 

101 local_area: tuple) -> tuple[bool, tuple[int, ...]]: 

102 """Validate disjoint shard coverage and identify copies of this area.""" 

103 representatives = {} 

104 for member, data in stage_data.items(): 

105 if data[fqn][0] != shape: 

106 raise ValueError(f"Norm shard global shape mismatch: {fqn}") 

107 area = data[fqn][1] 

108 representatives.setdefault(area, []).append(member) 

109 nonempty = [area for area in representatives if all(end > start for start, end in area)] 

110 volume = sum(math.prod(end - start for start, end in area) for area in nonempty) 

111 if volume != math.prod(shape): 

112 raise ValueError(f"Norm shard coverage is incomplete or duplicated: {fqn}") 

113 for index, left in enumerate(nonempty): 

114 for right in nonempty[index + 1:]: 

115 if all(max(a[0], b[0]) < min(a[1], b[1]) for a, b in zip(left, right)): 

116 raise ValueError(f"Norm shard rectangles overlap: {fqn}") 

117 copies = tuple(representatives[local_area]) 

118 return rank == min(copies), copies 

119 

120 

121def build_norm_shard_plan(model: torch.nn.Module, excluded_params: Iterable[torch.nn.Parameter], 

122 mesh_context: Any) -> NormShardPlan: 

123 """Choose one rank per identical global shard and verify full coverage. 

124 

125 Args: 

126 model: Model with final FSDP parameter layouts. 

127 excluded_params: Parameters managed by an external optimizer. 

128 mesh_context: Stage-local mesh and topology metadata. 

129 """ 

130 excluded_ids = {id(param) for param in excluded_params} 

131 local = { 

132 fqn: _norm_coverage(param) 

133 for fqn, param in model.named_parameters() 

134 if param.requires_grad and id(param) not in excluded_ids 

135 } 

136 if dist.is_initialized(): 

137 gathered = [None] * dist.get_world_size() 

138 dist.all_gather_object(gathered, local) 

139 rank = dist.get_rank() 

140 else: 

141 gathered = [local] 

142 rank = 0 

143 mesh = getattr(mesh_context, "device_mesh", None) 

144 stage_ranks = (tuple(mesh.rank_list) if mesh is not None and hasattr(mesh, "rank_list") 

145 else tuple(range(len(gathered)))) 

146 stage_data = {member: gathered[member] for member in stage_ranks} 

147 reference_names = set(local) 

148 if any(set(data) != reference_names for data in stage_data.values()): 

149 raise ValueError("Norm shard plan has different dense FQNs within one PP stage") 

150 representatives = {} 

151 replica_ranks = {} 

152 for fqn in sorted(reference_names): 

153 representatives[fqn], replica_ranks[fqn] = _norm_representative( 

154 fqn, local[fqn][0], stage_data, rank, local[fqn][1], 

155 ) 

156 return NormShardPlan(representatives, replica_ranks) 

157 

158 

159def _effective_dense_grads(parameters: list[tuple[str, torch.nn.Parameter]]) -> tuple[list, dict]: 

160 """Collect local gradients and reduce partial placements once.""" 

161 infos = [] 

162 mesh_cache = {} 

163 for _, param in parameters: 

164 mesh, shard_dims, partial_info = _get_param_mesh_info(param) 

165 if mesh is not None: 

166 mesh_cache[id(mesh)] = mesh 

167 infos.append((param, _get_local_grad(param), mesh, partial_info, 

168 (id(mesh) if mesh is not None else None, shard_dims))) 

169 return infos, _coalesce_partial_reduce(infos, mesh_cache) 

170 

171 

172def _check_dense_grad_presence(present: dict[str, bool], replica_ranks: dict[str, tuple[int, ...]]) -> None: 

173 """Reject mismatched gradient presence on copies of the same shard.""" 

174 if not dist.is_initialized(): 

175 return 

176 gathered = [None] * dist.get_world_size() 

177 dist.all_gather_object(gathered, present) 

178 for fqn, ranks in replica_ranks.items(): 

179 copies = [gathered[member][fqn] for member in ranks] 

180 if any(value != copies[0] for value in copies[1:]): 

181 raise ValueError(f"Norm shard gradient presence differs across replicas: {fqn}") 

182 

183 

184def dense_grad_norm_sq_and_refs(model: torch.nn.Module, 

185 excluded_params: Iterable[torch.nn.Parameter], 

186 norm_shard_plan: NormShardPlan, 

187 mesh_context: Any) -> tuple[torch.Tensor, list[torch.Tensor]]: 

188 """Sum each dense global shard once and return mutable effective gradients. 

189 

190 Args: 

191 model: Model with final FSDP parameter layouts. 

192 excluded_params: Parameters managed by an external optimizer. 

193 norm_shard_plan: Validated coverage and one representative per shard. 

194 mesh_context: Stage-local mesh and topology metadata. 

195 """ 

196 excluded_ids = {id(param) for param in excluded_params} 

197 parameters = [ 

198 (fqn, param) for fqn, param in model.named_parameters() 

199 if param.requires_grad and id(param) not in excluded_ids 

200 ] 

201 device = next(((_param_device(param)) for _, param in parameters), None) 

202 if device is None: 

203 mesh = getattr(mesh_context, "device_mesh", None) 

204 device = torch.device(getattr(mesh, "device_type", "cpu")) 

205 infos, reduced = _effective_dense_grads(parameters) 

206 refs = [] 

207 local_sq = torch.zeros((), device=device, dtype=torch.float32) 

208 present = {} 

209 for index, (fqn, _) in enumerate(parameters): 

210 grad = infos[index][1] 

211 present[fqn] = grad is not None 

212 if grad is None: 

213 continue 

214 refs.append(grad) 

215 if norm_shard_plan.representatives[fqn]: 

216 view = reduced.get(index, grad) 

217 local_sq.add_(view.float().square().sum().to(device)) 

218 _check_dense_grad_presence(present, norm_shard_plan.replica_ranks) 

219 for name in getattr(getattr(mesh_context, "device_mesh", None), "mesh_dim_names", ()) or (): 

220 axis = mesh_context.device_mesh[name] 

221 if axis.size() > 1: 

222 dist.all_reduce(local_sq, op=dist.ReduceOp.SUM, group=axis.get_group()) 

223 return local_sq, refs 

224 

225 

226def scale_grad_refs_(refs: Iterable[torch.Tensor], coefficient: torch.Tensor) -> None: 

227 """Scale the exact dense gradients that the optimizer will consume. 

228 

229 Args: 

230 refs: Mutable dense effective gradients. 

231 coefficient: Shared clipping coefficient. 

232 """ 

233 for grad in refs: 

234 grad.mul_(coefficient) 

235 

236 

237# (id(mesh) or None, shard_dims) -> list of local grads for norm computation 

238_GradGroupKey = Tuple[Optional[int], Tuple[int, ...]] 

239 

240# (mesh_dim_index, dist.ReduceOp, needs_manual_avg) 

241_PartialReduceInfo = Tuple[int, "dist.ReduceOp", bool] 

242 

243# Result of _build_grad_groups; tuple-unpacking compatible with prior 7-tuple. 

244_GradGroups = namedtuple( 

245 "_GradGroups", 

246 "grad_groups all_grads norm_grads key_per_grad mesh_cache device " 

247 "has_dtensor_grad", 

248) 

249 

250 

251def _is_dtensor(value: object) -> bool: 

252 """Return whether *value* is a HyperParallel DTensor. 

253 

254 The DTensor module imports ``core.utils`` during initialization, so keep 

255 this import lazy to avoid a package initialization cycle. 

256 """ 

257 from hyper_parallel.core.dtensor.dtensor import DTensor # pylint: disable=C0415 

258 

259 return isinstance(value, DTensor) 

260 

261 

262# --------------------------------------------------------------------------- 

263# Reduce-op mapping 

264# --------------------------------------------------------------------------- 

265 

266_REDUCE_OP_AVG_SUPPORTED = hasattr(dist.ReduceOp, "AVG") 

267 

268_STR_TO_REDUCE_OP: Dict[str, "dist.ReduceOp"] = { 

269 "sum": dist.ReduceOp.SUM, 

270 "max": dist.ReduceOp.MAX, 

271 "min": dist.ReduceOp.MIN, 

272} 

273if _REDUCE_OP_AVG_SUPPORTED: 

274 _STR_TO_REDUCE_OP["avg"] = dist.ReduceOp.AVG 

275 

276 

277def _str_to_reduce_op(op_str: str) -> Tuple["dist.ReduceOp", bool]: 

278 """Map a ``Partial`` placement's *reduce_op* string to ``dist.ReduceOp``. 

279 

280 Returns ``(reduce_op, needs_manual_avg)`` where *needs_manual_avg* 

281 is ``True`` when ``"avg"`` is requested but the backend does not 

282 support ``dist.ReduceOp.AVG`` — the caller should use SUM and 

283 manually divide by the group size. 

284 """ 

285 lower = op_str.lower() 

286 if lower == "avg" and not _REDUCE_OP_AVG_SUPPORTED: 

287 return dist.ReduceOp.SUM, True 

288 op = _STR_TO_REDUCE_OP.get(lower) 

289 if op is None: 

290 raise ValueError( 

291 f"Unsupported Partial reduce_op: {op_str!r}. " 

292 f"Supported: {sorted(set(list(_STR_TO_REDUCE_OP) + ['avg']))}" 

293 ) 

294 return op, False 

295 

296 

297# --------------------------------------------------------------------------- 

298# Helpers 

299# --------------------------------------------------------------------------- 

300 

301def _normalize_parameters( 

302 parameters: Union["torch.nn.Module", torch.Tensor, Iterable[torch.Tensor]], 

303) -> List[torch.Tensor]: 

304 """Normalize *parameters* to a flat list of tensors. 

305 

306 * ``torch.nn.Module`` -> ``list(module.parameters())`` 

307 * single ``torch.Tensor`` -> ``[tensor]`` 

308 * iterable of tensors -> ``list(iterable)`` 

309 """ 

310 if isinstance(parameters, torch.nn.Module): 

311 return list(parameters.parameters()) 

312 if isinstance(parameters, torch.Tensor): 

313 return [parameters] 

314 return list(parameters) 

315 

316 

317def _param_device(param: torch.Tensor) -> torch.device: 

318 """Return the local device of *param* (unwrap DTensor if needed).""" 

319 if _is_dtensor(param): 

320 return param._local_tensor.device # pylint: disable=protected-access 

321 return param.device 

322 

323 

324def _all_reduce_offloaded( 

325 tensor: torch.Tensor, op: dist.ReduceOp, group: object, 

326) -> torch.Tensor: 

327 """Reduce CPU-offloaded values on the backend's communication device.""" 

328 original_device = tensor.device 

329 if original_device.type == "cpu": 

330 device_type = {"nccl": "cuda", "hccl": "npu"}.get(dist.get_backend(group)) 

331 if device_type is not None: 

332 device_module = torch.get_device_module(device_type) 

333 tensor = tensor.to(torch.device(device_type, device_module.current_device())) 

334 dist.all_reduce(tensor, op=op, group=group) 

335 return tensor.to(original_device) 

336 

337 

338def _get_grad_obj(param: torch.nn.Parameter) -> Optional[torch.Tensor]: 

339 """Return the gradient object for *param*. 

340 

341 Checks ``param.main_grad`` first (used when 

342 ``MixedPrecisionPolicy.apply_grad_on_fp32_main_grad=True``), 

343 falling back to ``param.grad``. 

344 """ 

345 grad = getattr(param, "main_grad", None) 

346 if grad is not None: 

347 return grad 

348 return param.grad 

349 

350 

351def _get_local_grad(param: torch.nn.Parameter) -> Optional[torch.Tensor]: 

352 """Return the local gradient tensor, or ``None`` if absent. 

353 

354 Supports ``main_grad`` for fp32 mixed-precision training. 

355 """ 

356 if not param.requires_grad: 

357 return None 

358 grad = _get_grad_obj(param) 

359 if grad is None: 

360 return None 

361 if _is_dtensor(grad): 

362 return grad._local_tensor # pylint: disable=protected-access 

363 return grad 

364 

365 

366def _get_param_mesh_info( 

367 param: torch.nn.Parameter, 

368 parameter_layout: Optional["Layout"] = None, 

369) -> Tuple[ 

370 Optional[object], 

371 Tuple[int, ...], 

372 Tuple[_PartialReduceInfo, ...], 

373]: 

374 """Derive DeviceMesh, Shard dims and Partial info from DTensorSpec. 

375 

376 Checks the *gradient's* spec first; falls back to the *parameter's* 

377 spec when the gradient is a plain tensor on a DTensor parameter 

378 (common after FSDP/HSDP backward where ``param.grad`` is stored as 

379 the local shard tensor). 

380 

381 Returns ``(mesh, shard_dims, partial_info)`` where *partial_info* 

382 is a tuple of ``(mesh_dim, dist.ReduceOp, needs_manual_avg)`` 

383 triples that respect the ``Partial`` placement's ``reduce_op`` 

384 attribute. *needs_manual_avg* is ``True`` when ``"avg"`` was 

385 requested but the backend lacks ``dist.ReduceOp.AVG`` support. 

386 """ 

387 grad = _get_grad_obj(param) 

388 # Prefer grad's spec (most accurate); fall back to param's. 

389 spec_source = grad if _is_dtensor(grad) else param 

390 if parameter_layout is None: 

391 if not _is_dtensor(spec_source): 

392 return None, (), () 

393 placements = spec_source.placements 

394 mesh = spec_source.device_mesh 

395 else: 

396 layout = spec_source.layout if _is_dtensor(spec_source) else None 

397 if layout is not None and ( 

398 layout.mesh_shape, layout.rank_list, layout.alias_name, layout.placements, layout.tensor_shape 

399 ) != ( 

400 parameter_layout.mesh_shape, parameter_layout.rank_list, parameter_layout.alias_name, 

401 parameter_layout.placements, parameter_layout.tensor_shape 

402 ): 

403 raise ValueError("Explicit parameter layout conflicts with its DTensor layout") 

404 layout = layout if layout is not None else parameter_layout 

405 placements = layout.placements 

406 mesh = layout.mesh 

407 

408 shard_dims = tuple( 

409 i for i, p in enumerate(placements) 

410 if p.is_shard() 

411 ) 

412 partial_info = tuple( 

413 (i, *_str_to_reduce_op(p.reduce_op)) 

414 for i, p in enumerate(placements) 

415 if isinstance(p, Partial) 

416 ) 

417 return mesh, shard_dims, partial_info 

418 

419 

420def _sum_p_norms( 

421 dev_grads: List[torch.Tensor], 

422 norm_type: float, 

423 device: torch.device, 

424 total: torch.Tensor, 

425) -> None: 

426 """Accumulate sum-of-p-th-powers for *dev_grads* into *total*.""" 

427 for g in dev_grads: 

428 n = torch.linalg.vector_norm(g, norm_type) 

429 total.add_(n.to(device=device) ** norm_type) 

430 

431 

432def _foreach_p_norms( 

433 grads: List[torch.Tensor], 

434 norm_type: float, 

435 device: torch.device, 

436) -> torch.Tensor: 

437 """Fast path: fuse per-tensor norms via ``_foreach_norm``. 

438 

439 Restricted to float32 tensors to preserve the same numerical 

440 precision as ``vector_norm(dtype=float32)``. Non-float32 tensors 

441 and backends that raise ``RuntimeError`` fall back to per-tensor 

442 ``vector_norm``. 

443 """ 

444 total = torch.tensor(0.0, device=device, dtype=torch.float32) 

445 grouped = _group_tensors_by_device_and_dtype( 

446 [[g.detach() for g in grads]], 

447 ) 

448 for (dev, _), ([dev_grads], _) in grouped.items(): 

449 if ( 

450 dev_grads[0].dtype == torch.float32 

451 and _has_foreach_support(dev_grads, dev) 

452 ): 

453 try: 

454 per_norms = torch._foreach_norm( # pylint: disable=W0212 

455 dev_grads, norm_type, 

456 ) 

457 except RuntimeError: 

458 per_norms = None 

459 if per_norms is not None: 

460 total.add_( 

461 torch.stack([ 

462 n.to(device=device) ** norm_type 

463 for n in per_norms 

464 ]).sum(), 

465 ) 

466 else: 

467 _sum_p_norms(dev_grads, norm_type, device, total) 

468 else: 

469 _sum_p_norms(dev_grads, norm_type, device, total) 

470 return total 

471 

472 

473def _per_tensor_norms( 

474 grads: List[torch.Tensor], 

475 norm_type: float, 

476 device: torch.device, 

477) -> List[torch.Tensor]: 

478 """Return per-tensor norms as a list of scalar tensors on *device*.""" 

479 if not grads: 

480 return [] 

481 

482 if _group_tensors_by_device_and_dtype is None or not hasattr(torch, "_foreach_norm"): 

483 return [ 

484 torch.linalg.vector_norm(g.detach(), norm_type).to(device=device) 

485 for g in grads 

486 ] 

487 

488 norms: List[torch.Tensor] = [] 

489 grouped = _group_tensors_by_device_and_dtype( 

490 [[g.detach() for g in grads]], 

491 ) 

492 for (dev, _), ([dev_grads], _) in grouped.items(): 

493 if dev_grads and _has_foreach_support(dev_grads, dev): 

494 try: 

495 per_norms = torch._foreach_norm( # pylint: disable=W0212 

496 dev_grads, norm_type, 

497 ) 

498 except RuntimeError: 

499 per_norms = None 

500 if per_norms is not None: 

501 norms.extend( 

502 [n.to(device=device) for n in per_norms], 

503 ) 

504 continue 

505 norms.extend([ 

506 torch.linalg.vector_norm(g, norm_type).to(device=device) 

507 for g in dev_grads 

508 ]) 

509 return norms 

510 

511 

512def _compute_local_norm( # pylint: disable=R0911 

513 grads: List[torch.Tensor], 

514 norm_type: float, 

515 device: torch.device, 

516) -> torch.Tensor: 

517 """Compute the combined norm of *grads* locally in FP32. 

518 

519 When *grads* is empty, returns the **identity element** for the 

520 subsequent all-reduce so that this rank contributes a neutral value 

521 (aligned with FSDP1's ``_zero_scalar`` approach): 

522 

523 * ``inf`` -> 0 (neutral for MAX; norms are non-negative) 

524 * ``-inf`` -> +inf (neutral for MIN) 

525 * ``0`` -> 0 (neutral for SUM) 

526 * finite -> 0 (neutral for SUM) 

527 """ 

528 if not grads: 

529 if norm_type == -math.inf: 

530 return torch.tensor( 

531 float("inf"), device=device, dtype=torch.float32, 

532 ) 

533 return torch.tensor(0.0, device=device, dtype=torch.float32) 

534 

535 if norm_type == math.inf: 

536 norms = [ 

537 torch.linalg.vector_norm(g.detach(), math.inf) 

538 for g in grads 

539 ] 

540 return torch.stack(norms).max().to(device) 

541 

542 if norm_type == -math.inf: 

543 norms = [ 

544 torch.linalg.vector_norm(g.detach(), -math.inf) 

545 for g in grads 

546 ] 

547 return torch.stack(norms).min().to(device) 

548 

549 if norm_type == 0: 

550 norms = [ 

551 torch.linalg.vector_norm(g.detach(), 0) 

552 for g in grads 

553 ] 

554 return torch.stack(norms).sum().to(device) 

555 

556 # Finite p-norm: return sum of p-th powers. 

557 if ( 

558 len(grads) > 1 

559 and _group_tensors_by_device_and_dtype is not None 

560 and hasattr(torch, "_foreach_norm") 

561 ): 

562 return _foreach_p_norms(grads, norm_type, device) 

563 

564 # Scalar fallback when foreach utilities are unavailable. 

565 norms = [ 

566 torch.linalg.vector_norm(g.detach(), norm_type) 

567 for g in grads 

568 ] 

569 norm_powers = [n.to(device=device) ** norm_type for n in norms] 

570 return torch.stack(norm_powers).sum() 

571 

572 

573# --------------------------------------------------------------------------- 

574# Total norm aggregation with collectives 

575# --------------------------------------------------------------------------- 

576 

577def _get_total_norm( 

578 grad_groups: Dict[_GradGroupKey, List[torch.Tensor]], 

579 norm_type: float, 

580 mesh_cache: Dict[int, object], 

581 device: torch.device, 

582 norm_grads: List[torch.Tensor], 

583 key_per_grad: List[_GradGroupKey], 

584) -> torch.Tensor: 

585 """Compute total gradient norm with per-group all-reduce. 

586 

587 ``norm_grads`` (parallel to ``key_per_grad``) holds the tensor whose 

588 norm to take per parameter; only the finite p-norm path consumes it. 

589 """ 

590 if norm_type == math.inf: 

591 return _total_norm_inf( 

592 grad_groups, norm_type, mesh_cache, device, 

593 dist.ReduceOp.MAX, 

594 ) 

595 

596 if norm_type == -math.inf: 

597 return _total_norm_inf( 

598 grad_groups, norm_type, mesh_cache, device, 

599 dist.ReduceOp.MIN, 

600 ) 

601 

602 if norm_type == 0: 

603 return _total_norm_sum( 

604 grad_groups, norm_type, mesh_cache, device, 

605 ) 

606 

607 # Finite p-norm: FSDP2-aligned sequence. 

608 total_p = _total_norm_fsdp2_aligned( 

609 grad_groups, norm_type, mesh_cache, device, 

610 norm_grads, key_per_grad, 

611 ) 

612 return total_p ** (1.0 / norm_type) 

613 

614 

615def _total_norm_inf( # pylint: disable=R0913,R0917 

616 grad_groups, norm_type, mesh_cache, device, reduce_op, 

617): 

618 """Shared logic for inf / -inf norms.""" 

619 group_norms: List[torch.Tensor] = [] 

620 for (mesh_id, shard_dims), grads in grad_groups.items(): 

621 local_norm = _compute_local_norm(grads, norm_type, device) 

622 if mesh_id is not None: 

623 mesh = mesh_cache[mesh_id] 

624 for dim in shard_dims: 

625 local_norm = _all_reduce_offloaded(local_norm, reduce_op, mesh.get_group(dim)) 

626 group_norms.append(local_norm) 

627 if not group_norms: 

628 if norm_type == -math.inf: 

629 return torch.tensor(float("inf"), device=device) 

630 return torch.tensor(0.0, device=device) 

631 stacked = torch.stack(group_norms) 

632 return stacked.max() if reduce_op == dist.ReduceOp.MAX else stacked.min() 

633 

634 

635def _total_norm_sum(grad_groups, norm_type, mesh_cache, device): 

636 """Shared logic for finite norms and L0 (all use SUM all-reduce).""" 

637 total = torch.tensor(0.0, device=device) 

638 for (mesh_id, shard_dims), grads in grad_groups.items(): 

639 local_val = _compute_local_norm(grads, norm_type, device) 

640 if mesh_id is not None: 

641 mesh = mesh_cache[mesh_id] 

642 for dim in shard_dims: 

643 local_val = _all_reduce_offloaded(local_val, dist.ReduceOp.SUM, mesh.get_group(dim)) 

644 total.add_(local_val) 

645 return total 

646 

647 

648def _reduction_signature(grad_groups, mesh_cache): 

649 """Bucket grad-group keys by the *process group(s)* they reduce over. 

650 

651 Two keys that reduce over the same set of process groups (same global 

652 ranks per shard dim) must be **pooled** so their norms accumulate in 

653 one stack -- matching FSDP2's single foreach-norm + single reduce and 

654 keeping the loss bit-exact even when params live on several distinct 

655 ``DeviceMesh`` objects that share the same DP process group (common 

656 for multi-component models). Keys that reduce over *different* 

657 process groups (TP+FSDP heterogeneous sharding, expert parallel) get 

658 separate buckets, each reduced over its own group. 

659 

660 Returns ``(key_to_sig, sig_groups, sig_order)``: 

661 

662 * ``key_to_sig`` -- ``key -> signature`` (hashable; ``()`` = replicate 

663 / no communication). 

664 * ``sig_groups`` -- ``signature -> list[ProcessGroup]`` to all-reduce. 

665 * ``sig_order`` -- signatures in first-seen (parameter) order, so all 

666 ranks issue the same collectives in the same order. 

667 """ 

668 key_to_sig: Dict[_GradGroupKey, Tuple] = {} 

669 sig_groups: Dict[Tuple, List[object]] = {} 

670 sig_order: List[Tuple] = [] 

671 for mesh_id, shard_dims in grad_groups: 

672 if shard_dims and mesh_id is not None: 

673 mesh = mesh_cache[mesh_id] 

674 groups = [mesh.get_group(dim) for dim in shard_dims] 

675 sig = tuple( 

676 tuple(dist.get_process_group_ranks(group)) for group in groups 

677 ) 

678 else: 

679 groups = [] 

680 sig = () 

681 key_to_sig[(mesh_id, shard_dims)] = sig 

682 if sig not in sig_groups: 

683 sig_groups[sig] = groups 

684 sig_order.append(sig) 

685 return key_to_sig, sig_groups, sig_order 

686 

687 

688def _total_norm_fsdp2_aligned(grad_groups, norm_type, mesh_cache, device, 

689 norm_grads, key_per_grad): 

690 """FSDP2-aligned norm for finite p-norms. 

691 

692 Grads are bucketed by the *process group* they reduce over (see 

693 :func:`_reduction_signature`); ``norm_grads`` supplies, in global 

694 parameter order, the Partial-reduced (already-global) view for Partial 

695 grads and the raw local grad otherwise. Each bucket does ONE ``stack`` 

696 → ONE ``vector_norm`` → ``^p`` → ONE ``all_reduce SUM`` per shard dim, 

697 and the buckets are summed locally. 

698 

699 Reduction rules: 

700 

701 * Sharded bucket -- reduce over its own group(s). Same-group params 

702 across distinct ``DeviceMesh`` objects pool into one reduce; distinct 

703 groups (TP+FSDP heterogeneous, expert parallel) stay separate -- the 

704 per-group convention also used by :func:`_total_norm_inf` / 

705 :func:`_total_norm_sum` and by VeOmni / torchtitan. 

706 * Replicate bucket (signature ``()``) -- contribute norm² locally with 

707 NO communication; reducing it over the shard group would over-count 

708 it ``shard_world_size`` times (the FSDP1 convention). 

709 

710 Empty-but-present buckets still issue their all_reduce (identity ``0`` 

711 for SUM) so all ranks run the same collectives. 

712 

713 Bit-exact with upstream ``torch.nn.utils.clip_grad_norm_`` only for the 

714 pure-FSDP, single-bucket, single-dtype case. Mixed shard + replicate is 

715 mathematically correct but intentionally diverges (upstream folds the 

716 replicate norm into one ``_NormPartial`` reduce and over-counts it); 

717 mixed-dtype follows upstream's :func:`_per_tensor_norms` device/dtype 

718 regrouping rather than strict global order. 

719 

720 Returns the global sum of p-th powers (caller takes p-th root). 

721 """ 

722 key_to_sig, sig_groups, sig_order = _reduction_signature( 

723 grad_groups, mesh_cache, 

724 ) 

725 

726 # Bucket norms by reduction signature, preserving global parameter order. 

727 sig_grads: Dict[Tuple, List[torch.Tensor]] = {sig: [] for sig in sig_order} 

728 for grad, key in zip(norm_grads, key_per_grad): 

729 sig_grads[key_to_sig[key]].append(grad) 

730 

731 total_p = torch.tensor(0.0, device=device, dtype=torch.float32) 

732 for sig in sig_order: 

733 grads = sig_grads[sig] 

734 if grads: 

735 norms = _per_tensor_norms(grads, norm_type, device) 

736 local_p = torch.linalg.vector_norm( 

737 torch.stack(norms).to(torch.float32), norm_type, 

738 ) ** norm_type 

739 else: 

740 local_p = torch.tensor(0.0, device=device, dtype=torch.float32) 

741 

742 for group in sig_groups[sig]: 

743 local_p = _all_reduce_offloaded(local_p, dist.ReduceOp.SUM, group) 

744 

745 total_p = total_p + local_p 

746 

747 return total_p 

748 

749 

750def _build_coalesce_buffer( 

751 param_infos: List[Tuple], 

752 indices: List[int], 

753) -> Tuple[List[torch.Tensor], List[int], List[bool], List[int]]: 

754 """Build flat fp32 chunks for one coalesce group. 

755 

756 Returns ``(chunks, chunk_sizes, has_grad, active_indices)``. 

757 Frozen params are skipped; trainable grad-free params contribute 

758 zeros so the collective matches ranks that have a grad. 

759 """ 

760 chunks: List[torch.Tensor] = [] 

761 chunk_sizes: List[int] = [] 

762 has_grad: List[bool] = [] 

763 active_indices: List[int] = [] 

764 

765 for idx in indices: 

766 param = param_infos[idx][0] 

767 local_grad = param_infos[idx][1] 

768 if local_grad is not None: 

769 chunks.append( 

770 local_grad.detach().reshape(-1).to(torch.float32), 

771 ) 

772 chunk_sizes.append(local_grad.numel()) 

773 has_grad.append(True) 

774 active_indices.append(idx) 

775 elif param.requires_grad: 

776 local_p = ( 

777 param._local_tensor # pylint: disable=W0212 

778 if _is_dtensor(param) else param.data 

779 ) 

780 numel = local_p.numel() 

781 chunks.append( 

782 torch.zeros( 

783 numel, device=local_p.device, 

784 dtype=torch.float32, 

785 ), 

786 ) 

787 chunk_sizes.append(numel) 

788 has_grad.append(False) 

789 active_indices.append(idx) 

790 

791 return chunks, chunk_sizes, has_grad, active_indices 

792 

793 

794def _coalesce_partial_reduce( # pylint: disable=R0914 

795 param_infos: List[Tuple], 

796 mesh_cache: Dict[int, object], 

797) -> Dict[int, torch.Tensor]: 

798 """Coalesce Partial all-reduces: O(N) collectives → O(G). 

799 

800 Groups parameters sharing the same ``(mesh, partial_info)`` and 

801 flattens their gradients (or zeros for trainable grad-free params) 

802 into a single fp32 buffer. **One** ``all_reduce`` per buffer 

803 replaces the previous per-parameter collective calls. 

804 

805 For TP+FSDP (all params share the same mesh / placements), this 

806 turns ~200 individual all-reduces into 1 — saving 10-20 ms per 

807 training step at typical HCCS/NCCL latencies. 

808 

809 Frozen params (``requires_grad=False``) are consistently grad-free 

810 across all ranks and are excluded from the buffer to avoid wasting 

811 bandwidth. 

812 

813 All buffers use float32 to guarantee dtype consistency across ranks 

814 in mixed-precision training (grad may be fp16/bf16 while param is 

815 fp32). 

816 

817 Returns a dict mapping *param_infos* index → reduced gradient view 

818 (1-D fp32 slice of the coalesced buffer). Only entries for params 

819 with actual gradients are included. 

820 """ 

821 # Group by Partial coalesce key: (mesh_id, partial_info) 

822 coalesce_groups: Dict[ 

823 Tuple, List[int], 

824 ] = defaultdict(list) 

825 for idx, info in enumerate(param_infos): 

826 mesh, partial_info = info[2], info[3] 

827 if partial_info: 

828 if mesh is None: 

829 raise RuntimeError( 

830 "clip_grad_norm_: parameter has Partial placements " 

831 "but no DeviceMesh. This is a DTensor invariant " 

832 "violation." 

833 ) 

834 pck = (id(mesh), partial_info) 

835 coalesce_groups[pck].append(idx) 

836 

837 reduced: Dict[int, torch.Tensor] = {} 

838 

839 for (mesh_id, partial_info), indices in coalesce_groups.items(): 

840 mesh = mesh_cache[mesh_id] 

841 chunks, chunk_sizes, has_grad, active_indices = ( 

842 _build_coalesce_buffer(param_infos, indices) 

843 ) 

844 

845 if not chunks: 

846 continue # all params frozen, no collective needed 

847 

848 # Sanity check: same mesh → same device. Fail fast on 

849 # misconfigured inputs rather than silent NCCL errors. 

850 buf_device = chunks[0].device 

851 for chunk in chunks[1:]: 

852 if chunk.device != buf_device: 

853 raise RuntimeError( 

854 f"clip_grad_norm_: parameters in the same Partial " 

855 f"coalesce group are on different devices " 

856 f"({buf_device} vs {chunk.device}). All parameters " 

857 f"sharing the same DeviceMesh must reside on the " 

858 f"same local device." 

859 ) 

860 

861 buf = torch.cat(chunks) 

862 

863 for pdim, reduce_op, needs_avg in partial_info: 

864 group = mesh.get_group(pdim) 

865 buf = _all_reduce_offloaded(buf, reduce_op, group) 

866 if needs_avg: 

867 buf /= dist.get_world_size(group=group) 

868 

869 # Extract views for params with actual gradients. 

870 offset = 0 

871 for i, idx in enumerate(active_indices): 

872 numel = chunk_sizes[i] 

873 if has_grad[i]: 

874 reduced[idx] = buf[offset:offset + numel] 

875 offset += numel 

876 

877 return reduced 

878 

879 

880def _build_grad_groups( # pylint: disable=R0914 

881 params: List[torch.Tensor], 

882 parameter_layouts: Optional[Mapping[torch.nn.Parameter, "Layout"]] = None, 

883) -> Tuple[ 

884 Dict[_GradGroupKey, List[torch.Tensor]], 

885 List[torch.Tensor], 

886 List[torch.Tensor], 

887 List[_GradGroupKey], 

888 Dict[int, object], 

889 torch.device, 

890 bool, 

891]: 

892 """Classify parameters into grad groups and pre-reduce Partial grads. 

893 

894 Group structure is derived from *parameter* DTensorSpecs (always 

895 present on every rank) rather than gradients (which may be ``None`` 

896 on some ranks). This ensures every rank enters the same set of 

897 collectives, preventing deadlocks (aligned with FSDP1 where all 

898 ranks unconditionally execute the same all-reduce path). 

899 

900 Partial gradients are reduced via a **coalesced** all-reduce 

901 (see ``_coalesce_partial_reduce``), turning O(N) per-parameter 

902 collectives into O(G) where G is the number of distinct 

903 ``(mesh, partial_info)`` groups (typically 1 for TP+FSDP). 

904 

905 Returns 

906 ``(grad_groups, all_grads, norm_grads, key_per_grad, mesh_cache, 

907 device, has_dtensor_grad)``. ``grad_groups`` maps each 

908 ``(mesh_id, shard_dims)`` key to its grads; ``all_grads`` is the 

909 flat list of raw local grads (global parameter order) scaled 

910 in-place by the clip step; ``norm_grads`` is parallel to 

911 ``all_grads`` but holds the Partial-reduced view for Partial grads 

912 (the value whose norm is taken on the finite-p path); 

913 ``key_per_grad`` is parallel to both and maps each grad back to its 

914 group key. 

915 """ 

916 # --- Phase 1: classify all parameters --- 

917 param_infos: List[Tuple] = [] 

918 mesh_cache: Dict[int, object] = {} 

919 device: Optional[torch.device] = None 

920 

921 for param in params: 

922 if parameter_layouts is None: 

923 mesh, shard_dims, partial_info = _get_param_mesh_info(param) 

924 else: 

925 layout = parameter_layouts.get(param) 

926 if layout is None and not _is_dtensor(param) and not _is_dtensor(_get_grad_obj(param)): 

927 raise ValueError("Explicit layouts must cover every local parameter participating in clipping") 

928 mesh, shard_dims, partial_info = _get_param_mesh_info(param, layout) 

929 key: _GradGroupKey = ( 

930 id(mesh) if mesh is not None else None, shard_dims, 

931 ) 

932 if mesh is not None: 

933 mesh_cache[id(mesh)] = mesh 

934 if device is None: 

935 device = _param_device(param) 

936 local_grad = _get_local_grad(param) 

937 param_infos.append( 

938 (param, local_grad, mesh, partial_info, key), 

939 ) 

940 

941 if device is None: 

942 device = torch.device("cpu") 

943 

944 # --- Phase 2: coalesced Partial reduction (O(N) → O(G)) --- 

945 reduced = _coalesce_partial_reduce(param_infos, mesh_cache) 

946 

947 # --- Phase 3: build grad_groups --- 

948 grad_groups: Dict[_GradGroupKey, List[torch.Tensor]] = defaultdict( 

949 list, 

950 ) 

951 all_grads: List[torch.Tensor] = [] 

952 # norm_grads / key_per_grad are parallel to all_grads (see Returns): the 

953 # norm-input view (Partial-reduced where coalesced, else raw local) and 

954 # the group key per grad, kept in global parameter order. 

955 norm_grads: List[torch.Tensor] = [] 

956 key_per_grad: List[_GradGroupKey] = [] 

957 has_dtensor_grad = False 

958 

959 for idx, info in enumerate(param_infos): 

960 param, local_grad, key = info[0], info[1], info[4] 

961 if local_grad is None: 

962 # Ensure the key exists so the Shard norm all-reduce is 

963 # entered even when this rank has no grads for the group. 

964 if key not in grad_groups: 

965 grad_groups[key] = [] 

966 continue 

967 

968 grad_obj = _get_grad_obj(param) 

969 if _is_dtensor(grad_obj): 

970 has_dtensor_grad = True 

971 all_grads.append(local_grad) 

972 key_per_grad.append(key) 

973 if idx in reduced: 

974 grad_groups[key].append(reduced[idx]) 

975 norm_grads.append(reduced[idx]) 

976 else: 

977 grad_groups[key].append(local_grad) 

978 norm_grads.append(local_grad) 

979 

980 return _GradGroups( 

981 grad_groups, all_grads, norm_grads, key_per_grad, 

982 mesh_cache, device, has_dtensor_grad, 

983 ) 

984 

985 

986def _clip_grads_with_norm_( 

987 all_grads: List[torch.Tensor], 

988 max_norm: float, 

989 total_norm: torch.Tensor, 

990 foreach: Optional[bool] = None, 

991) -> None: 

992 """Scale gradients in-place so the total norm <= *max_norm*.""" 

993 clip_coef = max_norm / (total_norm + 1e-6) 

994 clip_coef_clamped = torch.clamp(clip_coef, max=1.0) 

995 

996 if _group_tensors_by_device_and_dtype is not None: 

997 grouped_grads = _group_tensors_by_device_and_dtype( 

998 [all_grads], 

999 ) 

1000 for (device, dtype), ([device_grads], _) in grouped_grads.items(): 

1001 use_foreach = ( 

1002 foreach is None and _has_foreach_support(device_grads, device) 

1003 ) or (foreach and _device_has_foreach_support(device)) 

1004 if use_foreach: 

1005 torch._foreach_mul_( # pylint: disable=W0212 

1006 device_grads, 

1007 clip_coef_clamped.to(device=device, dtype=dtype), 

1008 ) 

1009 elif foreach: 

1010 raise RuntimeError( 

1011 f"foreach=True was passed, but can't use the " 

1012 f"foreach API on {device.type} tensors" 

1013 ) 

1014 else: 

1015 clip_coef_clamped_cast = clip_coef_clamped.to(device=device, dtype=dtype) 

1016 for g in device_grads: 

1017 g.mul_(clip_coef_clamped_cast) 

1018 else: 

1019 # Fallback when _foreach_utils is unavailable. 

1020 if foreach: 

1021 raise RuntimeError( 

1022 "foreach=True was passed, but " 

1023 "torch.utils._foreach_utils is not available" 

1024 ) 

1025 for grad in all_grads: 

1026 grad.mul_(clip_coef_clamped.to(grad.device, grad.dtype)) 

1027 

1028 

1029# --------------------------------------------------------------------------- 

1030# Public API 

1031# --------------------------------------------------------------------------- 

1032 

1033@torch.no_grad() 

1034def clip_grad_norm_( 

1035 parameters: Union[ 

1036 "torch.nn.Module", torch.Tensor, Iterable[torch.Tensor], 

1037 ], 

1038 max_norm: float, 

1039 norm_type: float = 2.0, 

1040 error_if_nonfinite: bool = False, 

1041 foreach: Optional[bool] = None, 

1042 *, 

1043 parameter_layouts: Optional[Mapping[torch.nn.Parameter, "Layout"]] = None, 

1044) -> torch.Tensor: 

1045 """Compute and clip gradient norm for distributed models. 

1046 

1047 Drop-in replacement for the standard ``clip_grad_norm_`` that 

1048 correctly handles DTensor-sharded parameters by deriving 

1049 communication from each parameter's DTensorSpec. 

1050 

1051 .. warning:: This function uses collective communications. It 

1052 **must be called on all ranks** to avoid deadlocks. Aligned 

1053 with FSDP1: every rank participates in the same collectives 

1054 regardless of local gradient availability. 

1055 

1056 Communication is derived from each parameter's DTensorSpec: 

1057 

1058 * ``Shard`` on mesh dim *d* -- all-reduce norm statistics 

1059 across ``device_mesh.get_group(d)`` 

1060 * ``Partial`` on mesh dim *d* -- all-reduce gradient values 

1061 using the placement's ``reduce_op`` before norm computation 

1062 * ``Replicate`` / plain tensor -- no communication 

1063 

1064 This covers FSDP, HSDP, TP+FSDP, and any combination expressible 

1065 via DTensor placements. PP cross-stage norm aggregation is not 

1066 yet handled (requires manual all-reduce across stages). 

1067 

1068 Args: 

1069 parameters: An ``nn.Module``, a single ``Tensor``, or an 

1070 iterable of ``Tensor`` s whose gradients to clip. 

1071 max_norm: Maximum allowed gradient norm. 

1072 norm_type: Type of the norm (default ``2.0``). 

1073 error_if_nonfinite: If ``True``, raise a ``RuntimeError`` 

1074 when the total norm is non-finite. Default ``False``. 

1075 foreach: Use the faster foreach-based implementation for the 

1076 gradient clipping step. If ``None``, use the foreach 

1077 implementation for devices that support it and silently 

1078 fall back to the per-tensor implementation for others. 

1079 Default ``None``. 

1080 parameter_layouts: Optional parameter-identity mapping to final HyperParallel 

1081 Layouts for local tensors. DTensor gradient/parameter layouts take 

1082 precedence and must agree with explicit metadata. Local trainable 

1083 parameters must be fully covered; frozen parameters are ignored. 

1084 

1085 Returns: 

1086 The total (unclipped) gradient norm as a scalar tensor, 

1087 cast to the promoted dtype of all gradient tensors. 

1088 """ 

1089 max_norm = float(max_norm) 

1090 norm_type = float(norm_type) 

1091 

1092 params = _normalize_parameters(parameters) 

1093 if parameter_layouts is not None: 

1094 params = [param for param in dict.fromkeys(params) if param.requires_grad] 

1095 if foreach is None: 

1096 foreach = False 

1097 

1098 ( 

1099 grad_groups, all_grads, norm_grads, key_per_grad, 

1100 mesh_cache, device, has_dtensor_grad, 

1101 ) = (_build_grad_groups(params) if parameter_layouts is None 

1102 else _build_grad_groups(params, parameter_layouts)) 

1103 

1104 # -- Norm + clip (all ranks participate) -------------------------------- 

1105 # _compute_local_norm returns identity elements for empty groups, 

1106 # so the subsequent all-reduce is safe and semantically neutral. 

1107 total_norm = _get_total_norm( 

1108 grad_groups, norm_type, mesh_cache, device, 

1109 norm_grads, key_per_grad, 

1110 ) 

1111 

1112 if error_if_nonfinite and torch.logical_or( 

1113 total_norm.isnan(), total_norm.isinf() 

1114 ): 

1115 raise RuntimeError( 

1116 f"The total norm of order {norm_type} for gradients from " 

1117 "`parameters` is non-finite, so it cannot be clipped. To " 

1118 "disable this error and scale the gradients by the " 

1119 "non-finite norm anyway, set " 

1120 "`error_if_nonfinite=False`" 

1121 ) 

1122 

1123 if all_grads: 

1124 # Disable foreach for dtensor-backed grads to avoid dispatch issues. 

1125 effective_foreach = False if has_dtensor_grad and foreach is None else foreach 

1126 _clip_grads_with_norm_( 

1127 all_grads, max_norm, total_norm, effective_foreach, 

1128 ) 

1129 

1130 # Promote return dtype to match gradient dtypes (FSDP1 convention). 

1131 # When this rank has no gradients, return in the default FP32 dtype 

1132 # (same as FSDP1's behavior to avoid extra communication). 

1133 if not all_grads: 

1134 warnings.warn( 

1135 "clip_grad_norm_ called on this rank with no gradients -- " 

1136 "returning the local norm in the default dtype " 

1137 f"{total_norm.dtype}", 

1138 stacklevel=2, 

1139 ) 

1140 return total_norm 

1141 

1142 total_norm_dtype = functools.reduce( 

1143 torch.promote_types, 

1144 [g.dtype for g in all_grads], 

1145 ) 

1146 # Return global all-reduced norm, consistent with torchtitan's 

1147 # full_tensor() approach — .item() returns the correct global value. 

1148 return total_norm.to(total_norm_dtype)