Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / core / fully_shard / hsdp_state.py: 93%

302 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""HSDP cell state""" 

16# pylint: disable=protected-access 

17 

18from collections import defaultdict 

19from typing import List, Mapping, Optional 

20 

21import torch 

22from torch import nn 

23 

24from hyper_parallel.tools.logging import get_logger 

25from hyper_parallel.core.dtensor.dtensor import DTensor 

26from hyper_parallel.core.fully_shard.hsdp_utils import ( 

27 _get_param_module_infos, 

28) 

29from hyper_parallel.core.fully_shard.utils import ( 

30 CPUOffloadPolicy, 

31 DDPMeshInfo, 

32 FSDPMeshInfo, 

33 HSDPMeshInfo, 

34 SourceShardMetaInfo, 

35) 

36from hyper_parallel.core.fully_shard.hsdp_param import HSDPParamV2 

37from hyper_parallel.core.fully_shard.param_group import HSDPParamGroup, AllReduceParamGroup 

38 

39logger = get_logger("FSDP") 

40 

41nn.Module = nn.Module 

42 

43 

44def _to_dtype_if_needed( 

45 tensor: torch.Tensor, dtype: Optional[torch.dtype] 

46) -> torch.Tensor: 

47 """Cast tensor to the given dtype if it differs from current dtype. 

48 

49 Args: 

50 tensor: The input tensor to potentially cast. 

51 dtype: Target dtype. If None or same as tensor dtype, no-op. 

52 """ 

53 if dtype is not None and tensor.dtype != dtype: 

54 return tensor.to(dtype) 

55 return tensor 

56 

57 

58class HSDPStateV2: 

59 """HSDP state for cell""" 

60 

61 def __init__( 

62 self, 

63 cell, 

64 mesh, 

65 shard_placement_fn, 

66 comm_fusion_policy, 

67 mp_policy, 

68 offload_policy, 

69 raw_ignored_params, 

70 raw_replicate_params, 

71 scheduler_ctx, 

72 device, 

73 source_shard_infos: Optional[Mapping[torch.nn.Parameter, SourceShardMetaInfo]] = None, 

74 ): 

75 """ 

76 Initialize HSDPStateV2. 

77 

78 Args: 

79 cell (nn.Module): The module whose parameters are managed by this state. 

80 mesh: Mesh topology for shard/replicate dimensions. 

81 shard_placement_fn: Optional function selecting the parameter shard dimension. 

82 comm_fusion_policy: Communication fusion configuration. 

83 mp_policy: Mixed-precision policy. 

84 offload_policy: Parameter offload policy. 

85 raw_ignored_params: Parameters excluded from fully_shard management. 

86 raw_replicate_params: Managed parameters that remain replicated. 

87 scheduler_ctx: Scheduler context shared by this module tree. 

88 device (torch.device): Target device. 

89 source_shard_infos: Optional per-parameter TP/EP source metadata. 

90 """ 

91 self.source_shard_infos = source_shard_infos 

92 self.modules = (cell,) if isinstance(cell, nn.Module) else tuple(cell) 

93 self.cell = self.modules[0] 

94 self.mesh = mesh 

95 self.shard_placement_fn = shard_placement_fn 

96 self.mp_policy = mp_policy 

97 self.offload_policy = offload_policy 

98 self.comm_fusion_policy = comm_fusion_policy 

99 self.raw_ignored_params = set(raw_ignored_params or ()) 

100 self.raw_replicate_params = set(raw_replicate_params or ()) 

101 self.scheduler_ctx = scheduler_ctx 

102 self.device = device 

103 self.hsdp_params: List[HSDPParamV2] = [] 

104 self.param_group = None 

105 self._move_states_to_device() 

106 self._init_hsdp_params() 

107 self.is_shard = True 

108 self.module_name = None 

109 # requires_gradient_sync 

110 self.reduce_grads = True 

111 # Reshard parameter after backward 

112 self.reshard_after_backward = True 

113 # Requires AllReduce for grad When HSDP 

114 self.requires_all_reduce = True 

115 self.set_reduce_op_type("avg") 

116 self._reset_sharded_params = False 

117 self._init_param_group() 

118 

119 def __repr__(self) -> str: 

120 """Stable debug name used in log lines. 

121 

122 ``module_name`` is only assigned by the root forward pre-hook, so fall 

123 back to the managed module class and object id before names are set. 

124 Logging's ``%s`` calls this lazily -- only when a record is emitted. 

125 """ 

126 if self.module_name: 

127 return str(self.module_name) 

128 return f"{self.cell.__class__.__name__}@{id(self.cell):x}" 

129 

130 def _init_param_group(self): 

131 """Initialize fused parameter group for communication fusion. 

132 

133 All managed parameters enter one ``HSDPParamGroup``. Parameters without 

134 an FSDP shard dimension use the group's local all-gather and 

135 reduce-scatter paths before entering replicate all-reduce buckets. 

136 """ 

137 self.param_group = None 

138 if not self.comm_fusion_policy.enable_comm_fusion: 

139 return 

140 if self.hsdp_params: 

141 # pylint: disable=E1128 

142 self.param_group = HSDPParamGroup( 

143 self.hsdp_params, 

144 self.device, 

145 self.comm_fusion_policy.comm_fusion_zero_copy, 

146 comm_ctx=self.scheduler_ctx.param_group_comm_ctx, 

147 ) 

148 

149 def _move_states_to_device(self): 

150 """move states to device""" 

151 for mod in self.modules: 

152 for param in mod.parameters(): 

153 if hasattr(param, "_hsdp_param_initialized") and param._hsdp_param_initialized: 

154 continue 

155 if param.device == self.device or param.device.type == "meta": 

156 continue 

157 param.data = param.to(self.device) 

158 for buffer in mod.buffers(): 

159 if buffer.device == self.device or buffer.device.type == "meta": 

160 continue 

161 buffer.data = buffer.to(self.device) 

162 

163 def _build_param_source_shard_info( 

164 self, param: torch.nn.Parameter 

165 ) -> Optional[SourceShardMetaInfo]: 

166 """Build normalized source-layout metadata for one managed parameter.""" 

167 if isinstance(param, DTensor): 

168 if self.source_shard_infos is not None: 

169 raise ValueError( 

170 "source_shard_infos cannot be provided when fully_shard manages a native DTensor parameter" 

171 ) 

172 return SourceShardMetaInfo( 

173 mesh=param.device_mesh, 

174 placements=tuple(param.placements), 

175 origin_is_dtensor=True, 

176 ) 

177 if self.source_shard_infos is None: 

178 return None 

179 return self.source_shard_infos.get(param) 

180 

181 def _init_hsdp_params(self): 

182 """Initialize all fully_shard-managed parameters for the module.""" 

183 # all parameters in the module tree(s), deduplicated 

184 visited_params = set() 

185 filtered_params = [] 

186 for mod in self.modules: 

187 for _, param in mod.named_parameters(): 

188 if param in self.raw_ignored_params: 

189 continue 

190 if hasattr(param, "_hsdp_param_initialized") and param._hsdp_param_initialized: 

191 continue 

192 if param in visited_params: 

193 continue 

194 visited_params.add(param) 

195 filtered_params.append(param) 

196 

197 module_infos = _get_param_module_infos(filtered_params, tuple(self.modules)) 

198 for param, module_info in zip(filtered_params, module_infos): 

199 self.hsdp_params.append( 

200 HSDPParamV2( 

201 param, 

202 module_info, 

203 self._build_param_mesh_info(param), 

204 shard_placement_fn=self.shard_placement_fn, 

205 mp_policy=self.mp_policy, 

206 offload_policy=self.offload_policy, 

207 device=self.device, 

208 source_shard_info=self._build_param_source_shard_info(param), 

209 ) 

210 ) 

211 

212 def _build_param_mesh_info(self, parameter): 

213 if self.mesh.ndim not in (1, 2): 

214 raise ValueError( 

215 "fully_shard only supports explicit 1D DP/FSDP meshes or 2D HSDP meshes. " 

216 f"Got mesh.ndim={self.mesh.ndim}." 

217 ) 

218 if parameter in self.raw_replicate_params: 

219 return DDPMeshInfo( 

220 mesh=self.mesh if self.mesh.ndim == 1 else self.mesh.flatten(), 

221 replicate_mesh_dim=0, 

222 ) 

223 if self.mesh.ndim == 1: 

224 return FSDPMeshInfo(mesh=self.mesh, shard_mesh_dim=0) 

225 return HSDPMeshInfo( 

226 mesh=self.mesh, 

227 shard_mesh_dim=1, 

228 replicate_mesh_dim=0, 

229 ) 

230 

231 def _init_mp_dtypes(self): 

232 """Initialize dtype attributes for each parameter from the mixed-precision policy.""" 

233 for hsdp_param in self.hsdp_params: 

234 hsdp_param.init_dtype_attrs(self.mp_policy) 

235 

236 def _validate_cpu_offload_params(self): 

237 """Validate that all parameters are on CPU when CPU offload policy is enabled.""" 

238 if not isinstance(self.offload_policy, CPUOffloadPolicy): 

239 return 

240 hsdp_params_not_on_cpu = [ 

241 hsdp_param 

242 for hsdp_param in self.hsdp_params 

243 if hsdp_param.sharded_param.device.type != "cpu" 

244 ] 

245 if hsdp_params_not_on_cpu: 

246 raise RuntimeError( 

247 "HSDP parameters should be materialized on CPU when enabling CPU offloading. " 

248 'For example, load a CPU state dict or call module.to_empty(device="cpu"). ' 

249 "Found following parameters on non-CPU device: " 

250 f"{[(p._param_fqn, p.sharded_param.device) for p in hsdp_params_not_on_cpu]}\n" 

251 ) 

252 

253 def lazy_init(self): 

254 """Deferred initialization: reset sharded params, validate devices, and set mixed-precision dtypes.""" 

255 if self.is_shard and not self._reset_sharded_params: 

256 for hsdp_param in self.hsdp_params: 

257 hsdp_param.reset_sharded_param() 

258 self._reset_sharded_params = True 

259 self._validate_no_meta_params() 

260 self._validate_cpu_offload_params() 

261 self._init_mp_dtypes() 

262 

263 def _validate_no_meta_params(self): 

264 param_names_on_meta = [ 

265 hsdp_param._param_fqn 

266 for hsdp_param in self.hsdp_params 

267 if hsdp_param.sharded_param.device.type == "meta" 

268 ] 

269 if param_names_on_meta: 

270 raise RuntimeError( 

271 "HSDP parameters should be materialized from meta device before training, " 

272 f"but the following were still on meta device: {param_names_on_meta}\n" 

273 "For example, call module.to_empty(device) to materialize to device and " 

274 "call module.reset_parameters() on each module to initialize values." 

275 ) 

276 

277 def post_backward_for_comm_fusion(self): 

278 """post_backward_for_comm_fusion.""" 

279 logger.debug("post_backward module=%s mode=comm_fusion enter", self) 

280 # Fused gradient reduction path: first apply any pending async reduction 

281 # from the previous module's backward (pipelined overlap), then issue 

282 # this module's fused reduce-scatter (+ all-reduce for HSDP). 

283 comm_ctx = self.scheduler_ctx.param_group_comm_ctx 

284 # Phase 2: save gradients for the param group whose all-reduce is done. 

285 if comm_ctx.all_reduce_param_group is not None: 

286 logger.debug("post_backward module=%s wait=comm_fusion_all_reduce", self) 

287 comm_ctx.all_reduce_param_group.wait_all_reduce_and_save_grad() 

288 comm_ctx.all_reduce_param_group = None 

289 # Phase 1: wait reduce_scatter, issue async all_reduce for previous layer 

290 if comm_ctx.pre_param_group is not None: 

291 logger.debug("post_backward module=%s wait=comm_fusion_reduce_scatter", self) 

292 comm_ctx.pre_param_group.wait_reduce_scatter_and_issue_all_reduce() 

293 comm_ctx.pre_param_group = None 

294 if self.param_group is not None: 

295 logger.debug("post_backward module=%s launch=comm_fusion_reduce_scatter", self) 

296 self.param_group.foreach_reducescatter( 

297 reduce_scatter_reduce_op=self.reduce_op_type, 

298 ) 

299 

300 def post_backward(self, *unused): # pylint: disable=unused-argument 

301 """Reduce gradients and reshard parameters after backward.""" 

302 logger.debug( 

303 "post_backward module=%s enter reduce_grads=%s comm_fusion=%s reshard_after_backward=%s", 

304 self, 

305 self.reduce_grads, 

306 self.comm_fusion_policy.enable_comm_fusion, 

307 self.reshard_after_backward, 

308 ) 

309 for hsdp_param in self.hsdp_params: 

310 hsdp_param.accumulate_unsharded_grad_if_needed() 

311 if not self.reduce_grads: 

312 if self.reshard_after_backward: 

313 self.shard() 

314 for hsdp_param in self.hsdp_params: 

315 hsdp_param.to_accumulated_grad_if_needed() 

316 return 

317 if self.reshard_after_backward: 

318 # Reshard before gradient communication to reduce backward memory peak. 

319 self.shard() 

320 if not self.comm_fusion_policy.enable_comm_fusion: 

321 prev_group = [] 

322 if len(self.scheduler_ctx.pre_reduce_scatter_params) >= self.scheduler_ctx.reduce_interval: 

323 prev_group = self._wait_prev_reduce_scatter() 

324 self._wait_prev_reduce_scatter_without_all_reduce() 

325 

326 # Step 3: issue current reduce_scatter 

327 self._issue_reduce_scatter_for_current_module() 

328 

329 # Step 4: issue previous fused all-reduce asynchronously 

330 self._issue_prev_fused_all_reduce(prev_group) 

331 else: 

332 self.post_backward_for_comm_fusion() 

333 

334 def _issue_reduce_scatter_for_current_module(self): 

335 """Issue reduce_scatter for current module's parameters with fused all-reduce support. 

336 

337 This method groups parameters by their replicate_process_group and: 

338 1. For params without all_reduce needs: issue reduce_scatter directly 

339 2. For params with all_reduce needs: allocate fused buffer and issue reduce_scatter 

340 into aligned views, enabling zero-copy fused all_reduce later. 

341 """ 

342 # Collect parameters that need gradient reduction 

343 params_to_reduce = [] 

344 for hsdp_param in self.hsdp_params: 

345 skip_param = ( 

346 not hasattr(hsdp_param, "_unsharded_param") 

347 or not hsdp_param.unsharded_param_buffers 

348 or not hsdp_param.sharded_param.requires_grad 

349 or ( 

350 hsdp_param.unsharded_param.grad is None 

351 and hsdp_param.unsharded_accumulated_grad_data is None 

352 ) 

353 ) 

354 if skip_param: 

355 continue 

356 params_to_reduce.append(hsdp_param) 

357 

358 params_without_all_reduce = [] 

359 all_reduce_groups = [] 

360 

361 # Group by replicate process group and reduction dtype so every fused 

362 # all-reduce buffer has one communication group and one element type. 

363 groups_by_comm = defaultdict(list) 

364 for hsdp_param in params_to_reduce: 

365 if self.requires_all_reduce and hsdp_param.replicate_world_size > 1: 

366 replicate_process_group = hsdp_param.mesh_info.replicate_process_group 

367 group_key = (id(replicate_process_group), hsdp_param.reduce_comm_dtype()) 

368 groups_by_comm[group_key].append(hsdp_param) 

369 else: 

370 groups_by_comm[None].append(hsdp_param) 

371 

372 # Handle params that don't need all_reduce (FSDP or single replica) 

373 if None in groups_by_comm: 

374 for hsdp_param in groups_by_comm[None]: 

375 logger.debug( 

376 "post_backward module=%s launch=reduce_scatter param=%s all_reduce=False", 

377 self, 

378 hsdp_param, 

379 ) 

380 hsdp_param.reduce_scatter_grad( 

381 reduce_op=self.reduce_op_type, 

382 ) 

383 params_without_all_reduce.append(hsdp_param) 

384 

385 # Handle params that need all_reduce (HSDP with multiple replicas) 

386 for group_key, hsdp_params in groups_by_comm.items(): 

387 if group_key is None: 

388 continue 

389 

390 # Create AllReduceParamGroup for fused all-reduce 

391 group = AllReduceParamGroup( 

392 replicate_group=hsdp_params[0].mesh_info.replicate_process_group, 

393 hsdp_params=hsdp_params, 

394 reduce_op=self.reduce_op_type, 

395 ) 

396 

397 # Allocate fused buffer with 512-byte alignment 

398 group.allocate_fused_buffer(self.device) 

399 

400 # Issue reduce_scatter with output directly into fused buffer views 

401 logger.debug( 

402 "post_backward module=%s launch=fused_reduce_scatter group_params=%s", 

403 self, 

404 hsdp_params, 

405 ) 

406 for idx, hsdp_param in enumerate(hsdp_params): 

407 buffer_view = group.get_param_buffer_view(idx) 

408 hsdp_param.reduce_scatter_grad( 

409 reduce_op=self.reduce_op_type, 

410 output_buffer=buffer_view, 

411 ) 

412 

413 all_reduce_groups.append(group) 

414 

415 # Empty slots preserve the interval measured in non-fused module units. 

416 self.scheduler_ctx.pre_reduce_scatter_params.append(params_without_all_reduce) 

417 self.scheduler_ctx.pre_all_reduce_groups.append(all_reduce_groups) 

418 

419 def _wait_prev_reduce_scatter(self) -> List[AllReduceParamGroup]: 

420 """Wait the oldest unit's reduce-scatter before replica all-reduce. 

421 

422 The shared interval determines how many subsequent units overlap it. 

423 

424 Returns: 

425 List of previous AllReduceParamGroups (one per communication group). 

426 """ 

427 if self.scheduler_ctx.pre_all_reduce_groups: 

428 prev_groups = self.scheduler_ctx.pre_all_reduce_groups.popleft() 

429 for prev_group in prev_groups: 

430 logger.debug( 

431 "post_backward module=%s wait=fused_reduce_scatter group_params=%s", 

432 self, 

433 prev_group.hsdp_params, 

434 ) 

435 for hsdp_param in prev_group.hsdp_params: 

436 reduced_grad = hsdp_param.reduce_scatter_output() 

437 hsdp_param.all_reduce_source_replicate_grad_inplace(reduced_grad, torch.distributed.ReduceOp.SUM) 

438 hsdp_param.clear_reduce_scatter_output() 

439 if hsdp_param.unsharded_accumulated_grad_data is not None: 

440 hsdp_param.unsharded_accumulated_grad = None 

441 elif hsdp_param.unsharded_param.grad is not None: 

442 hsdp_param.unsharded_param.grad = None 

443 return prev_groups 

444 return [] 

445 

446 def _issue_prev_fused_all_reduce(self, prev_groups: List[AllReduceParamGroup]) -> None: 

447 """Step 4: issue the previous module's fused all-reduce asynchronously. 

448 

449 The all-reduce work is collected in ``pending_all_reduce_groups`` 

450 and is waited in the root backward hook. 

451 

452 Args: 

453 prev_groups: Previous parameter groups whose all-reduce should be issued. 

454 """ 

455 for prev_group in prev_groups: 

456 prev_group.accumulate_reduce_partial_outputs() 

457 logger.debug( 

458 "post_backward module=%s launch=fused_all_reduce group_params=%s", 

459 self, 

460 prev_group.hsdp_params, 

461 ) 

462 prev_group.issue_async_allreduce() 

463 self.scheduler_ctx.pending_all_reduce_groups.append(prev_group) 

464 

465 def _wait_prev_reduce_scatter_without_all_reduce(self) -> None: 

466 """Wait RS outputs, accumulate partial results, and clear communication slots. 

467 

468 Single-replica partials are applied immediately. Multi-replica partials 

469 remain for the existing AllReduceParamGroup to consume before all-reduce. 

470 """ 

471 need_synchronize = False 

472 params = ( 

473 self.scheduler_ctx.pre_reduce_scatter_params.popleft() 

474 if self.scheduler_ctx.pre_reduce_scatter_params else () 

475 ) 

476 for pre_hsdp_param in params: 

477 logger.debug( 

478 "post_backward module=%s wait=reduce_scatter param=%s", 

479 self, 

480 pre_hsdp_param, 

481 ) 

482 reduced_grad = pre_hsdp_param.reduce_scatter_output() 

483 pre_hsdp_param.all_reduce_source_replicate_grad_inplace(reduced_grad, torch.distributed.ReduceOp.SUM) 

484 if pre_hsdp_param.reduce_partial_output is None: 

485 pre_hsdp_param.reduce_partial_output = reduced_grad 

486 else: 

487 pre_hsdp_param.reduce_partial_output.add_(reduced_grad) 

488 pre_hsdp_param.clear_reduce_scatter_output() 

489 

490 if pre_hsdp_param.replicate_world_size <= 1: 

491 # No replica all-reduce follows, so accumulate into the parameter 

492 # gradient immediately and clear the consumed partial. 

493 need_synchronize = ( 

494 pre_hsdp_param.apply_reduced_grad(pre_hsdp_param.reduce_partial_output) or need_synchronize 

495 ) 

496 pre_hsdp_param.reduce_partial_output = None 

497 

498 if pre_hsdp_param.unsharded_accumulated_grad_data is not None: 

499 pre_hsdp_param.unsharded_accumulated_grad = None 

500 elif pre_hsdp_param.unsharded_param.grad is not None: 

501 pre_hsdp_param.unsharded_param.grad = None 

502 self._sync_current_stream_if_needed(need_synchronize) 

503 

504 def wait_and_split_all_reduce_work_groups(self) -> None: 

505 """Wait fused all-reduce work and expose each parameter result.""" 

506 for group in self.scheduler_ctx.pending_all_reduce_groups: 

507 logger.debug( 

508 "post_backward module=%s wait=fused_all_reduce group_params=%s", 

509 self, 

510 group.hsdp_params, 

511 ) 

512 group.wait_and_split_grads() 

513 self.scheduler_ctx.pending_all_reduce_groups.clear() 

514 

515 def reset_iter_state(self) -> None: 

516 """Clear Torch communication bookkeeping without clearing optimizer gradients.""" 

517 self.scheduler_ctx.pre_reduce_scatter_params.clear() 

518 self.scheduler_ctx.pre_all_reduce_params.clear() 

519 self.scheduler_ctx.pre_all_reduce_groups.clear() 

520 self.scheduler_ctx.pending_all_reduce_groups.clear() 

521 if self.param_group is not None: 

522 self.param_group.reset_iter_state() 

523 for hsdp_param in self.hsdp_params: 

524 hsdp_param.allgather_comm_ctx.allgather_handle = None 

525 hsdp_param.allgather_comm_ctx.allgather_output = None 

526 hsdp_param.reduce_scatter_comm_ctx.reduce_scatter_handle = None 

527 hsdp_param.reduce_scatter_comm_ctx.reduce_scatter_output = None 

528 hsdp_param.all_reduce_comm_ctx.all_reduce_handle = None 

529 hsdp_param.all_reduce_comm_ctx.all_reduce_output = None 

530 hsdp_param.reduce_partial_output = None 

531 hsdp_param.unsharded_accumulated_grad = None 

532 hsdp_param._grad = None 

533 if hsdp_param.unsharded_param_buffers: 

534 hsdp_param.unsharded_param.grad = None 

535 

536 def set_requires_grad_sync(self, requires_grad_sync: bool) -> None: 

537 """set requires grad sync flag to control gradient sync.""" 

538 self.reduce_grads = requires_grad_sync 

539 

540 def set_reduce_op_type(self, reduce_op_type: str) -> None: 

541 """set reduce op type for gradient reduction.""" 

542 fsdp_support_reduce_op = { 

543 "sum": torch.distributed.ReduceOp.SUM, 

544 "avg": torch.distributed.ReduceOp.AVG, 

545 } 

546 reduce_op = reduce_op_type.lower().strip() if isinstance(reduce_op_type, str) else reduce_op_type 

547 reduce_op_value = fsdp_support_reduce_op.get(reduce_op) 

548 if reduce_op_value is None: 

549 raise ValueError( 

550 f"Unsupported reduce op type {reduce_op_type}, " 

551 f"supported types are {list(fsdp_support_reduce_op.keys())}" 

552 ) 

553 self.reduce_op_type = reduce_op_value 

554 

555 def shard(self) -> None: 

556 """change parameters to sharded state""" 

557 logger.debug( 

558 "action=reshard module=%s params=%s", 

559 self, 

560 self.hsdp_params, 

561 ) 

562 if self.is_shard: 

563 return 

564 for param in self.hsdp_params: 

565 param.to_sharded() 

566 self.is_shard = True 

567 

568 def unshard(self, async_op: bool = False) -> None: 

569 """change parameters to unsharded state""" 

570 logger.debug( 

571 "action=unshard module=%s async_op=%s params=%s", 

572 self, 

573 async_op, 

574 self.hsdp_params, 

575 ) 

576 if not self.is_shard: 

577 return 

578 

579 if self.comm_fusion_policy.enable_comm_fusion and self.param_group is not None: 

580 self.param_group.unshard(async_op) 

581 else: 

582 for param in self.hsdp_params: 

583 param.unshard(async_op) 

584 if not async_op: 

585 self.wait_for_unshard() 

586 

587 def prefetch(self) -> None: 

588 """prefetch unsharded parameters""" 

589 logger.debug( 

590 "action=prefetch module=%s params=%s", 

591 self, 

592 self.hsdp_params, 

593 ) 

594 self.unshard(async_op=True) 

595 

596 def wait_for_unshard(self) -> None: 

597 """wait for all unshard parameters""" 

598 logger.debug( 

599 "action=wait_unshard module=%s params=%s", 

600 self, 

601 self.hsdp_params, 

602 ) 

603 if not self.is_shard: 

604 return 

605 

606 if self.comm_fusion_policy.enable_comm_fusion and self.param_group is not None: 

607 self.param_group.wait_for_unshard() 

608 else: 

609 for param in self.hsdp_params: 

610 param.wait_for_unshard() 

611 self.is_shard = False 

612 

613 def set_gradient_scaling_factor(self, factor): 

614 """Propagate the gradient scaling factor to the layer that applies it. 

615 

616 The factor is consumed on the reduce input: ``param_group.foreach_reduce`` 

617 for the fused (comm_fusion) path, or per-parameter ``reduce_scatter_grad`` 

618 / ``all_reduce_grad`` otherwise. The state does not hold a copy. 

619 """ 

620 if self.param_group is not None: 

621 self.param_group.gradient_scaling_factor = factor 

622 else: 

623 for hsdp_param in self.hsdp_params: 

624 hsdp_param.gradient_scaling_factor = factor 

625 

626 def set_requires_all_reduce(self, requires_all_reduce: bool) -> None: 

627 """Propagate the HSDP all-reduce switch to the active communication path.""" 

628 self.requires_all_reduce = requires_all_reduce 

629 if self.param_group is not None: 

630 self.param_group.requires_all_reduce = requires_all_reduce 

631 

632 def _sync_current_stream_if_needed(self, need_synchronize): 

633 if need_synchronize: 

634 if self.device.type == "npu": 

635 torch.npu.current_stream().synchronize() 

636 elif self.device.type == "cuda": 

637 torch.cuda.current_stream().synchronize() 

638 else: 

639 raise NotImplementedError( 

640 f"Unsupported device type {self.device.type} for synchronization after CPU offload." 

641 )