Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / tensor_probes.py: 77%

284 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Tensor summaries, input identity, and parameter probe recorders.""" 

16 

17from __future__ import annotations 

18 

19import dataclasses 

20import fnmatch 

21import hashlib 

22import json 

23import math 

24from typing import Any, Iterable, Mapping, Optional 

25 

26import torch 

27import torch.distributed as dist 

28 

29from hyper_parallel.models.validation_spec import ( 

30 ModelValidationSpec, 

31 ParameterProbeSpec, 

32) 

33from hyper_parallel.tools.model_integration.evidence_store import EvidenceStore 

34from hyper_parallel.tools.model_integration.optimizer_layout import ( 

35 optimizer_state_for_parameter, 

36 tensor_layout, 

37) 

38 

39_SMALL_TENSOR_NUMEL = 8192 

40 

41def _tensor_numel(tensor: Any) -> int: 

42 shape = getattr(tensor, "shape", ()) 

43 return math.prod(int(size) for size in shape) 

44 

45 

46def _local_tensor(tensor: Any) -> torch.Tensor: 

47 """Return one detached local shard without requesting redistribution.""" 

48 to_local = getattr(tensor, "to_local", None) 

49 value = to_local() if callable(to_local) else tensor 

50 return value.detach().float().cpu().contiguous() 

51 

52 

53def _shard_identity(tensor: Any) -> tuple[tuple[int, int], ...]: 

54 """Identify the logical shard while excluding replicated mesh axes.""" 

55 layout = getattr(tensor, "layout", None) 

56 mesh = getattr(layout, "mesh", getattr(tensor, "device_mesh", None)) 

57 placements = getattr(layout, "placements", getattr(tensor, "placements", ())) 

58 get_coordinate = getattr(mesh, "get_coordinate", None) 

59 coordinate = get_coordinate() if callable(get_coordinate) else None 

60 if coordinate is None: 

61 return () 

62 shard_coordinates = [] 

63 for mesh_dim, placement in enumerate(placements or ()): 

64 is_shard = getattr(placement, "is_shard", None) 

65 is_ragged = getattr(placement, "is_ragged_shard", None) 

66 if (callable(is_shard) and is_shard()) or ( 

67 callable(is_ragged) and is_ragged() 

68 ): 

69 shard_coordinates.append((mesh_dim, int(coordinate[mesh_dim]))) 

70 return tuple(shard_coordinates) 

71 

72 

73def _local_shards_follow_flattened_order(tensor: Any) -> bool: 

74 """Return whether rank-ordered shard values reconstruct the flattened tensor.""" 

75 layout = getattr(tensor, "layout", None) 

76 placements = getattr(layout, "placements", getattr(tensor, "placements", ())) 

77 shard_placements = [] 

78 for placement in placements or (): 

79 is_shard = getattr(placement, "is_shard", None) 

80 is_ragged = getattr(placement, "is_ragged_shard", None) 

81 if (callable(is_shard) and is_shard()) or ( 

82 callable(is_ragged) and is_ragged() 

83 ): 

84 shard_placements.append(placement) 

85 if not shard_placements: 

86 return True 

87 if len(shard_placements) != 1: 

88 return False 

89 placement = shard_placements[0] 

90 return getattr(placement, "dim", None) == 0 and not hasattr( 

91 placement, 

92 "split_factor", 

93 ) 

94 

95 

96def _local_tensor_summary(tensor: Any) -> dict[str, Any]: 

97 """Summarize one local shard for cross-rank aggregation.""" 

98 value = _local_tensor(tensor).reshape(-1) 

99 digest = hashlib.sha256(value.numpy().tobytes()).hexdigest() 

100 return { 

101 "rank": dist.get_rank() if dist.is_available() and dist.is_initialized() else 0, 

102 "shard": _shard_identity(tensor), 

103 "sha256": digest, 

104 "numel": value.numel(), 

105 "finite": bool(torch.isfinite(value).all()) if value.numel() else True, 

106 "sum": float(value.double().sum()), 

107 "squared_l2": float(torch.square(value.double()).sum()), 

108 "min": float(value.min()) if value.numel() else None, 

109 "max": float(value.max()) if value.numel() else None, 

110 "values": value.tolist() if value.numel() <= _SMALL_TENSOR_NUMEL else None, 

111 "global_shape": [int(size) for size in getattr(tensor, "shape", ())], 

112 "dtype": str(getattr(tensor, "dtype", None)), 

113 "layout": tensor_layout(tensor), 

114 } 

115 

116 

117def _gather_shard_summaries( 

118 local_summary: Optional[dict[str, Any]], 

119) -> tuple[list[dict[str, Any]], list[Optional[dict[str, Any]]]]: 

120 """Gather optional metadata records and retain one value per logical shard.""" 

121 rank_summaries = [local_summary] 

122 if dist.is_available() and dist.is_initialized(): 

123 rank_summaries = [None] * dist.get_world_size() 

124 dist.all_gather_object(rank_summaries, local_summary) 

125 by_shard: dict[tuple[tuple[int, int], ...], list[dict[str, Any]]] = {} 

126 for summary in rank_summaries: 

127 if summary is None: 

128 continue 

129 shard = tuple(tuple(item) for item in summary["shard"]) 

130 by_shard.setdefault(shard, []).append(summary) 

131 shards = [ 

132 min(replicas, key=lambda item: item["rank"]) 

133 for replicas in by_shard.values() 

134 ] 

135 return shards, rank_summaries 

136 

137 

138def _replica_digest_groups( 

139 rank_summaries: list[Optional[dict[str, Any]]], 

140) -> dict[tuple[Any, ...], set[str]]: 

141 """Group per-rank payload digests by their logical shard identity.""" 

142 replica_groups: dict[tuple[Any, ...], set[str]] = {} 

143 for rank_summary in rank_summaries: 

144 if rank_summary is None: 

145 continue 

146 shard = tuple(tuple(item) for item in rank_summary["shard"]) 

147 replica_groups.setdefault(shard, set()).add(rank_summary["sha256"]) 

148 return replica_groups 

149 

150 

151def _canonical_shard_digest(shards: list[dict[str, Any]]) -> str: 

152 """Hash the ordered shard digests into one canonical tensor identity.""" 

153 return hashlib.sha256( 

154 json.dumps( 

155 [(summary["shard"], summary["sha256"]) for summary in shards], 

156 sort_keys=True, 

157 ).encode("utf-8") 

158 ).hexdigest() 

159 

160 

161def _flattened_shard_values(tensor: Any, shards: list[dict[str, Any]]) -> Optional[list[Any]]: 

162 """Reconstruct full values only from flattened-order small shard sets.""" 

163 if not ( 

164 _local_shards_follow_flattened_order(tensor) 

165 and sum(shard["numel"] for shard in shards) <= _SMALL_TENSOR_NUMEL 

166 and all(shard["values"] is not None for shard in shards) 

167 ): 

168 return None 

169 return [value for shard in shards for value in shard["values"]] 

170 

171 

172def _global_shard_summary( 

173 name: str, 

174 tensor: Any, 

175 shards: list[dict[str, Any]], 

176 rank_summaries: list[Optional[dict[str, Any]]], 

177) -> dict[str, Any]: 

178 """Aggregate sorted shard summaries into one canonical global record.""" 

179 replica_groups = _replica_digest_groups(rank_summaries) 

180 minima = [summary["min"] for summary in shards if summary["min"] is not None] 

181 maxima = [summary["max"] for summary in shards if summary["max"] is not None] 

182 reference = shards[0] 

183 summary = { 

184 "name": name, 

185 "global_shape": reference["global_shape"], 

186 "canonical_shape": reference["global_shape"], 

187 "dtype": reference["dtype"], 

188 "sha256": _canonical_shard_digest(shards), 

189 "finite": all(shard["finite"] for shard in shards), 

190 "sum": sum(shard["sum"] for shard in shards), 

191 "l2": math.sqrt(sum(shard["squared_l2"] for shard in shards)), 

192 "min": min(minima) if minima else None, 

193 "max": max(maxima) if maxima else None, 

194 "layout": reference["layout"], 

195 "replica": { 

196 "logical_shards": len(replica_groups), 

197 "replicas_equal": all(len(digests) == 1 for digests in replica_groups.values()), 

198 }, 

199 "missing_ranks": [ 

200 rank for rank, rank_summary in enumerate(rank_summaries) if rank_summary is None 

201 ], 

202 } 

203 values = _flattened_shard_values(tensor, shards) 

204 if values is not None: 

205 summary["values"] = values 

206 return summary 

207 

208 

209def _tensor_summary(name: str, tensor: Any) -> Optional[dict[str, Any]]: 

210 """Build one global summary while allowing rank-local lazy state absence.""" 

211 local_summary = None if tensor is None else _local_tensor_summary(tensor) 

212 shards, rank_summaries = _gather_shard_summaries(local_summary) 

213 if not shards: 

214 return None 

215 shards.sort(key=lambda item: item["shard"]) 

216 return _global_shard_summary(name, tensor, shards, rank_summaries) 

217 

218 

219def _global_tensor_state_names(local_names: Iterable[str]) -> tuple[str, ...]: 

220 """Return a rank-consistent tensor-state key order for collective probes.""" 

221 local_names = tuple(sorted(local_names)) 

222 rank_names = [local_names] 

223 if dist.is_available() and dist.is_initialized(): 

224 rank_names = [None] * dist.get_world_size() 

225 dist.all_gather_object(rank_names, local_names) 

226 return tuple(sorted({name for names in rank_names for name in (names or ())})) 

227 

228 

229class InputIdentityRecorder: 

230 """Hash global step inputs from deterministic per-rank tensor payloads.""" 

231 

232 def __init__( 

233 self, 

234 mesh_context: Any = None, 

235 cp_replicated_forward_fields: Iterable[str] = (), 

236 ) -> None: 

237 """Configure mesh-aware canonicalization for one Trainer run.""" 

238 self.mesh_context = mesh_context 

239 self.cp_replicated_forward_fields = set(cp_replicated_forward_fields) 

240 self._entries: list[dict[str, Any]] = [] 

241 self._micro_steps = 0 

242 

243 def begin_step(self) -> None: 

244 """Reset the current optimizer-step identity.""" 

245 self._entries = [] 

246 self._micro_steps = 0 

247 

248 @staticmethod 

249 def _tensor_leaves(value: Any) -> list[Any]: 

250 """Find tensors inside mappings, sequences, and metadata dataclasses.""" 

251 if torch.is_tensor(value): 

252 return [value] 

253 if dataclasses.is_dataclass(value) and not isinstance(value, type): 

254 return [ 

255 tensor 

256 for dataclass_field in dataclasses.fields(value) 

257 for field_value in (getattr(value, dataclass_field.name),) 

258 for tensor in InputIdentityRecorder._tensor_leaves(field_value) 

259 ] 

260 if isinstance(value, Mapping): 

261 return [ 

262 tensor 

263 for name in sorted(value) 

264 for tensor in InputIdentityRecorder._tensor_leaves(value[name]) 

265 ] 

266 if isinstance(value, (list, tuple)): 

267 return [ 

268 tensor 

269 for child in value 

270 for tensor in InputIdentityRecorder._tensor_leaves(child) 

271 ] 

272 return [] 

273 

274 def record(self, model_inputs: Mapping[str, Any], loss_inputs: Mapping[str, Any]) -> None: 

275 """Add model/loss tensor fields without changing their device or values.""" 

276 for owner, values in (("model", model_inputs), ("loss", loss_inputs)): 

277 for field_name in sorted(values): 

278 flattened = self._tensor_leaves(values[field_name]) 

279 for value_index, value in enumerate(flattened): 

280 local = value.detach() 

281 to_local = getattr(local, "to_local", None) 

282 if callable(to_local): 

283 local = to_local() 

284 self._entries.append( 

285 { 

286 "micro_step": self._micro_steps, 

287 "owner": owner, 

288 "field": field_name, 

289 "value_index": value_index, 

290 "cp_replicated": ( 

291 owner == "model" 

292 and field_name in self.cp_replicated_forward_fields 

293 ), 

294 "tensor": local.cpu().contiguous(), 

295 } 

296 ) 

297 self._micro_steps += 1 

298 

299 def _coordinates(self) -> dict[str, int]: 

300 return { 

301 name: int(getattr(self.mesh_context, f"{name}_rank", 0)) 

302 for name in ("dp", "cp", "tp", "pp") 

303 } 

304 

305 @staticmethod 

306 def _hash_tensor(value: torch.Tensor) -> str: 

307 hasher = hashlib.sha256() 

308 hasher.update(f"{tuple(value.shape)}:{value.dtype}".encode()) 

309 byte_view = value.contiguous().view(torch.uint8) 

310 hasher.update(byte_view.numpy().tobytes()) 

311 return hasher.hexdigest() 

312 

313 @staticmethod 

314 def _require_input_payloads(rank_payloads: list[dict[str, Any]]) -> None: 

315 """Reject collectives polluted by non-input rank payloads.""" 

316 malformed_ranks = [ 

317 rank 

318 for rank, payload in enumerate(rank_payloads) 

319 if not isinstance(payload, Mapping) 

320 or "coordinates" not in payload 

321 or "entries" not in payload 

322 ] 

323 if malformed_ranks: 

324 raise RuntimeError( 

325 "input-identity collective received non-input payloads from ranks " 

326 f"{malformed_ranks}; parameter probes must execute a rank-consistent " 

327 "collective sequence" 

328 ) 

329 

330 @staticmethod 

331 def _group_input_entries( 

332 rank_payloads: list[dict[str, Any]], 

333 ) -> dict[tuple[Any, ...], list[tuple[int, torch.Tensor, bool]]]: 

334 """Group entry shards by logical field across tp0/pp0 ranks.""" 

335 selected = [ 

336 payload 

337 for payload in rank_payloads 

338 if payload["coordinates"]["tp"] == 0 and payload["coordinates"]["pp"] == 0 

339 ] 

340 grouped: dict[tuple[Any, ...], list[tuple[int, torch.Tensor, bool]]] = {} 

341 for payload in selected: 

342 coordinates = payload["coordinates"] 

343 for entry in payload["entries"]: 

344 key = ( 

345 entry["micro_step"], 

346 coordinates["dp"], 

347 entry["owner"], 

348 entry["field"], 

349 entry["value_index"], 

350 ) 

351 grouped.setdefault(key, []).append( 

352 ( 

353 coordinates["cp"], 

354 entry["tensor"], 

355 bool(entry.get("cp_replicated", False)), 

356 ) 

357 ) 

358 return grouped 

359 

360 @staticmethod 

361 def _merge_entry_shards( 

362 shards: list[tuple[int, torch.Tensor, bool]], 

363 ) -> torch.Tensor: 

364 """Rebuild one logical field tensor from ordered CP shards.""" 

365 ordered_shards = sorted(shards, key=lambda item: item[0]) 

366 ordered = [tensor for _, tensor, _ in ordered_shards] 

367 tensor = ordered[0] 

368 cp_replicated = all(replicated for _, _, replicated in ordered_shards) 

369 if cp_replicated and any( 

370 not torch.equal(tensor, replica) for replica in ordered[1:] 

371 ): 

372 cp_replicated = False 

373 if len(ordered) > 1 and tensor.ndim > 0 and not cp_replicated: 

374 try: 

375 tensor = torch.cat(ordered, dim=-1) 

376 except RuntimeError: 

377 tensor = torch.cat([value.reshape(-1) for value in ordered]) 

378 return tensor 

379 

380 @classmethod 

381 def _entry_component_hashes( 

382 cls, 

383 grouped: dict[tuple[Any, ...], list[tuple[int, torch.Tensor, bool]]], 

384 ) -> list[str]: 

385 """Hash every logical field occurrence independently of rank order.""" 

386 component_hashes = [] 

387 for key, shards in grouped.items(): 

388 tensor = cls._merge_entry_shards(shards) 

389 field_identity = f"{key[2]}:{key[3]}:{key[4]}" 

390 if tensor.ndim > 0: 

391 values = tensor.reshape(tensor.shape[0], -1) 

392 component_hashes.extend( 

393 f"{field_identity}:{cls._hash_tensor(value)}" for value in values 

394 ) 

395 else: 

396 component_hashes.append(f"{field_identity}:{cls._hash_tensor(tensor)}") 

397 return component_hashes 

398 

399 @classmethod 

400 def _canonical_global_hash(cls, rank_payloads: list[dict[str, Any]]) -> str: 

401 cls._require_input_payloads(rank_payloads) 

402 grouped = cls._group_input_entries(rank_payloads) 

403 component_hashes = cls._entry_component_hashes(grouped) 

404 hasher = hashlib.sha256() 

405 for component_hash in sorted(component_hashes): 

406 hasher.update(component_hash.encode()) 

407 return hasher.hexdigest() 

408 

409 def finish_step(self) -> dict[str, Any]: 

410 """Return rank and global digests for the optimizer step.""" 

411 local_payload = { 

412 "coordinates": self._coordinates(), 

413 "entries": self._entries, 

414 } 

415 rank_payloads = [local_payload] 

416 if dist.is_available() and dist.is_initialized(): 

417 rank_payloads = [None] * dist.get_world_size() 

418 dist.all_gather_object(rank_payloads, local_payload) 

419 return { 

420 "global_sha256": self._canonical_global_hash(rank_payloads), 

421 "rank_count": len(rank_payloads), 

422 "micro_steps": self._micro_steps, 

423 } 

424 

425 

426class ParameterProbeRecorder: 

427 """Capture canonical parameter/main-gradient/update evidence by semantic watchlist.""" 

428 

429 def __init__( 

430 self, 

431 model: Any, 

432 optimizer: Any, 

433 evidence_store: EvidenceStore, 

434 *, 

435 rank: int, 

436 validation_spec: Optional[ModelValidationSpec] = None, 

437 include_small_parameters_numel_le: int = _SMALL_TENSOR_NUMEL, 

438 representative_parameter_names: Iterable[str] = (), 

439 ) -> None: 

440 """Resolve a stable watchlist from the final model and adapter declarations.""" 

441 self.model = model 

442 self.optimizer = optimizer 

443 self.evidence_store = evidence_store 

444 self.rank = rank 

445 self.validation_spec = validation_spec 

446 self.include_small_parameters_numel_le = include_small_parameters_numel_le 

447 self.representative_parameter_names = set(representative_parameter_names) 

448 self._named_parameters = dict(model.named_parameters(remove_duplicate=False)) 

449 self._probes = self._resolve_probes() 

450 self._validate_required_patterns() 

451 self._step = 0 

452 

453 def _resolve_probes(self) -> dict[str, ParameterProbeSpec]: 

454 probes = {} 

455 declared = self.validation_spec.parameter_probes if self.validation_spec else () 

456 replaced_module_fqns = tuple( 

457 getattr(self.model, "_hp_replaced_module_fqns", ()) or () 

458 ) 

459 for parameter_name, parameter in self._named_parameters.items(): 

460 main_parameter = getattr(parameter, "main_param", parameter) 

461 replacement_parameter = any( 

462 parameter_name == module_fqn 

463 or parameter_name.startswith(f"{module_fqn}.") 

464 for module_fqn in replaced_module_fqns 

465 ) 

466 if ( 

467 replacement_parameter 

468 or parameter_name in self.representative_parameter_names 

469 or _tensor_numel(main_parameter) <= self.include_small_parameters_numel_le 

470 ): 

471 probes[parameter_name] = ParameterProbeSpec( 

472 match=parameter_name, 

473 required=True, 

474 stages=( 

475 "model_before", 

476 "main_before", 

477 "main_grad_before_clip", 

478 "main_grad_after_clip", 

479 "main_after_optimizer", 

480 "model_after_copy_back", 

481 "optimizer_state", 

482 ), 

483 ) 

484 for probe in declared: 

485 if fnmatch.fnmatchcase(parameter_name, probe.match): 

486 probes[parameter_name] = probe 

487 return dict(sorted(probes.items())) 

488 

489 def _validate_required_patterns(self) -> None: 

490 for probe in self.validation_spec.parameter_probes if self.validation_spec else (): 

491 if probe.required and not any( 

492 fnmatch.fnmatchcase(name, probe.match) for name in self._named_parameters 

493 ): 

494 raise ValueError( 

495 f"required parameter probe {probe.match!r} matched no final parameter" 

496 ) 

497 

498 @property 

499 def watched_parameter_names(self) -> tuple[str, ...]: 

500 """Return stable final FQNs in capture order.""" 

501 return tuple(self._probes) 

502 

503 def begin_step(self, step: int) -> None: 

504 """Start evidence for one optimizer step.""" 

505 self._step = step 

506 self.capture("model_before") 

507 self.capture("main_before") 

508 

509 @staticmethod 

510 def _stage_tensor(parameter: Any, stage: str) -> Any: 

511 main_parameter = getattr(parameter, "main_param", parameter) 

512 if stage in ("model_before", "model_after_copy_back"): 

513 return parameter 

514 if stage in ("main_before", "main_after_optimizer"): 

515 return main_parameter 

516 if stage in ("main_grad_before_clip", "main_grad_after_clip"): 

517 main_gradient = getattr(parameter, "main_grad", None) 

518 return main_gradient if main_gradient is not None else getattr(main_parameter, "grad", None) 

519 return None 

520 

521 def _optimizer_state(self, parameter: Any) -> dict[str, Any]: 

522 main_parameter = ( 

523 None if parameter is None else getattr(parameter, "main_param", parameter) 

524 ) 

525 state = ( 

526 {} if main_parameter is None 

527 else optimizer_state_for_parameter(self.optimizer, main_parameter) 

528 ) 

529 state_names = _global_tensor_state_names( 

530 name for name, value in state.items() if torch.is_tensor(value) 

531 ) 

532 summaries = {} 

533 for name in state_names: 

534 value = state.get(name) 

535 summary = _tensor_summary(name, value if torch.is_tensor(value) else None) 

536 if summary is not None: 

537 summaries[name] = summary 

538 return summaries 

539 

540 def capture(self, stage: str) -> None: 

541 """Capture global summaries and persist one canonical rank-zero copy.""" 

542 records = [] 

543 for parameter_name, probe in self._probes.items(): 

544 if stage not in probe.stages: 

545 continue 

546 parameter = self._named_parameters.get(parameter_name) 

547 if stage == "optimizer_state": 

548 summary = self._optimizer_state(parameter) 

549 if not summary: 

550 continue 

551 record = {"name": parameter_name, "optimizer_state": summary} 

552 else: 

553 tensor = None if parameter is None else self._stage_tensor(parameter, stage) 

554 record = _tensor_summary(parameter_name, tensor) 

555 if record is None: 

556 continue 

557 if not probe.check_replica_consistency: 

558 record.pop("replica", None) 

559 record.update( 

560 { 

561 "step": self._step, 

562 "stage": stage, 

563 "rank": self.rank, 

564 "comparison": probe.compare, 

565 } 

566 ) 

567 records.append(record) 

568 if self.rank != 0: 

569 return 

570 for record in records: 

571 self.evidence_store.append_jsonl( 

572 f"parameter_probes/rank{self.rank}.jsonl", 

573 record, 

574 ) 

575 

576 

577__all__ = [ 

578 "InputIdentityRecorder", 

579 "ParameterProbeRecorder", 

580]