Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / case_compare.py: 85%

327 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Comparators across validation-case evidence directories.""" 

16 

17from __future__ import annotations 

18 

19import json 

20import math 

21import statistics 

22from pathlib import Path 

23from typing import Any, Iterable, Mapping, Optional 

24 

25import torch 

26 

27from hyper_parallel.tools.model_integration.schemas import tolerance_value 

28 

29 

30def _optional_tolerance(tolerance: Mapping[str, Any], name: str) -> Optional[float]: 

31 """Return one declared tolerance without inventing an absent zero bound.""" 

32 value = tolerance.get(name) 

33 return None if value is None else float(value) 

34 

35 

36def _relative_error(reference: float, candidate: float) -> float: 

37 """Return reference-relative error with a finite zero-safe denominator.""" 

38 return abs(candidate - reference) / max(abs(reference), 1.0e-30) 

39 

40 

41def _combine_tolerance_checks( 

42 checks: Iterable[bool], 

43 combination: Any, 

44 *, 

45 owner: str, 

46) -> bool: 

47 """Combine explicitly declared numerical checks with validated semantics.""" 

48 if combination not in ("all", "any"): 

49 raise ValueError( 

50 f"{owner} tolerance combination must be 'all' or 'any', got " 

51 f"{combination!r}" 

52 ) 

53 resolved_checks = tuple(checks) 

54 if not resolved_checks: 

55 raise ValueError(f"{owner} tolerance must declare at least one numerical bound") 

56 return all(resolved_checks) if combination == "all" else any(resolved_checks) 

57 

58 

59def _scalar_metric_delta( 

60 reference: float, 

61 candidate: float, 

62 tolerance: Mapping[str, Any], 

63 prefix: str, 

64) -> dict[str, Any]: 

65 """Compare one scalar using explicitly declared absolute/relative bounds.""" 

66 max_abs = abs(candidate - reference) 

67 max_rel = _relative_error(reference, candidate) 

68 max_abs_limit = _optional_tolerance(tolerance, f"{prefix}_max_abs") 

69 max_rel_limit = _optional_tolerance(tolerance, f"{prefix}_max_rel") 

70 if max_abs_limit is None and max_rel_limit is None: 

71 # Preserve the schema-v1 contract: an omitted scalar policy means 

72 # exact absolute equality, rather than silently disabling the gate. 

73 max_abs_limit = 0.0 

74 checks = [] 

75 if max_abs_limit is not None: 

76 checks.append(max_abs <= max_abs_limit) 

77 if max_rel_limit is not None: 

78 checks.append(max_rel <= max_rel_limit) 

79 combination = tolerance.get( 

80 f"{prefix}_combination", 

81 tolerance.get("combination", "all"), 

82 ) 

83 return { 

84 "status": _combine_tolerance_checks( 

85 checks, 

86 combination, 

87 owner=f"{prefix} scalar", 

88 ), 

89 "max_abs": max_abs, 

90 "max_rel": max_rel, 

91 "combination": combination, 

92 } 

93 

94 

95def _align_scalar_rows( 

96 baseline: list[Mapping[str, Any]], 

97 candidate: list[Mapping[str, Any]], 

98 allow_candidate_subset: bool, 

99) -> tuple[list[Mapping[str, Any]], Optional[dict[str, Any]]]: 

100 """Align candidate rows with baseline steps or explain why not.""" 

101 if allow_candidate_subset: 

102 baseline_by_step = {row.get("step"): row for row in baseline} 

103 missing_steps = [row.get("step") for row in candidate if row.get("step") not in baseline_by_step] 

104 if missing_steps: 

105 return baseline, { 

106 "status": "FAIL", 

107 "reason": f"resume steps are absent from uninterrupted baseline: {missing_steps}", 

108 } 

109 return [baseline_by_step[row.get("step")] for row in candidate], None 

110 if len(baseline) != len(candidate): 

111 return baseline, { 

112 "status": "FAIL", 

113 "reason": f"step count differs: baseline={len(baseline)}, candidate={len(candidate)}", 

114 } 

115 return baseline, None 

116 

117 

118def _optional_pair_equal( 

119 baseline_row: Mapping[str, Any], 

120 candidate_row: Mapping[str, Any], 

121 field: str, 

122) -> bool: 

123 """Compare one optional field, treating absent values as unequal.""" 

124 baseline_value = baseline_row.get(field) 

125 candidate_value = candidate_row.get(field) 

126 return ( 

127 baseline_value is not None 

128 and candidate_value is not None 

129 and baseline_value == candidate_value 

130 ) 

131 

132 

133def _post_clip_norm_delta( 

134 baseline_row: Mapping[str, Any], 

135 candidate_row: Mapping[str, Any], 

136 tolerance: Mapping[str, Any], 

137) -> tuple[Optional[float], Optional[float], bool]: 

138 """Compare optional post-clip norms only when both sides recorded them.""" 

139 baseline_post_norm = baseline_row.get("grad_norm_post_clip") 

140 candidate_post_norm = candidate_row.get("grad_norm_post_clip") 

141 if baseline_post_norm is None or candidate_post_norm is None: 

142 return None, None, baseline_post_norm is None and candidate_post_norm is None 

143 comparison = _scalar_metric_delta( 

144 float(baseline_post_norm), 

145 float(candidate_post_norm), 

146 tolerance, 

147 "norm", 

148 ) 

149 return comparison["max_abs"], comparison["max_rel"], comparison["status"] 

150 

151 

152def _compare_scalar_row( 

153 baseline_row: Mapping[str, Any], 

154 candidate_row: Mapping[str, Any], 

155 tolerance: Mapping[str, Any], 

156) -> dict[str, Any]: 

157 """Compare one aligned step row against the declared scalar tolerances.""" 

158 baseline_loss = float(baseline_row["loss"]) 

159 candidate_loss = float(candidate_row["loss"]) 

160 baseline_norm = float(baseline_row["grad_norm_pre_clip"]) 

161 candidate_norm = float(candidate_row["grad_norm_pre_clip"]) 

162 loss_delta = _scalar_metric_delta(baseline_loss, candidate_loss, tolerance, "loss") 

163 norm_delta = _scalar_metric_delta(baseline_norm, candidate_norm, tolerance, "norm") 

164 input_equal = _optional_pair_equal(baseline_row, candidate_row, "global_input_sha256") 

165 learning_rate_equal = _optional_pair_equal(baseline_row, candidate_row, "lr") 

166 step_equal = _optional_pair_equal(baseline_row, candidate_row, "step") 

167 post_norm_delta, post_norm_relative_delta, post_norm_equal = _post_clip_norm_delta( 

168 baseline_row, 

169 candidate_row, 

170 tolerance, 

171 ) 

172 finite = all( 

173 math.isfinite(float(value)) 

174 for value in ( 

175 baseline_row["loss"], 

176 candidate_row["loss"], 

177 baseline_row["grad_norm_pre_clip"], 

178 candidate_row["grad_norm_pre_clip"], 

179 ) 

180 ) 

181 numerical_pass = ( 

182 loss_delta["status"] 

183 and norm_delta["status"] 

184 and post_norm_equal 

185 and learning_rate_equal 

186 and step_equal 

187 ) 

188 if not finite: 

189 row_status = "FAIL" 

190 elif not input_equal: 

191 # Different logical inputs make a numerical A/B verdict invalid; 

192 # the topology is not proven wrong, but this experiment is blocked. 

193 row_status = "BLOCKED" 

194 else: 

195 row_status = "PASS" if numerical_pass else "FAIL" 

196 return { 

197 "step": baseline_row.get("step"), 

198 "status": row_status, 

199 "loss_max_abs": loss_delta["max_abs"], 

200 "loss_max_rel": loss_delta["max_rel"], 

201 "loss_tolerance_combination": loss_delta["combination"], 

202 "norm_max_abs": norm_delta["max_abs"], 

203 "norm_max_rel": norm_delta["max_rel"], 

204 "norm_tolerance_combination": norm_delta["combination"], 

205 "post_clip_norm_max_abs": post_norm_delta, 

206 "post_clip_norm_max_rel": post_norm_relative_delta, 

207 "input_identity": input_equal, 

208 "learning_rate_equal": learning_rate_equal, 

209 "step_equal": step_equal, 

210 "finite": finite, 

211 } 

212 

213 

214def compare_scalar_curves( 

215 baseline_rows: Iterable[Mapping[str, Any]], 

216 candidate_rows: Iterable[Mapping[str, Any]], 

217 tolerance: Mapping[str, Any], 

218 *, 

219 allow_candidate_subset: bool = False, 

220) -> dict[str, Any]: 

221 """Compare aligned loss/norm/LR rows without parsing Trainer text logs.""" 

222 baseline = list(baseline_rows) 

223 candidate = list(candidate_rows) 

224 if not baseline or not candidate: 

225 return { 

226 "status": "BLOCKED", 

227 "reason": "structured scalar evidence is missing or empty", 

228 } 

229 baseline, alignment_error = _align_scalar_rows( 

230 baseline, 

231 candidate, 

232 allow_candidate_subset, 

233 ) 

234 if alignment_error is not None: 

235 return alignment_error 

236 comparisons = [] 

237 status = "PASS" 

238 for baseline_row, candidate_row in zip(baseline, candidate): 

239 comparison = _compare_scalar_row(baseline_row, candidate_row, tolerance) 

240 row_status = comparison["status"] 

241 if row_status == "FAIL": 

242 status = "FAIL" 

243 elif row_status == "BLOCKED" and status == "PASS": 

244 status = "BLOCKED" 

245 comparisons.append(comparison) 

246 return {"status": status, "steps": comparisons} 

247 

248 

249def load_jsonl(path: str | Path) -> list[dict[str, Any]]: 

250 """Load a structured JSON-lines evidence stream.""" 

251 input_path = Path(path) 

252 if not input_path.is_file(): 

253 return [] 

254 return [json.loads(line) for line in input_path.read_text(encoding="utf-8").splitlines() if line] 

255 

256 

257def _nested_value(value: Mapping[str, Any], path: str) -> Any: 

258 current: Any = value 

259 for name in path.split("."): 

260 if not isinstance(current, Mapping) or name not in current: 

261 return None 

262 current = current[name] 

263 return current 

264 

265 

266def compare_preflight_configs( 

267 baseline_dir: str | Path, 

268 candidate_dir: str | Path, 

269) -> dict[str, Any]: 

270 """Prove that topology cases retain one training/precision contract.""" 

271 baseline_path = Path(baseline_dir) / "trainer_config.resolved.json" 

272 candidate_path = Path(candidate_dir) / "trainer_config.resolved.json" 

273 if not baseline_path.is_file() or not candidate_path.is_file(): 

274 return {"status": "BLOCKED", "reason": "resolved Trainer config evidence is missing"} 

275 baseline = json.loads(baseline_path.read_text(encoding="utf-8")) 

276 candidate = json.loads(candidate_path.read_text(encoding="utf-8")) 

277 invariant_paths = ( 

278 "model_init_dtype", 

279 "training.global_batch_size", 

280 "training.max_grad_norm", 

281 "training.loss_aggregation", 

282 "training.seed", 

283 "fsdp_config.mix_precision.param_dtype", 

284 "fsdp_config.mix_precision.reduce_dtype", 

285 "fsdp_config.mix_precision.output_dtype", 

286 "fsdp_config.mix_precision.cast_forward_inputs", 

287 "optimizer.fp32_main_params", 

288 ) 

289 mismatches = [ 

290 { 

291 "path": path, 

292 "baseline": _nested_value(baseline, path), 

293 "candidate": _nested_value(candidate, path), 

294 } 

295 for path in invariant_paths 

296 if _nested_value(baseline, path) != _nested_value(candidate, path) 

297 ] 

298 return { 

299 "status": "FAIL" if mismatches else "PASS", 

300 "mismatches": mismatches, 

301 } 

302 

303 

304def _probe_rows(case_dir: Path) -> list[dict[str, Any]]: 

305 candidates = tuple(case_dir.glob("parameter_probes/rank*.jsonl")) 

306 if not candidates: 

307 candidates = tuple(case_dir.glob("**/parameter_probes/rank*.jsonl")) 

308 return [row for path in sorted(candidates) for row in load_jsonl(path)] 

309 

310 

311def _numeric_summary_delta( 

312 baseline: Mapping[str, Any], 

313 candidate: Mapping[str, Any], 

314 tolerance: Mapping[str, Any], 

315) -> dict[str, Any]: 

316 max_abs_limit = tolerance_value(tolerance, "max_abs", "atol") 

317 relative_l2_limit = tolerance_value(tolerance, "relative_l2", "rtol") 

318 tolerance_combination = tolerance.get("combination", "all") 

319 baseline_values = baseline.get("values") 

320 candidate_values = candidate.get("values") 

321 exact_values = baseline_values is not None and candidate_values is not None 

322 summary_policy_source = None 

323 if exact_values: 

324 baseline_tensor = torch.tensor(baseline_values, dtype=torch.float64) 

325 candidate_tensor = torch.tensor(candidate_values, dtype=torch.float64) 

326 difference = candidate_tensor - baseline_tensor 

327 max_abs = float(difference.abs().max()) if difference.numel() else 0.0 

328 denominator = max( 

329 float(torch.linalg.vector_norm(baseline_tensor)), # pylint: disable=not-callable 

330 1.0e-30, 

331 ) 

332 relative_l2 = ( 

333 float(torch.linalg.vector_norm(difference)) # pylint: disable=not-callable 

334 / denominator 

335 ) 

336 within_tolerance = _combine_tolerance_checks( 

337 (max_abs <= max_abs_limit, relative_l2 <= relative_l2_limit), 

338 tolerance_combination, 

339 owner="parameter", 

340 ) 

341 comparison_mode = "exact_values" 

342 l2_norm_max_abs = None 

343 l2_norm_relative_error = None 

344 aggregate_deltas = None 

345 else: 

346 max_abs = None 

347 relative_l2 = None 

348 comparison_mode = "aggregate_summary" 

349 baseline_l2 = float(baseline.get("l2") or 0.0) 

350 candidate_l2 = float(candidate.get("l2") or 0.0) 

351 l2_norm_max_abs = abs(baseline_l2 - candidate_l2) 

352 l2_norm_relative_error = _relative_error(baseline_l2, candidate_l2) 

353 aggregate_deltas = { 

354 name: abs(float(baseline.get(name) or 0.0) - float(candidate.get(name) or 0.0)) 

355 for name in ("sum", "min", "max") 

356 } 

357 summary_tolerance = tolerance.get("summary", {}) 

358 if not isinstance(summary_tolerance, Mapping): 

359 raise ValueError("parameter summary tolerance must be a mapping") 

360 summary_relative_limit = _optional_tolerance( 

361 summary_tolerance, 

362 "l2_norm_relative", 

363 ) 

364 summary_policy_source = "summary.l2_norm_relative" 

365 if summary_relative_limit is None: 

366 summary_relative_limit = relative_l2_limit 

367 summary_policy_source = "parameters.relative_l2 (compatibility fallback)" 

368 summary_abs_limit = _optional_tolerance( 

369 summary_tolerance, 

370 "l2_norm_max_abs", 

371 ) 

372 summary_checks = [] 

373 if summary_abs_limit is not None: 

374 summary_checks.append(l2_norm_max_abs <= summary_abs_limit) 

375 if summary_relative_limit is not None: 

376 summary_checks.append(l2_norm_relative_error <= summary_relative_limit) 

377 summary_combination = summary_tolerance.get("combination", "all") 

378 within_tolerance = _combine_tolerance_checks( 

379 summary_checks, 

380 summary_combination, 

381 owner="parameter summary", 

382 ) 

383 tolerance_combination = summary_combination 

384 layout_equal = baseline.get("layout", {}).get("global_shape") == candidate.get( 

385 "layout", {} 

386 ).get("global_shape") 

387 finite = bool(baseline.get("finite")) and bool(candidate.get("finite")) 

388 status = "PASS" if finite and layout_equal and within_tolerance else "FAIL" 

389 result = { 

390 "status": status, 

391 "comparison_mode": comparison_mode, 

392 "max_abs": max_abs, 

393 "relative_l2": relative_l2, 

394 "l2_norm_max_abs": l2_norm_max_abs, 

395 "l2_norm_relative_error": l2_norm_relative_error, 

396 "aggregate_deltas": aggregate_deltas, 

397 "global_shape_equal": layout_equal, 

398 "finite": finite, 

399 "full_value_comparison": exact_values, 

400 "tolerance_combination": tolerance_combination, 

401 } 

402 if summary_policy_source is not None: 

403 result["summary_policy_source"] = summary_policy_source 

404 return result 

405 

406 

407def _first_rank_rows( 

408 rows: Iterable[Mapping[str, Any]], 

409) -> dict[tuple[Any, ...], Mapping[str, Any]]: 

410 """Index one canonical row per step, stage, and parameter.""" 

411 indexed = {} 

412 for row in rows: 

413 key = (row.get("step"), row.get("stage"), row.get("name")) 

414 if key not in indexed or int(row.get("rank", 0)) < int(indexed[key].get("rank", 0)): 

415 indexed[key] = row 

416 return indexed 

417 

418 

419def _zero_optimizer_state(row: Optional[Mapping[str, Any]]) -> bool: 

420 """Treat a load-primed zero state as equivalent to an absent lazy state.""" 

421 states = None if row is None else row.get("optimizer_state") 

422 return bool(states) and all( 

423 bool(summary.get("finite")) and float(summary.get("l2") or 0.0) == 0.0 

424 for summary in states.values() 

425 ) 

426 

427 

428def _unmatched_probe_keys( 

429 baseline: Mapping[tuple[Any, ...], Mapping[str, Any]], 

430 candidate: Mapping[tuple[Any, ...], Mapping[str, Any]], 

431 allow_candidate_subset: bool, 

432) -> set[tuple[Any, ...]]: 

433 """Diff the canonical key spaces of both evidence directories.""" 

434 if allow_candidate_subset: 

435 candidate_steps = {key[0] for key in candidate} 

436 expected_baseline = {key for key in baseline if key[0] in candidate_steps} 

437 return expected_baseline ^ set(candidate) 

438 return set(baseline) ^ set(candidate) 

439 

440 

441def _compare_optimizer_state_probes( 

442 key: tuple[Any, ...], 

443 baseline_row: Mapping[str, Any], 

444 candidate_row: Mapping[str, Any], 

445 tolerance: Mapping[str, Any], 

446) -> list[dict[str, Any]]: 

447 """Compare every declared optimizer-state summary for one parameter.""" 

448 baseline_states = baseline_row.get("optimizer_state", {}) 

449 candidate_states = candidate_row.get("optimizer_state", {}) 

450 comparisons = [] 

451 for state_name in sorted(set(baseline_states) | set(candidate_states)): 

452 if state_name not in baseline_states or state_name not in candidate_states: 

453 comparisons.append( 

454 {"key": (*key, state_name), "status": "FAIL", "reason": "state missing"} 

455 ) 

456 continue 

457 comparison = _numeric_summary_delta( 

458 baseline_states[state_name], 

459 candidate_states[state_name], 

460 tolerance, 

461 ) 

462 comparison["key"] = (*key, state_name) 

463 comparisons.append(comparison) 

464 return comparisons 

465 

466 

467def _compare_parameter_probe_row( 

468 key: tuple[Any, ...], 

469 baseline_row: Mapping[str, Any], 

470 candidate_row: Mapping[str, Any], 

471 tolerance: Mapping[str, Any], 

472) -> dict[str, Any]: 

473 """Compare one parameter lifecycle row including hash and replica gates.""" 

474 comparison = _numeric_summary_delta(baseline_row, candidate_row, tolerance) 

475 comparison["key"] = key 

476 if baseline_row.get("comparison") == "exact_hash": 

477 comparison["hash_equal"] = baseline_row.get("sha256") == candidate_row.get("sha256") 

478 if not comparison["hash_equal"]: 

479 comparison["status"] = "FAIL" 

480 baseline_replica = baseline_row.get("replica", {}).get("replicas_equal", True) 

481 candidate_replica = candidate_row.get("replica", {}).get("replicas_equal", True) 

482 comparison["replicas_equal"] = baseline_replica and candidate_replica 

483 if not comparison["replicas_equal"]: 

484 comparison["status"] = "FAIL" 

485 return comparison 

486 

487 

488def compare_parameter_probes( 

489 baseline_dir: str | Path, 

490 candidate_dir: str | Path, 

491 tolerance: Mapping[str, Any], 

492 *, 

493 allow_candidate_subset: bool = False, 

494 compare_optimizer_states: bool = True, 

495) -> dict[str, Any]: 

496 """Compare canonical parameter lifecycle summaries from two topologies. 

497 

498 Optimizer internals such as Muon momentum may be topology-local across HSDP 

499 replica groups even when synchronized gradients and model updates are 

500 identical. Cross-topology cases therefore validate their live layouts and 

501 checkpoint recovery, but reserve numerical state comparison for matching 

502 topologies. 

503 """ 

504 baseline_rows = _probe_rows(Path(baseline_dir)) 

505 candidate_rows = _probe_rows(Path(candidate_dir)) 

506 if not baseline_rows or not candidate_rows: 

507 return { 

508 "status": "BLOCKED", 

509 "reason": "strict parameter probe evidence is missing", 

510 } 

511 baseline = _first_rank_rows(baseline_rows) 

512 candidate = _first_rank_rows(candidate_rows) 

513 unmatched = _unmatched_probe_keys(baseline, candidate, allow_candidate_subset) 

514 zero_initialized = sorted( 

515 ( 

516 key 

517 for key in unmatched 

518 if _zero_optimizer_state(baseline.get(key)) 

519 or _zero_optimizer_state(candidate.get(key)) 

520 ), 

521 key=repr, 

522 ) 

523 missing = sorted(set(unmatched) - set(zero_initialized), key=repr) 

524 comparisons = [] 

525 skipped_optimizer_states = 0 

526 for key in sorted(set(baseline).intersection(candidate), key=repr): 

527 baseline_row = baseline[key] 

528 candidate_row = candidate[key] 

529 if "optimizer_state" in baseline_row or "optimizer_state" in candidate_row: 

530 if not compare_optimizer_states: 

531 skipped_optimizer_states += len( 

532 set(baseline_row.get("optimizer_state", {})) 

533 | set(candidate_row.get("optimizer_state", {})) 

534 ) 

535 continue 

536 comparisons.extend( 

537 _compare_optimizer_state_probes( 

538 key, 

539 baseline_row, 

540 candidate_row, 

541 tolerance, 

542 ) 

543 ) 

544 continue 

545 comparisons.append( 

546 _compare_parameter_probe_row(key, baseline_row, candidate_row, tolerance) 

547 ) 

548 status = "FAIL" if missing or any(row["status"] == "FAIL" for row in comparisons) else "PASS" 

549 return { 

550 "status": status, 

551 "missing": [repr(key) for key in missing], 

552 "zero_initialized_optimizer_states": [repr(key) for key in zero_initialized], 

553 "optimizer_state_numeric_comparison": ( 

554 "enabled" if compare_optimizer_states else "skipped_cross_topology" 

555 ), 

556 "skipped_optimizer_states": skipped_optimizer_states, 

557 "values": comparisons, 

558 } 

559 

560 

561def summarize_performance( 

562 rows: Iterable[Mapping[str, Any]], 

563 warmup_steps: int = 1, 

564 memory_rows: Optional[Iterable[Mapping[str, Any]]] = None, 

565) -> dict[str, Any]: 

566 """Summarize steady-state wall time and throughput independently of precision.""" 

567 values = list(rows)[warmup_steps:] 

568 step_times = sorted(float(row["step_time_seconds"]) for row in values) 

569 if not step_times: 

570 return {"status": "BLOCKED", "reason": "no steady-state performance steps"} 

571 p90_index = min(math.ceil(0.9 * len(step_times)) - 1, len(step_times) - 1) 

572 token_rates = [ 

573 float(row["tokens_per_second"]) 

574 for row in values 

575 if row.get("tokens_per_second") is not None 

576 ] 

577 sample_rates = [ 

578 float(row["samples_per_second"]) 

579 for row in values 

580 if row.get("samples_per_second") is not None 

581 ] 

582 allocator_rows = list(memory_rows) if memory_rows is not None else values 

583 peak_allocated = [ 

584 int(row["peak_memory_allocated_bytes"]) 

585 for row in allocator_rows 

586 if row.get("peak_memory_allocated_bytes") is not None 

587 ] 

588 peak_reserved = [ 

589 int(row["peak_memory_reserved_bytes"]) 

590 for row in allocator_rows 

591 if row.get("peak_memory_reserved_bytes") is not None 

592 ] 

593 return { 

594 "status": "MEASURED", 

595 "steady_steps": len(step_times), 

596 "step_time_p50_seconds": statistics.median(step_times), 

597 "step_time_p90_seconds": step_times[p90_index], 

598 "tokens_per_second_p50": statistics.median(token_rates) if token_rates else None, 

599 "samples_per_second_p50": statistics.median(sample_rates) if sample_rates else None, 

600 "peak_memory_allocated_bytes": max(peak_allocated) if peak_allocated else None, 

601 "peak_memory_reserved_bytes": max(peak_reserved) if peak_reserved else None, 

602 } 

603 

604 

605def _group_checkpoint_events(files: list[Path]) -> dict[str, list[Path]]: 

606 """Group every rank file by its versioned checkpoint event name.""" 

607 grouped: dict[str, list[Path]] = {} 

608 for path in files: 

609 event_name = path.name.rsplit("_rank", 1)[0] 

610 grouped.setdefault(event_name, []).append(path) 

611 return grouped 

612 

613 

614def _checkpoint_event_order(event_name: str) -> tuple[int, str]: 

615 """Sort numbered checkpoint events after legacy unnumbered events.""" 

616 suffix = event_name.rsplit("_", 1)[-1] 

617 return (int(suffix), event_name) if suffix.isdigit() else (-1, event_name) 

618 

619 

620def _latest_checkpoint_event(events: Mapping[str, list[Path]], preferred: str) -> str: 

621 """Choose the newest numbered event, retaining legacy-name support.""" 

622 matching = [ 

623 event_name 

624 for event_name in events 

625 if event_name == preferred or event_name.startswith(f"{preferred}_") 

626 ] 

627 if not matching: 

628 return sorted(events)[-1] 

629 return max(matching, key=_checkpoint_event_order) 

630 

631 

632def _load_rank_layouts(files: list[Path]) -> list[dict[str, Any]]: 

633 """Load checkpoint tensor-layout maps for all selected ranks.""" 

634 return [ 

635 json.loads(path.read_text(encoding="utf-8")).get("tensor_layouts", {}) 

636 for path in files 

637 ] 

638 

639 

640def _missing_layout_names( 

641 baseline: Mapping[str, Any], 

642 candidate: Mapping[str, Any], 

643) -> tuple[list[str], list[str]]: 

644 """List one-sided keys, excluding lazily initialized optimizer state.""" 

645 baseline_only = set(baseline) - set(candidate) 

646 candidate_only = set(candidate) - set(baseline) 

647 initialized_optimizer_entries = sorted( 

648 name for name in candidate_only if name.startswith("optimizer.") 

649 ) 

650 missing = sorted( 

651 baseline_only 

652 | { 

653 name 

654 for name in candidate_only 

655 if name not in initialized_optimizer_entries 

656 } 

657 ) 

658 return missing, initialized_optimizer_entries 

659 

660 

661def _layout_mismatches( 

662 baseline: Mapping[str, Any], 

663 candidate: Mapping[str, Any], 

664 same_topology: bool, 

665) -> list[dict[str, Any]]: 

666 """Diff shared tensor layouts over the topology-appropriate field set.""" 

667 fields = ["global_shape"] 

668 if same_topology: 

669 fields.extend(("mesh_dim_names", "mesh_shape", "placements")) 

670 mismatches = [] 

671 for name in sorted(set(baseline).intersection(candidate)): 

672 baseline_layout = baseline[name] 

673 candidate_layout = candidate[name] 

674 differences = { 

675 field: { 

676 "baseline": baseline_layout.get(field), 

677 "candidate": candidate_layout.get(field), 

678 } 

679 for field in fields 

680 if baseline_layout.get(field) != candidate_layout.get(field) 

681 } 

682 if differences: 

683 mismatches.append({"name": name, "differences": differences}) 

684 return mismatches 

685 

686 

687def _rank_consistency_inconsistencies( 

688 side_name: str, 

689 rank_layouts: list[dict[str, Any]], 

690) -> list[dict[str, Any]]: 

691 """Flag keys whose rank-local payloads disagree on presence or shape.""" 

692 inconsistencies = [] 

693 all_names = set().union(*(set(layouts) for layouts in rank_layouts)) 

694 for name in sorted(all_names): 

695 values = [layouts.get(name) for layouts in rank_layouts] 

696 global_shapes = { 

697 tuple(value.get("global_shape", ())) 

698 for value in values 

699 if value is not None 

700 } 

701 if any(value is None for value in values) or len(global_shapes) != 1: 

702 inconsistencies.append( 

703 { 

704 "side": side_name, 

705 "name": name, 

706 "reason": "rank-local payloads disagree on key presence or global shape", 

707 } 

708 ) 

709 return inconsistencies 

710 

711 

712def _same_topology_rank_inconsistencies( 

713 baseline_ranks: list[dict[str, Any]], 

714 candidate_ranks: list[dict[str, Any]], 

715) -> list[dict[str, Any]]: 

716 """Require equal rank counts and stable local shard shapes.""" 

717 inconsistencies = [] 

718 if len(baseline_ranks) != len(candidate_ranks): 

719 inconsistencies.append( 

720 { 

721 "side": "comparison", 

722 "reason": "same-topology checkpoint evidence has different rank counts", 

723 "baseline_ranks": len(baseline_ranks), 

724 "candidate_ranks": len(candidate_ranks), 

725 } 

726 ) 

727 return inconsistencies 

728 for rank, (baseline_layouts, candidate_layouts) in enumerate( 

729 zip(baseline_ranks, candidate_ranks) 

730 ): 

731 for name in sorted(set(baseline_layouts).intersection(candidate_layouts)): 

732 if baseline_layouts[name].get("local_shape") != candidate_layouts[name].get( 

733 "local_shape" 

734 ): 

735 inconsistencies.append( 

736 { 

737 "side": "comparison", 

738 "rank": rank, 

739 "name": name, 

740 "reason": "same-topology local shard shape changed", 

741 } 

742 ) 

743 return inconsistencies 

744 

745 

746def compare_checkpoint_layouts( 

747 baseline_dir: str | Path, 

748 candidate_dir: str | Path, 

749 *, 

750 same_topology: bool, 

751) -> dict[str, Any]: 

752 """Compare DCP-bound logical layouts while allowing legal resharding.""" 

753 baseline_files = sorted(Path(baseline_dir).glob("**/checkpoint/*_rank*.json")) 

754 candidate_files = sorted(Path(candidate_dir).glob("**/checkpoint/*_rank*.json")) 

755 if not baseline_files or not candidate_files: 

756 return {"status": "BLOCKED", "reason": "checkpoint layout evidence is missing"} 

757 baseline_events = _group_checkpoint_events(baseline_files) 

758 candidate_events = _group_checkpoint_events(candidate_files) 

759 candidate_event = _latest_checkpoint_event(candidate_events, "after_load") 

760 checkpoint_identity = candidate_event.removeprefix("after_load").lstrip("_") 

761 baseline_event = ( 

762 f"before_save_{checkpoint_identity}" 

763 if checkpoint_identity 

764 else _latest_checkpoint_event(baseline_events, "before_save") 

765 ) 

766 if baseline_event not in baseline_events: 

767 return { 

768 "status": "BLOCKED", 

769 "reason": "baseline checkpoint evidence for the restored checkpoint is missing", 

770 "baseline_event": baseline_event, 

771 "candidate_event": candidate_event, 

772 } 

773 baseline_ranks = _load_rank_layouts(baseline_events[baseline_event]) 

774 candidate_ranks = _load_rank_layouts(candidate_events[candidate_event]) 

775 baseline = baseline_ranks[0] 

776 candidate = candidate_ranks[0] 

777 missing, initialized_optimizer_entries = _missing_layout_names(baseline, candidate) 

778 mismatches = _layout_mismatches(baseline, candidate, same_topology) 

779 rank_inconsistencies = [] 

780 for side_name, rank_layouts in ( 

781 ("baseline", baseline_ranks), 

782 ("candidate", candidate_ranks), 

783 ): 

784 rank_inconsistencies.extend( 

785 _rank_consistency_inconsistencies(side_name, rank_layouts) 

786 ) 

787 if same_topology: 

788 rank_inconsistencies.extend( 

789 _same_topology_rank_inconsistencies(baseline_ranks, candidate_ranks) 

790 ) 

791 return { 

792 "status": "FAIL" if missing or mismatches or rank_inconsistencies else "PASS", 

793 "missing": missing, 

794 "mismatches": mismatches, 

795 "rank_inconsistencies": rank_inconsistencies, 

796 "baseline_rank_count": len(baseline_ranks), 

797 "candidate_rank_count": len(candidate_ranks), 

798 "baseline_event": baseline_event, 

799 "candidate_event": candidate_event, 

800 "initialized_optimizer_entries": initialized_optimizer_entries, 

801 } 

802 

803 

804__all__ = [ 

805 "compare_checkpoint_layouts", 

806 "compare_parameter_probes", 

807 "compare_preflight_configs", 

808 "compare_scalar_curves", 

809 "load_jsonl", 

810 "summarize_performance", 

811]