Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / case_compare.py: 85%
327 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Comparators across validation-case evidence directories."""
17from __future__ import annotations
19import json
20import math
21import statistics
22from pathlib import Path
23from typing import Any, Iterable, Mapping, Optional
25import torch
27from hyper_parallel.tools.model_integration.schemas import tolerance_value
30def _optional_tolerance(tolerance: Mapping[str, Any], name: str) -> Optional[float]:
31 """Return one declared tolerance without inventing an absent zero bound."""
32 value = tolerance.get(name)
33 return None if value is None else float(value)
36def _relative_error(reference: float, candidate: float) -> float:
37 """Return reference-relative error with a finite zero-safe denominator."""
38 return abs(candidate - reference) / max(abs(reference), 1.0e-30)
41def _combine_tolerance_checks(
42 checks: Iterable[bool],
43 combination: Any,
44 *,
45 owner: str,
46) -> bool:
47 """Combine explicitly declared numerical checks with validated semantics."""
48 if combination not in ("all", "any"):
49 raise ValueError(
50 f"{owner} tolerance combination must be 'all' or 'any', got "
51 f"{combination!r}"
52 )
53 resolved_checks = tuple(checks)
54 if not resolved_checks:
55 raise ValueError(f"{owner} tolerance must declare at least one numerical bound")
56 return all(resolved_checks) if combination == "all" else any(resolved_checks)
59def _scalar_metric_delta(
60 reference: float,
61 candidate: float,
62 tolerance: Mapping[str, Any],
63 prefix: str,
64) -> dict[str, Any]:
65 """Compare one scalar using explicitly declared absolute/relative bounds."""
66 max_abs = abs(candidate - reference)
67 max_rel = _relative_error(reference, candidate)
68 max_abs_limit = _optional_tolerance(tolerance, f"{prefix}_max_abs")
69 max_rel_limit = _optional_tolerance(tolerance, f"{prefix}_max_rel")
70 if max_abs_limit is None and max_rel_limit is None:
71 # Preserve the schema-v1 contract: an omitted scalar policy means
72 # exact absolute equality, rather than silently disabling the gate.
73 max_abs_limit = 0.0
74 checks = []
75 if max_abs_limit is not None:
76 checks.append(max_abs <= max_abs_limit)
77 if max_rel_limit is not None:
78 checks.append(max_rel <= max_rel_limit)
79 combination = tolerance.get(
80 f"{prefix}_combination",
81 tolerance.get("combination", "all"),
82 )
83 return {
84 "status": _combine_tolerance_checks(
85 checks,
86 combination,
87 owner=f"{prefix} scalar",
88 ),
89 "max_abs": max_abs,
90 "max_rel": max_rel,
91 "combination": combination,
92 }
95def _align_scalar_rows(
96 baseline: list[Mapping[str, Any]],
97 candidate: list[Mapping[str, Any]],
98 allow_candidate_subset: bool,
99) -> tuple[list[Mapping[str, Any]], Optional[dict[str, Any]]]:
100 """Align candidate rows with baseline steps or explain why not."""
101 if allow_candidate_subset:
102 baseline_by_step = {row.get("step"): row for row in baseline}
103 missing_steps = [row.get("step") for row in candidate if row.get("step") not in baseline_by_step]
104 if missing_steps:
105 return baseline, {
106 "status": "FAIL",
107 "reason": f"resume steps are absent from uninterrupted baseline: {missing_steps}",
108 }
109 return [baseline_by_step[row.get("step")] for row in candidate], None
110 if len(baseline) != len(candidate):
111 return baseline, {
112 "status": "FAIL",
113 "reason": f"step count differs: baseline={len(baseline)}, candidate={len(candidate)}",
114 }
115 return baseline, None
118def _optional_pair_equal(
119 baseline_row: Mapping[str, Any],
120 candidate_row: Mapping[str, Any],
121 field: str,
122) -> bool:
123 """Compare one optional field, treating absent values as unequal."""
124 baseline_value = baseline_row.get(field)
125 candidate_value = candidate_row.get(field)
126 return (
127 baseline_value is not None
128 and candidate_value is not None
129 and baseline_value == candidate_value
130 )
133def _post_clip_norm_delta(
134 baseline_row: Mapping[str, Any],
135 candidate_row: Mapping[str, Any],
136 tolerance: Mapping[str, Any],
137) -> tuple[Optional[float], Optional[float], bool]:
138 """Compare optional post-clip norms only when both sides recorded them."""
139 baseline_post_norm = baseline_row.get("grad_norm_post_clip")
140 candidate_post_norm = candidate_row.get("grad_norm_post_clip")
141 if baseline_post_norm is None or candidate_post_norm is None:
142 return None, None, baseline_post_norm is None and candidate_post_norm is None
143 comparison = _scalar_metric_delta(
144 float(baseline_post_norm),
145 float(candidate_post_norm),
146 tolerance,
147 "norm",
148 )
149 return comparison["max_abs"], comparison["max_rel"], comparison["status"]
152def _compare_scalar_row(
153 baseline_row: Mapping[str, Any],
154 candidate_row: Mapping[str, Any],
155 tolerance: Mapping[str, Any],
156) -> dict[str, Any]:
157 """Compare one aligned step row against the declared scalar tolerances."""
158 baseline_loss = float(baseline_row["loss"])
159 candidate_loss = float(candidate_row["loss"])
160 baseline_norm = float(baseline_row["grad_norm_pre_clip"])
161 candidate_norm = float(candidate_row["grad_norm_pre_clip"])
162 loss_delta = _scalar_metric_delta(baseline_loss, candidate_loss, tolerance, "loss")
163 norm_delta = _scalar_metric_delta(baseline_norm, candidate_norm, tolerance, "norm")
164 input_equal = _optional_pair_equal(baseline_row, candidate_row, "global_input_sha256")
165 learning_rate_equal = _optional_pair_equal(baseline_row, candidate_row, "lr")
166 step_equal = _optional_pair_equal(baseline_row, candidate_row, "step")
167 post_norm_delta, post_norm_relative_delta, post_norm_equal = _post_clip_norm_delta(
168 baseline_row,
169 candidate_row,
170 tolerance,
171 )
172 finite = all(
173 math.isfinite(float(value))
174 for value in (
175 baseline_row["loss"],
176 candidate_row["loss"],
177 baseline_row["grad_norm_pre_clip"],
178 candidate_row["grad_norm_pre_clip"],
179 )
180 )
181 numerical_pass = (
182 loss_delta["status"]
183 and norm_delta["status"]
184 and post_norm_equal
185 and learning_rate_equal
186 and step_equal
187 )
188 if not finite:
189 row_status = "FAIL"
190 elif not input_equal:
191 # Different logical inputs make a numerical A/B verdict invalid;
192 # the topology is not proven wrong, but this experiment is blocked.
193 row_status = "BLOCKED"
194 else:
195 row_status = "PASS" if numerical_pass else "FAIL"
196 return {
197 "step": baseline_row.get("step"),
198 "status": row_status,
199 "loss_max_abs": loss_delta["max_abs"],
200 "loss_max_rel": loss_delta["max_rel"],
201 "loss_tolerance_combination": loss_delta["combination"],
202 "norm_max_abs": norm_delta["max_abs"],
203 "norm_max_rel": norm_delta["max_rel"],
204 "norm_tolerance_combination": norm_delta["combination"],
205 "post_clip_norm_max_abs": post_norm_delta,
206 "post_clip_norm_max_rel": post_norm_relative_delta,
207 "input_identity": input_equal,
208 "learning_rate_equal": learning_rate_equal,
209 "step_equal": step_equal,
210 "finite": finite,
211 }
214def compare_scalar_curves(
215 baseline_rows: Iterable[Mapping[str, Any]],
216 candidate_rows: Iterable[Mapping[str, Any]],
217 tolerance: Mapping[str, Any],
218 *,
219 allow_candidate_subset: bool = False,
220) -> dict[str, Any]:
221 """Compare aligned loss/norm/LR rows without parsing Trainer text logs."""
222 baseline = list(baseline_rows)
223 candidate = list(candidate_rows)
224 if not baseline or not candidate:
225 return {
226 "status": "BLOCKED",
227 "reason": "structured scalar evidence is missing or empty",
228 }
229 baseline, alignment_error = _align_scalar_rows(
230 baseline,
231 candidate,
232 allow_candidate_subset,
233 )
234 if alignment_error is not None:
235 return alignment_error
236 comparisons = []
237 status = "PASS"
238 for baseline_row, candidate_row in zip(baseline, candidate):
239 comparison = _compare_scalar_row(baseline_row, candidate_row, tolerance)
240 row_status = comparison["status"]
241 if row_status == "FAIL":
242 status = "FAIL"
243 elif row_status == "BLOCKED" and status == "PASS":
244 status = "BLOCKED"
245 comparisons.append(comparison)
246 return {"status": status, "steps": comparisons}
249def load_jsonl(path: str | Path) -> list[dict[str, Any]]:
250 """Load a structured JSON-lines evidence stream."""
251 input_path = Path(path)
252 if not input_path.is_file():
253 return []
254 return [json.loads(line) for line in input_path.read_text(encoding="utf-8").splitlines() if line]
257def _nested_value(value: Mapping[str, Any], path: str) -> Any:
258 current: Any = value
259 for name in path.split("."):
260 if not isinstance(current, Mapping) or name not in current:
261 return None
262 current = current[name]
263 return current
266def compare_preflight_configs(
267 baseline_dir: str | Path,
268 candidate_dir: str | Path,
269) -> dict[str, Any]:
270 """Prove that topology cases retain one training/precision contract."""
271 baseline_path = Path(baseline_dir) / "trainer_config.resolved.json"
272 candidate_path = Path(candidate_dir) / "trainer_config.resolved.json"
273 if not baseline_path.is_file() or not candidate_path.is_file():
274 return {"status": "BLOCKED", "reason": "resolved Trainer config evidence is missing"}
275 baseline = json.loads(baseline_path.read_text(encoding="utf-8"))
276 candidate = json.loads(candidate_path.read_text(encoding="utf-8"))
277 invariant_paths = (
278 "model_init_dtype",
279 "training.global_batch_size",
280 "training.max_grad_norm",
281 "training.loss_aggregation",
282 "training.seed",
283 "fsdp_config.mix_precision.param_dtype",
284 "fsdp_config.mix_precision.reduce_dtype",
285 "fsdp_config.mix_precision.output_dtype",
286 "fsdp_config.mix_precision.cast_forward_inputs",
287 "optimizer.fp32_main_params",
288 )
289 mismatches = [
290 {
291 "path": path,
292 "baseline": _nested_value(baseline, path),
293 "candidate": _nested_value(candidate, path),
294 }
295 for path in invariant_paths
296 if _nested_value(baseline, path) != _nested_value(candidate, path)
297 ]
298 return {
299 "status": "FAIL" if mismatches else "PASS",
300 "mismatches": mismatches,
301 }
304def _probe_rows(case_dir: Path) -> list[dict[str, Any]]:
305 candidates = tuple(case_dir.glob("parameter_probes/rank*.jsonl"))
306 if not candidates:
307 candidates = tuple(case_dir.glob("**/parameter_probes/rank*.jsonl"))
308 return [row for path in sorted(candidates) for row in load_jsonl(path)]
311def _numeric_summary_delta(
312 baseline: Mapping[str, Any],
313 candidate: Mapping[str, Any],
314 tolerance: Mapping[str, Any],
315) -> dict[str, Any]:
316 max_abs_limit = tolerance_value(tolerance, "max_abs", "atol")
317 relative_l2_limit = tolerance_value(tolerance, "relative_l2", "rtol")
318 tolerance_combination = tolerance.get("combination", "all")
319 baseline_values = baseline.get("values")
320 candidate_values = candidate.get("values")
321 exact_values = baseline_values is not None and candidate_values is not None
322 summary_policy_source = None
323 if exact_values:
324 baseline_tensor = torch.tensor(baseline_values, dtype=torch.float64)
325 candidate_tensor = torch.tensor(candidate_values, dtype=torch.float64)
326 difference = candidate_tensor - baseline_tensor
327 max_abs = float(difference.abs().max()) if difference.numel() else 0.0
328 denominator = max(
329 float(torch.linalg.vector_norm(baseline_tensor)), # pylint: disable=not-callable
330 1.0e-30,
331 )
332 relative_l2 = (
333 float(torch.linalg.vector_norm(difference)) # pylint: disable=not-callable
334 / denominator
335 )
336 within_tolerance = _combine_tolerance_checks(
337 (max_abs <= max_abs_limit, relative_l2 <= relative_l2_limit),
338 tolerance_combination,
339 owner="parameter",
340 )
341 comparison_mode = "exact_values"
342 l2_norm_max_abs = None
343 l2_norm_relative_error = None
344 aggregate_deltas = None
345 else:
346 max_abs = None
347 relative_l2 = None
348 comparison_mode = "aggregate_summary"
349 baseline_l2 = float(baseline.get("l2") or 0.0)
350 candidate_l2 = float(candidate.get("l2") or 0.0)
351 l2_norm_max_abs = abs(baseline_l2 - candidate_l2)
352 l2_norm_relative_error = _relative_error(baseline_l2, candidate_l2)
353 aggregate_deltas = {
354 name: abs(float(baseline.get(name) or 0.0) - float(candidate.get(name) or 0.0))
355 for name in ("sum", "min", "max")
356 }
357 summary_tolerance = tolerance.get("summary", {})
358 if not isinstance(summary_tolerance, Mapping):
359 raise ValueError("parameter summary tolerance must be a mapping")
360 summary_relative_limit = _optional_tolerance(
361 summary_tolerance,
362 "l2_norm_relative",
363 )
364 summary_policy_source = "summary.l2_norm_relative"
365 if summary_relative_limit is None:
366 summary_relative_limit = relative_l2_limit
367 summary_policy_source = "parameters.relative_l2 (compatibility fallback)"
368 summary_abs_limit = _optional_tolerance(
369 summary_tolerance,
370 "l2_norm_max_abs",
371 )
372 summary_checks = []
373 if summary_abs_limit is not None:
374 summary_checks.append(l2_norm_max_abs <= summary_abs_limit)
375 if summary_relative_limit is not None:
376 summary_checks.append(l2_norm_relative_error <= summary_relative_limit)
377 summary_combination = summary_tolerance.get("combination", "all")
378 within_tolerance = _combine_tolerance_checks(
379 summary_checks,
380 summary_combination,
381 owner="parameter summary",
382 )
383 tolerance_combination = summary_combination
384 layout_equal = baseline.get("layout", {}).get("global_shape") == candidate.get(
385 "layout", {}
386 ).get("global_shape")
387 finite = bool(baseline.get("finite")) and bool(candidate.get("finite"))
388 status = "PASS" if finite and layout_equal and within_tolerance else "FAIL"
389 result = {
390 "status": status,
391 "comparison_mode": comparison_mode,
392 "max_abs": max_abs,
393 "relative_l2": relative_l2,
394 "l2_norm_max_abs": l2_norm_max_abs,
395 "l2_norm_relative_error": l2_norm_relative_error,
396 "aggregate_deltas": aggregate_deltas,
397 "global_shape_equal": layout_equal,
398 "finite": finite,
399 "full_value_comparison": exact_values,
400 "tolerance_combination": tolerance_combination,
401 }
402 if summary_policy_source is not None:
403 result["summary_policy_source"] = summary_policy_source
404 return result
407def _first_rank_rows(
408 rows: Iterable[Mapping[str, Any]],
409) -> dict[tuple[Any, ...], Mapping[str, Any]]:
410 """Index one canonical row per step, stage, and parameter."""
411 indexed = {}
412 for row in rows:
413 key = (row.get("step"), row.get("stage"), row.get("name"))
414 if key not in indexed or int(row.get("rank", 0)) < int(indexed[key].get("rank", 0)):
415 indexed[key] = row
416 return indexed
419def _zero_optimizer_state(row: Optional[Mapping[str, Any]]) -> bool:
420 """Treat a load-primed zero state as equivalent to an absent lazy state."""
421 states = None if row is None else row.get("optimizer_state")
422 return bool(states) and all(
423 bool(summary.get("finite")) and float(summary.get("l2") or 0.0) == 0.0
424 for summary in states.values()
425 )
428def _unmatched_probe_keys(
429 baseline: Mapping[tuple[Any, ...], Mapping[str, Any]],
430 candidate: Mapping[tuple[Any, ...], Mapping[str, Any]],
431 allow_candidate_subset: bool,
432) -> set[tuple[Any, ...]]:
433 """Diff the canonical key spaces of both evidence directories."""
434 if allow_candidate_subset:
435 candidate_steps = {key[0] for key in candidate}
436 expected_baseline = {key for key in baseline if key[0] in candidate_steps}
437 return expected_baseline ^ set(candidate)
438 return set(baseline) ^ set(candidate)
441def _compare_optimizer_state_probes(
442 key: tuple[Any, ...],
443 baseline_row: Mapping[str, Any],
444 candidate_row: Mapping[str, Any],
445 tolerance: Mapping[str, Any],
446) -> list[dict[str, Any]]:
447 """Compare every declared optimizer-state summary for one parameter."""
448 baseline_states = baseline_row.get("optimizer_state", {})
449 candidate_states = candidate_row.get("optimizer_state", {})
450 comparisons = []
451 for state_name in sorted(set(baseline_states) | set(candidate_states)):
452 if state_name not in baseline_states or state_name not in candidate_states:
453 comparisons.append(
454 {"key": (*key, state_name), "status": "FAIL", "reason": "state missing"}
455 )
456 continue
457 comparison = _numeric_summary_delta(
458 baseline_states[state_name],
459 candidate_states[state_name],
460 tolerance,
461 )
462 comparison["key"] = (*key, state_name)
463 comparisons.append(comparison)
464 return comparisons
467def _compare_parameter_probe_row(
468 key: tuple[Any, ...],
469 baseline_row: Mapping[str, Any],
470 candidate_row: Mapping[str, Any],
471 tolerance: Mapping[str, Any],
472) -> dict[str, Any]:
473 """Compare one parameter lifecycle row including hash and replica gates."""
474 comparison = _numeric_summary_delta(baseline_row, candidate_row, tolerance)
475 comparison["key"] = key
476 if baseline_row.get("comparison") == "exact_hash":
477 comparison["hash_equal"] = baseline_row.get("sha256") == candidate_row.get("sha256")
478 if not comparison["hash_equal"]:
479 comparison["status"] = "FAIL"
480 baseline_replica = baseline_row.get("replica", {}).get("replicas_equal", True)
481 candidate_replica = candidate_row.get("replica", {}).get("replicas_equal", True)
482 comparison["replicas_equal"] = baseline_replica and candidate_replica
483 if not comparison["replicas_equal"]:
484 comparison["status"] = "FAIL"
485 return comparison
488def compare_parameter_probes(
489 baseline_dir: str | Path,
490 candidate_dir: str | Path,
491 tolerance: Mapping[str, Any],
492 *,
493 allow_candidate_subset: bool = False,
494 compare_optimizer_states: bool = True,
495) -> dict[str, Any]:
496 """Compare canonical parameter lifecycle summaries from two topologies.
498 Optimizer internals such as Muon momentum may be topology-local across HSDP
499 replica groups even when synchronized gradients and model updates are
500 identical. Cross-topology cases therefore validate their live layouts and
501 checkpoint recovery, but reserve numerical state comparison for matching
502 topologies.
503 """
504 baseline_rows = _probe_rows(Path(baseline_dir))
505 candidate_rows = _probe_rows(Path(candidate_dir))
506 if not baseline_rows or not candidate_rows:
507 return {
508 "status": "BLOCKED",
509 "reason": "strict parameter probe evidence is missing",
510 }
511 baseline = _first_rank_rows(baseline_rows)
512 candidate = _first_rank_rows(candidate_rows)
513 unmatched = _unmatched_probe_keys(baseline, candidate, allow_candidate_subset)
514 zero_initialized = sorted(
515 (
516 key
517 for key in unmatched
518 if _zero_optimizer_state(baseline.get(key))
519 or _zero_optimizer_state(candidate.get(key))
520 ),
521 key=repr,
522 )
523 missing = sorted(set(unmatched) - set(zero_initialized), key=repr)
524 comparisons = []
525 skipped_optimizer_states = 0
526 for key in sorted(set(baseline).intersection(candidate), key=repr):
527 baseline_row = baseline[key]
528 candidate_row = candidate[key]
529 if "optimizer_state" in baseline_row or "optimizer_state" in candidate_row:
530 if not compare_optimizer_states:
531 skipped_optimizer_states += len(
532 set(baseline_row.get("optimizer_state", {}))
533 | set(candidate_row.get("optimizer_state", {}))
534 )
535 continue
536 comparisons.extend(
537 _compare_optimizer_state_probes(
538 key,
539 baseline_row,
540 candidate_row,
541 tolerance,
542 )
543 )
544 continue
545 comparisons.append(
546 _compare_parameter_probe_row(key, baseline_row, candidate_row, tolerance)
547 )
548 status = "FAIL" if missing or any(row["status"] == "FAIL" for row in comparisons) else "PASS"
549 return {
550 "status": status,
551 "missing": [repr(key) for key in missing],
552 "zero_initialized_optimizer_states": [repr(key) for key in zero_initialized],
553 "optimizer_state_numeric_comparison": (
554 "enabled" if compare_optimizer_states else "skipped_cross_topology"
555 ),
556 "skipped_optimizer_states": skipped_optimizer_states,
557 "values": comparisons,
558 }
561def summarize_performance(
562 rows: Iterable[Mapping[str, Any]],
563 warmup_steps: int = 1,
564 memory_rows: Optional[Iterable[Mapping[str, Any]]] = None,
565) -> dict[str, Any]:
566 """Summarize steady-state wall time and throughput independently of precision."""
567 values = list(rows)[warmup_steps:]
568 step_times = sorted(float(row["step_time_seconds"]) for row in values)
569 if not step_times:
570 return {"status": "BLOCKED", "reason": "no steady-state performance steps"}
571 p90_index = min(math.ceil(0.9 * len(step_times)) - 1, len(step_times) - 1)
572 token_rates = [
573 float(row["tokens_per_second"])
574 for row in values
575 if row.get("tokens_per_second") is not None
576 ]
577 sample_rates = [
578 float(row["samples_per_second"])
579 for row in values
580 if row.get("samples_per_second") is not None
581 ]
582 allocator_rows = list(memory_rows) if memory_rows is not None else values
583 peak_allocated = [
584 int(row["peak_memory_allocated_bytes"])
585 for row in allocator_rows
586 if row.get("peak_memory_allocated_bytes") is not None
587 ]
588 peak_reserved = [
589 int(row["peak_memory_reserved_bytes"])
590 for row in allocator_rows
591 if row.get("peak_memory_reserved_bytes") is not None
592 ]
593 return {
594 "status": "MEASURED",
595 "steady_steps": len(step_times),
596 "step_time_p50_seconds": statistics.median(step_times),
597 "step_time_p90_seconds": step_times[p90_index],
598 "tokens_per_second_p50": statistics.median(token_rates) if token_rates else None,
599 "samples_per_second_p50": statistics.median(sample_rates) if sample_rates else None,
600 "peak_memory_allocated_bytes": max(peak_allocated) if peak_allocated else None,
601 "peak_memory_reserved_bytes": max(peak_reserved) if peak_reserved else None,
602 }
605def _group_checkpoint_events(files: list[Path]) -> dict[str, list[Path]]:
606 """Group every rank file by its versioned checkpoint event name."""
607 grouped: dict[str, list[Path]] = {}
608 for path in files:
609 event_name = path.name.rsplit("_rank", 1)[0]
610 grouped.setdefault(event_name, []).append(path)
611 return grouped
614def _checkpoint_event_order(event_name: str) -> tuple[int, str]:
615 """Sort numbered checkpoint events after legacy unnumbered events."""
616 suffix = event_name.rsplit("_", 1)[-1]
617 return (int(suffix), event_name) if suffix.isdigit() else (-1, event_name)
620def _latest_checkpoint_event(events: Mapping[str, list[Path]], preferred: str) -> str:
621 """Choose the newest numbered event, retaining legacy-name support."""
622 matching = [
623 event_name
624 for event_name in events
625 if event_name == preferred or event_name.startswith(f"{preferred}_")
626 ]
627 if not matching:
628 return sorted(events)[-1]
629 return max(matching, key=_checkpoint_event_order)
632def _load_rank_layouts(files: list[Path]) -> list[dict[str, Any]]:
633 """Load checkpoint tensor-layout maps for all selected ranks."""
634 return [
635 json.loads(path.read_text(encoding="utf-8")).get("tensor_layouts", {})
636 for path in files
637 ]
640def _missing_layout_names(
641 baseline: Mapping[str, Any],
642 candidate: Mapping[str, Any],
643) -> tuple[list[str], list[str]]:
644 """List one-sided keys, excluding lazily initialized optimizer state."""
645 baseline_only = set(baseline) - set(candidate)
646 candidate_only = set(candidate) - set(baseline)
647 initialized_optimizer_entries = sorted(
648 name for name in candidate_only if name.startswith("optimizer.")
649 )
650 missing = sorted(
651 baseline_only
652 | {
653 name
654 for name in candidate_only
655 if name not in initialized_optimizer_entries
656 }
657 )
658 return missing, initialized_optimizer_entries
661def _layout_mismatches(
662 baseline: Mapping[str, Any],
663 candidate: Mapping[str, Any],
664 same_topology: bool,
665) -> list[dict[str, Any]]:
666 """Diff shared tensor layouts over the topology-appropriate field set."""
667 fields = ["global_shape"]
668 if same_topology:
669 fields.extend(("mesh_dim_names", "mesh_shape", "placements"))
670 mismatches = []
671 for name in sorted(set(baseline).intersection(candidate)):
672 baseline_layout = baseline[name]
673 candidate_layout = candidate[name]
674 differences = {
675 field: {
676 "baseline": baseline_layout.get(field),
677 "candidate": candidate_layout.get(field),
678 }
679 for field in fields
680 if baseline_layout.get(field) != candidate_layout.get(field)
681 }
682 if differences:
683 mismatches.append({"name": name, "differences": differences})
684 return mismatches
687def _rank_consistency_inconsistencies(
688 side_name: str,
689 rank_layouts: list[dict[str, Any]],
690) -> list[dict[str, Any]]:
691 """Flag keys whose rank-local payloads disagree on presence or shape."""
692 inconsistencies = []
693 all_names = set().union(*(set(layouts) for layouts in rank_layouts))
694 for name in sorted(all_names):
695 values = [layouts.get(name) for layouts in rank_layouts]
696 global_shapes = {
697 tuple(value.get("global_shape", ()))
698 for value in values
699 if value is not None
700 }
701 if any(value is None for value in values) or len(global_shapes) != 1:
702 inconsistencies.append(
703 {
704 "side": side_name,
705 "name": name,
706 "reason": "rank-local payloads disagree on key presence or global shape",
707 }
708 )
709 return inconsistencies
712def _same_topology_rank_inconsistencies(
713 baseline_ranks: list[dict[str, Any]],
714 candidate_ranks: list[dict[str, Any]],
715) -> list[dict[str, Any]]:
716 """Require equal rank counts and stable local shard shapes."""
717 inconsistencies = []
718 if len(baseline_ranks) != len(candidate_ranks):
719 inconsistencies.append(
720 {
721 "side": "comparison",
722 "reason": "same-topology checkpoint evidence has different rank counts",
723 "baseline_ranks": len(baseline_ranks),
724 "candidate_ranks": len(candidate_ranks),
725 }
726 )
727 return inconsistencies
728 for rank, (baseline_layouts, candidate_layouts) in enumerate(
729 zip(baseline_ranks, candidate_ranks)
730 ):
731 for name in sorted(set(baseline_layouts).intersection(candidate_layouts)):
732 if baseline_layouts[name].get("local_shape") != candidate_layouts[name].get(
733 "local_shape"
734 ):
735 inconsistencies.append(
736 {
737 "side": "comparison",
738 "rank": rank,
739 "name": name,
740 "reason": "same-topology local shard shape changed",
741 }
742 )
743 return inconsistencies
746def compare_checkpoint_layouts(
747 baseline_dir: str | Path,
748 candidate_dir: str | Path,
749 *,
750 same_topology: bool,
751) -> dict[str, Any]:
752 """Compare DCP-bound logical layouts while allowing legal resharding."""
753 baseline_files = sorted(Path(baseline_dir).glob("**/checkpoint/*_rank*.json"))
754 candidate_files = sorted(Path(candidate_dir).glob("**/checkpoint/*_rank*.json"))
755 if not baseline_files or not candidate_files:
756 return {"status": "BLOCKED", "reason": "checkpoint layout evidence is missing"}
757 baseline_events = _group_checkpoint_events(baseline_files)
758 candidate_events = _group_checkpoint_events(candidate_files)
759 candidate_event = _latest_checkpoint_event(candidate_events, "after_load")
760 checkpoint_identity = candidate_event.removeprefix("after_load").lstrip("_")
761 baseline_event = (
762 f"before_save_{checkpoint_identity}"
763 if checkpoint_identity
764 else _latest_checkpoint_event(baseline_events, "before_save")
765 )
766 if baseline_event not in baseline_events:
767 return {
768 "status": "BLOCKED",
769 "reason": "baseline checkpoint evidence for the restored checkpoint is missing",
770 "baseline_event": baseline_event,
771 "candidate_event": candidate_event,
772 }
773 baseline_ranks = _load_rank_layouts(baseline_events[baseline_event])
774 candidate_ranks = _load_rank_layouts(candidate_events[candidate_event])
775 baseline = baseline_ranks[0]
776 candidate = candidate_ranks[0]
777 missing, initialized_optimizer_entries = _missing_layout_names(baseline, candidate)
778 mismatches = _layout_mismatches(baseline, candidate, same_topology)
779 rank_inconsistencies = []
780 for side_name, rank_layouts in (
781 ("baseline", baseline_ranks),
782 ("candidate", candidate_ranks),
783 ):
784 rank_inconsistencies.extend(
785 _rank_consistency_inconsistencies(side_name, rank_layouts)
786 )
787 if same_topology:
788 rank_inconsistencies.extend(
789 _same_topology_rank_inconsistencies(baseline_ranks, candidate_ranks)
790 )
791 return {
792 "status": "FAIL" if missing or mismatches or rank_inconsistencies else "PASS",
793 "missing": missing,
794 "mismatches": mismatches,
795 "rank_inconsistencies": rank_inconsistencies,
796 "baseline_rank_count": len(baseline_ranks),
797 "candidate_rank_count": len(candidate_ranks),
798 "baseline_event": baseline_event,
799 "candidate_event": candidate_event,
800 "initialized_optimizer_entries": initialized_optimizer_entries,
801 }
804__all__ = [
805 "compare_checkpoint_layouts",
806 "compare_parameter_probes",
807 "compare_preflight_configs",
808 "compare_scalar_curves",
809 "load_jsonl",
810 "summarize_performance",
811]