Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / module_parity.py: 80%
212 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Generic final-module forward/intermediate/backward parity runner."""
17from __future__ import annotations
19import fnmatch
20import traceback
21from dataclasses import dataclass
22from typing import Any, Mapping
24import torch
25from torch.utils._pytree import tree_flatten, tree_map
27from hyper_parallel.tools.model_integration.evidence_store import EvidenceStore
28from hyper_parallel.tools.model_integration.schemas import (
29 ComparisonMetric,
30 tolerance_value,
31)
32from hyper_parallel.models.validation_spec import ModelValidationSpec, ModuleParityCase
35@dataclass(frozen=True)
36class ParityContext:
37 """Resolved execution context supplied to model-owned parity builders."""
39 manifest: Any
40 dtype: str
41 device: str
42 evidence_store: EvidenceStore
43 options: dict[str, Any]
46@dataclass
47class ParityExecution:
48 """Values exposed to observation getters after one forward/backward."""
50 module: Any
51 output: Any
52 args: tuple[Any, ...]
53 kwargs: dict[str, Any]
56def _select_module(value: Any, selector: str) -> Any:
57 if selector in ("", "<root>"):
58 return value
59 module_by_fqn = dict(value.named_modules())
60 selected = module_by_fqn.get(selector)
61 if selected is None:
62 raise ValueError(f"candidate selector matched no final module: {selector!r}")
63 return selected
66def _clone_value(value: Any) -> Any:
67 if not torch.is_tensor(value):
68 return value
69 clone = value.detach().clone()
70 if value.requires_grad and (clone.is_floating_point() or clone.is_complex()):
71 clone.requires_grad_(True)
72 return clone
75def _clone_call_inputs(inputs: Any) -> tuple[tuple[Any, ...], dict[str, Any]]:
76 if isinstance(inputs, Mapping) and ("args" in inputs or "kwargs" in inputs):
77 args = tuple(inputs.get("args", ()))
78 kwargs = dict(inputs.get("kwargs", {}))
79 elif isinstance(inputs, Mapping):
80 args = ()
81 kwargs = dict(inputs)
82 elif isinstance(inputs, tuple):
83 args = inputs
84 kwargs = {}
85 else:
86 args = (inputs,)
87 kwargs = {}
88 return tree_map(_clone_value, args), tree_map(_clone_value, kwargs)
91def _floating_tensors(value: Any) -> list[torch.Tensor]:
92 return [
93 item for item in tree_flatten(value)[0]
94 if torch.is_tensor(item) and (item.is_floating_point() or item.is_complex())
95 ]
98def _default_objective(output: Any) -> torch.Tensor:
99 tensors = _floating_tensors(output)
100 if not tensors:
101 raise ValueError("parity output contains no differentiable tensor")
102 return sum(tensor.float().sum() for tensor in tensors)
105def _metric(
106 name: str,
107 reference: torch.Tensor,
108 candidate: torch.Tensor,
109 *,
110 exact: bool,
111 atol: float,
112 rtol: float,
113 relative_l2_limit: float | None = None,
114) -> ComparisonMetric:
115 reference_cpu = reference.detach().float().cpu()
116 candidate_cpu = candidate.detach().float().cpu()
117 same_shape = tuple(reference_cpu.shape) == tuple(candidate_cpu.shape)
118 finite = bool(torch.isfinite(reference_cpu).all() and torch.isfinite(candidate_cpu).all())
119 if not same_shape:
120 return ComparisonMetric(
121 name=name,
122 status="FAIL",
123 max_abs=None,
124 relative_l2=None,
125 finite=finite,
126 reference_shape=tuple(reference_cpu.shape),
127 candidate_shape=tuple(candidate_cpu.shape),
128 reason="shape mismatch",
129 )
130 equal = bool(torch.equal(reference_cpu, candidate_cpu))
131 delta = candidate_cpu.double() - reference_cpu.double()
132 max_abs = float(delta.abs().max()) if delta.numel() else 0.0
133 reference_norm = torch.linalg.vector_norm( # pylint: disable=not-callable
134 reference_cpu.double().reshape(-1)
135 )
136 delta_norm = torch.linalg.vector_norm(delta.reshape(-1)) # pylint: disable=not-callable
137 relative_l2 = float(delta_norm / reference_norm.clamp_min(torch.finfo(torch.float64).eps))
138 passed = equal if exact else bool(torch.allclose(reference_cpu, candidate_cpu, atol=atol, rtol=rtol))
139 if not exact and relative_l2_limit is not None:
140 passed = passed and relative_l2 <= relative_l2_limit
141 passed = passed and finite
142 return ComparisonMetric(
143 name=name,
144 status="PASS" if passed else "FAIL",
145 max_abs=max_abs,
146 relative_l2=relative_l2,
147 finite=finite,
148 reference_shape=tuple(reference_cpu.shape),
149 candidate_shape=tuple(candidate_cpu.shape),
150 )
153def _compare_tree(
154 prefix: str,
155 reference: Any,
156 candidate: Any,
157 *,
158 exact: bool,
159 atol: float,
160 rtol: float,
161 relative_l2_limit: float | None = None,
162) -> list[ComparisonMetric]:
163 reference_values, reference_spec = tree_flatten(reference)
164 candidate_values, candidate_spec = tree_flatten(candidate)
165 if reference_spec != candidate_spec:
166 return [
167 ComparisonMetric(
168 name=prefix,
169 status="FAIL",
170 max_abs=None,
171 relative_l2=None,
172 finite=True,
173 reference_shape=(),
174 candidate_shape=(),
175 reason=(
176 "output structure mismatch: "
177 f"reference={reference_spec!r}, candidate={candidate_spec!r}"
178 ),
179 )
180 ]
181 metrics = []
182 for index, (reference_value, candidate_value) in enumerate(zip(reference_values, candidate_values)):
183 name = f"{prefix}.{index}"
184 if torch.is_tensor(reference_value) and torch.is_tensor(candidate_value):
185 tensor_exact = exact or not (
186 reference_value.is_floating_point() or reference_value.is_complex()
187 )
188 metrics.append(
189 _metric(
190 name,
191 reference_value,
192 candidate_value,
193 exact=tensor_exact,
194 atol=atol,
195 rtol=rtol,
196 relative_l2_limit=relative_l2_limit,
197 )
198 )
199 elif reference_value != candidate_value:
200 metrics.append(
201 ComparisonMetric(
202 name=name,
203 status="FAIL",
204 max_abs=None,
205 relative_l2=None,
206 finite=True,
207 reference_shape=(),
208 candidate_shape=(),
209 reason=f"non-tensor values differ: {reference_value!r} != {candidate_value!r}",
210 )
211 )
212 return metrics
215def _parameter_mapping(
216 case: ModuleParityCase,
217 reference: Any,
218 candidate: Any,
219 context: ParityContext,
220) -> dict[str, str]:
221 if case.weight_adapter is None:
222 raise TypeError("in-process parity case requires weight_adapter")
223 mapping_result = case.weight_adapter(reference, candidate, context)
224 if mapping_result is None:
225 mapping = {
226 name: name for name, parameter in candidate.named_parameters()
227 if parameter.requires_grad
228 }
229 elif isinstance(mapping_result, Mapping):
230 mapping = {str(key): str(value) for key, value in mapping_result.items()}
231 else:
232 raise TypeError("parity weight_adapter must return a candidate-to-reference mapping or None")
233 candidate_trainable = {
234 name for name, parameter in candidate.named_parameters() if parameter.requires_grad
235 }
236 reference_names = dict(reference.named_parameters())
237 missing = sorted(candidate_trainable - set(mapping))
238 invalid = sorted(set(mapping.values()) - set(reference_names))
239 if missing or invalid:
240 raise ValueError(
241 "parity weight coverage is incomplete: "
242 f"missing_candidate={missing}, unknown_reference={invalid}"
243 )
244 for pattern in case.required_parameter_patterns:
245 if not any(fnmatch.fnmatchcase(name, pattern) for name in candidate_trainable):
246 raise ValueError(f"required parity parameter pattern matched nothing: {pattern!r}")
247 return mapping
250def _input_gradients(args: tuple[Any, ...], kwargs: dict[str, Any]) -> list[torch.Tensor | None]:
251 return [
252 value.grad if torch.is_tensor(value) and value.requires_grad else None
253 for value in tree_flatten((args, kwargs))[0]
254 ]
257def _build_parity_modules(case: ModuleParityCase, context: ParityContext) -> tuple[Any, Any]:
258 """Instantiate reference/candidate and reject aliased implementations."""
259 candidate_builder = case.candidate_builder
260 reference_builder = case.reference_builder
261 input_builder = case.input_builder
262 if not all(callable(value) for value in (candidate_builder, reference_builder, input_builder)):
263 raise TypeError("in-process parity case has an incomplete callable contract")
264 candidate_tree = candidate_builder(context)
265 candidate = _select_module(candidate_tree, case.candidate_selector)
266 reference = reference_builder(context)
267 if candidate is reference or type(candidate) is type(reference):
268 raise ValueError(
269 "reference and candidate must be independent implementations; "
270 f"got {type(candidate).__module__}.{type(candidate).__qualname__}"
271 )
272 return candidate, reference
275def _resolve_parity_tolerance(
276 context: ParityContext,
277) -> tuple[float, float, float | None]:
278 """Read the scalar tolerance contract from the parity context options."""
279 tolerance = context.options.get("tolerance", {})
280 atol = tolerance_value(tolerance, "atol", "max_abs")
281 rtol = tolerance_value(tolerance, "rtol")
282 relative_l2_limit = tolerance.get("relative_l2")
283 if relative_l2_limit is not None:
284 relative_l2_limit = float(relative_l2_limit)
285 return atol, rtol, relative_l2_limit
288def _observation_metrics(
289 case: ModuleParityCase,
290 reference_execution: ParityExecution,
291 candidate_execution: ParityExecution,
292 reference_output: Any,
293 candidate_output: Any,
294 atol: float,
295 rtol: float,
296 relative_l2_limit: float | None,
297) -> list[ComparisonMetric]:
298 """Compare every declared observation between reference and candidate."""
299 metrics = []
300 for observation in case.observations:
301 reference_value = (
302 observation.reference_getter(reference_execution)
303 if observation.reference_getter is not None
304 else reference_output
305 )
306 candidate_value = (
307 observation.candidate_getter(candidate_execution)
308 if observation.candidate_getter is not None
309 else candidate_output
310 )
311 if reference_value is None or candidate_value is None:
312 if observation.required:
313 raise ValueError(
314 f"required parity observation {observation.name!r} was not produced: "
315 f"reference={reference_value is not None}, "
316 f"candidate={candidate_value is not None}"
317 )
318 if reference_value is None and candidate_value is None:
319 continue
320 metrics.extend(
321 _compare_tree(
322 observation.name,
323 reference_value,
324 candidate_value,
325 exact=observation.comparison == "exact",
326 atol=observation.atol if observation.atol is not None else atol,
327 rtol=observation.rtol if observation.rtol is not None else rtol,
328 relative_l2_limit=relative_l2_limit,
329 )
330 )
331 return metrics
334def _input_gradient_metrics(
335 reference_args: tuple[Any, ...],
336 reference_kwargs: dict[str, Any],
337 candidate_args: tuple[Any, ...],
338 candidate_kwargs: dict[str, Any],
339 atol: float,
340 rtol: float,
341 relative_l2_limit: float | None,
342) -> list[ComparisonMetric]:
343 """Compare flattened input gradients pairwise after backward."""
344 metrics = []
345 reference_input_grads = _input_gradients(reference_args, reference_kwargs)
346 candidate_input_grads = _input_gradients(candidate_args, candidate_kwargs)
347 for index, (reference_grad, candidate_grad) in enumerate(
348 zip(reference_input_grads, candidate_input_grads)
349 ):
350 if reference_grad is None and candidate_grad is None:
351 continue
352 if reference_grad is None or candidate_grad is None:
353 raise ValueError(f"input gradient presence differs at flattened input {index}")
354 metrics.append(
355 _metric(
356 f"input_grad.{index}",
357 reference_grad,
358 candidate_grad,
359 exact=False,
360 atol=atol,
361 rtol=rtol,
362 relative_l2_limit=relative_l2_limit,
363 )
364 )
365 return metrics
368def _parameter_gradient_metrics(
369 mapping: dict[str, str],
370 reference: Any,
371 candidate: Any,
372 atol: float,
373 rtol: float,
374 relative_l2_limit: float | None,
375) -> list[ComparisonMetric]:
376 """Compare mapped parameter gradients pairwise after backward."""
377 metrics = []
378 reference_parameters = dict(reference.named_parameters())
379 candidate_parameters = dict(candidate.named_parameters())
380 for candidate_name, reference_name in sorted(mapping.items()):
381 reference_grad = reference_parameters[reference_name].grad
382 candidate_grad = candidate_parameters[candidate_name].grad
383 if reference_grad is None and candidate_grad is None:
384 continue
385 if reference_grad is None or candidate_grad is None:
386 raise ValueError(
387 f"parameter gradient presence differs: {candidate_name} -> {reference_name}"
388 )
389 metrics.append(
390 _metric(
391 f"parameter_grad.{candidate_name}",
392 reference_grad,
393 candidate_grad,
394 exact=False,
395 atol=atol,
396 rtol=rtol,
397 relative_l2_limit=relative_l2_limit,
398 )
399 )
400 return metrics
403def _run_in_process_case(case: ModuleParityCase, context: ParityContext) -> dict[str, Any]:
404 candidate, reference = _build_parity_modules(case, context)
405 mapping = _parameter_mapping(case, reference, candidate, context)
406 inputs = case.input_builder(context)
407 reference_args, reference_kwargs = _clone_call_inputs(inputs)
408 candidate_args, candidate_kwargs = _clone_call_inputs(inputs)
409 reference_output = reference(*reference_args, **reference_kwargs)
410 candidate_output = candidate(*candidate_args, **candidate_kwargs)
411 atol, rtol, relative_l2_limit = _resolve_parity_tolerance(context)
412 metrics = _compare_tree(
413 "output",
414 reference_output,
415 candidate_output,
416 exact=False,
417 atol=atol,
418 rtol=rtol,
419 relative_l2_limit=relative_l2_limit,
420 )
421 reference_execution = ParityExecution(reference, reference_output, reference_args, reference_kwargs)
422 candidate_execution = ParityExecution(candidate, candidate_output, candidate_args, candidate_kwargs)
423 metrics.extend(
424 _observation_metrics(
425 case,
426 reference_execution,
427 candidate_execution,
428 reference_output,
429 candidate_output,
430 atol,
431 rtol,
432 relative_l2_limit,
433 )
434 )
435 objective = case.objective or _default_objective
436 objective(reference_output).backward()
437 objective(candidate_output).backward()
438 metrics.extend(
439 _input_gradient_metrics(
440 reference_args,
441 reference_kwargs,
442 candidate_args,
443 candidate_kwargs,
444 atol,
445 rtol,
446 relative_l2_limit,
447 )
448 )
449 metrics.extend(
450 _parameter_gradient_metrics(
451 mapping,
452 reference,
453 candidate,
454 atol,
455 rtol,
456 relative_l2_limit,
457 )
458 )
459 status = "PASS" if all(metric.status == "PASS" for metric in metrics) else "FAIL"
460 return {
461 "case": case.name,
462 "status": status,
463 "candidate_type": f"{type(candidate).__module__}.{type(candidate).__qualname__}",
464 "reference_type": f"{type(reference).__module__}.{type(reference).__qualname__}",
465 "weight_mapping": mapping,
466 "metrics": [metric.to_dict() for metric in metrics],
467 }
470def run_module_parity(
471 validation_spec: ModelValidationSpec,
472 context: ParityContext,
473) -> dict[str, Any]:
474 """Execute every declared parity case and persist per-case evidence."""
475 reports = []
476 for case in validation_spec.module_cases:
477 if context.dtype not in case.required_dtypes:
478 continue
479 try:
480 if case.execution == "isolated_process":
481 report = case.isolated_runner(context) # type: ignore[misc]
482 if not isinstance(report, dict) or report.get("status") not in (
483 "PASS", "FAIL", "BLOCKED"
484 ):
485 raise ValueError("isolated parity runner returned an invalid report")
486 else:
487 report = _run_in_process_case(case, context)
488 except NotImplementedError as error:
489 report = {
490 "case": case.name,
491 "status": "BLOCKED",
492 "reason": "OPERATOR_UNSUPPORTED",
493 "error": str(error),
494 }
495 except Exception as error: # pylint: disable=broad-except
496 report = {
497 "case": case.name,
498 "status": "FAIL",
499 "error_type": type(error).__name__,
500 "error": str(error),
501 "traceback": traceback.format_exc(),
502 }
503 reports.append(report)
504 context.evidence_store.write_json(
505 f"module_parity/{case.name}/comparison.json",
506 report,
507 )
508 statuses = [report["status"] for report in reports]
509 status = (
510 "FAIL" if "FAIL" in statuses
511 else "BLOCKED" if "BLOCKED" in statuses or not statuses
512 else "PASS"
513 )
514 aggregate = {"status": status, "cases": reports}
515 if not reports:
516 aggregate["reason"] = (
517 "no module parity case supports the requested dtype, or the adapter "
518 "declared no authoritative case"
519 )
520 context.evidence_store.write_json("module_parity/comparison.json", aggregate)
521 return aggregate
524__all__ = ["ParityContext", "ParityExecution", "run_module_parity"]