Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / module_parity.py: 80%

212 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Generic final-module forward/intermediate/backward parity runner.""" 

16 

17from __future__ import annotations 

18 

19import fnmatch 

20import traceback 

21from dataclasses import dataclass 

22from typing import Any, Mapping 

23 

24import torch 

25from torch.utils._pytree import tree_flatten, tree_map 

26 

27from hyper_parallel.tools.model_integration.evidence_store import EvidenceStore 

28from hyper_parallel.tools.model_integration.schemas import ( 

29 ComparisonMetric, 

30 tolerance_value, 

31) 

32from hyper_parallel.models.validation_spec import ModelValidationSpec, ModuleParityCase 

33 

34 

35@dataclass(frozen=True) 

36class ParityContext: 

37 """Resolved execution context supplied to model-owned parity builders.""" 

38 

39 manifest: Any 

40 dtype: str 

41 device: str 

42 evidence_store: EvidenceStore 

43 options: dict[str, Any] 

44 

45 

46@dataclass 

47class ParityExecution: 

48 """Values exposed to observation getters after one forward/backward.""" 

49 

50 module: Any 

51 output: Any 

52 args: tuple[Any, ...] 

53 kwargs: dict[str, Any] 

54 

55 

56def _select_module(value: Any, selector: str) -> Any: 

57 if selector in ("", "<root>"): 

58 return value 

59 module_by_fqn = dict(value.named_modules()) 

60 selected = module_by_fqn.get(selector) 

61 if selected is None: 

62 raise ValueError(f"candidate selector matched no final module: {selector!r}") 

63 return selected 

64 

65 

66def _clone_value(value: Any) -> Any: 

67 if not torch.is_tensor(value): 

68 return value 

69 clone = value.detach().clone() 

70 if value.requires_grad and (clone.is_floating_point() or clone.is_complex()): 

71 clone.requires_grad_(True) 

72 return clone 

73 

74 

75def _clone_call_inputs(inputs: Any) -> tuple[tuple[Any, ...], dict[str, Any]]: 

76 if isinstance(inputs, Mapping) and ("args" in inputs or "kwargs" in inputs): 

77 args = tuple(inputs.get("args", ())) 

78 kwargs = dict(inputs.get("kwargs", {})) 

79 elif isinstance(inputs, Mapping): 

80 args = () 

81 kwargs = dict(inputs) 

82 elif isinstance(inputs, tuple): 

83 args = inputs 

84 kwargs = {} 

85 else: 

86 args = (inputs,) 

87 kwargs = {} 

88 return tree_map(_clone_value, args), tree_map(_clone_value, kwargs) 

89 

90 

91def _floating_tensors(value: Any) -> list[torch.Tensor]: 

92 return [ 

93 item for item in tree_flatten(value)[0] 

94 if torch.is_tensor(item) and (item.is_floating_point() or item.is_complex()) 

95 ] 

96 

97 

98def _default_objective(output: Any) -> torch.Tensor: 

99 tensors = _floating_tensors(output) 

100 if not tensors: 

101 raise ValueError("parity output contains no differentiable tensor") 

102 return sum(tensor.float().sum() for tensor in tensors) 

103 

104 

105def _metric( 

106 name: str, 

107 reference: torch.Tensor, 

108 candidate: torch.Tensor, 

109 *, 

110 exact: bool, 

111 atol: float, 

112 rtol: float, 

113 relative_l2_limit: float | None = None, 

114) -> ComparisonMetric: 

115 reference_cpu = reference.detach().float().cpu() 

116 candidate_cpu = candidate.detach().float().cpu() 

117 same_shape = tuple(reference_cpu.shape) == tuple(candidate_cpu.shape) 

118 finite = bool(torch.isfinite(reference_cpu).all() and torch.isfinite(candidate_cpu).all()) 

119 if not same_shape: 

120 return ComparisonMetric( 

121 name=name, 

122 status="FAIL", 

123 max_abs=None, 

124 relative_l2=None, 

125 finite=finite, 

126 reference_shape=tuple(reference_cpu.shape), 

127 candidate_shape=tuple(candidate_cpu.shape), 

128 reason="shape mismatch", 

129 ) 

130 equal = bool(torch.equal(reference_cpu, candidate_cpu)) 

131 delta = candidate_cpu.double() - reference_cpu.double() 

132 max_abs = float(delta.abs().max()) if delta.numel() else 0.0 

133 reference_norm = torch.linalg.vector_norm( # pylint: disable=not-callable 

134 reference_cpu.double().reshape(-1) 

135 ) 

136 delta_norm = torch.linalg.vector_norm(delta.reshape(-1)) # pylint: disable=not-callable 

137 relative_l2 = float(delta_norm / reference_norm.clamp_min(torch.finfo(torch.float64).eps)) 

138 passed = equal if exact else bool(torch.allclose(reference_cpu, candidate_cpu, atol=atol, rtol=rtol)) 

139 if not exact and relative_l2_limit is not None: 

140 passed = passed and relative_l2 <= relative_l2_limit 

141 passed = passed and finite 

142 return ComparisonMetric( 

143 name=name, 

144 status="PASS" if passed else "FAIL", 

145 max_abs=max_abs, 

146 relative_l2=relative_l2, 

147 finite=finite, 

148 reference_shape=tuple(reference_cpu.shape), 

149 candidate_shape=tuple(candidate_cpu.shape), 

150 ) 

151 

152 

153def _compare_tree( 

154 prefix: str, 

155 reference: Any, 

156 candidate: Any, 

157 *, 

158 exact: bool, 

159 atol: float, 

160 rtol: float, 

161 relative_l2_limit: float | None = None, 

162) -> list[ComparisonMetric]: 

163 reference_values, reference_spec = tree_flatten(reference) 

164 candidate_values, candidate_spec = tree_flatten(candidate) 

165 if reference_spec != candidate_spec: 

166 return [ 

167 ComparisonMetric( 

168 name=prefix, 

169 status="FAIL", 

170 max_abs=None, 

171 relative_l2=None, 

172 finite=True, 

173 reference_shape=(), 

174 candidate_shape=(), 

175 reason=( 

176 "output structure mismatch: " 

177 f"reference={reference_spec!r}, candidate={candidate_spec!r}" 

178 ), 

179 ) 

180 ] 

181 metrics = [] 

182 for index, (reference_value, candidate_value) in enumerate(zip(reference_values, candidate_values)): 

183 name = f"{prefix}.{index}" 

184 if torch.is_tensor(reference_value) and torch.is_tensor(candidate_value): 

185 tensor_exact = exact or not ( 

186 reference_value.is_floating_point() or reference_value.is_complex() 

187 ) 

188 metrics.append( 

189 _metric( 

190 name, 

191 reference_value, 

192 candidate_value, 

193 exact=tensor_exact, 

194 atol=atol, 

195 rtol=rtol, 

196 relative_l2_limit=relative_l2_limit, 

197 ) 

198 ) 

199 elif reference_value != candidate_value: 

200 metrics.append( 

201 ComparisonMetric( 

202 name=name, 

203 status="FAIL", 

204 max_abs=None, 

205 relative_l2=None, 

206 finite=True, 

207 reference_shape=(), 

208 candidate_shape=(), 

209 reason=f"non-tensor values differ: {reference_value!r} != {candidate_value!r}", 

210 ) 

211 ) 

212 return metrics 

213 

214 

215def _parameter_mapping( 

216 case: ModuleParityCase, 

217 reference: Any, 

218 candidate: Any, 

219 context: ParityContext, 

220) -> dict[str, str]: 

221 if case.weight_adapter is None: 

222 raise TypeError("in-process parity case requires weight_adapter") 

223 mapping_result = case.weight_adapter(reference, candidate, context) 

224 if mapping_result is None: 

225 mapping = { 

226 name: name for name, parameter in candidate.named_parameters() 

227 if parameter.requires_grad 

228 } 

229 elif isinstance(mapping_result, Mapping): 

230 mapping = {str(key): str(value) for key, value in mapping_result.items()} 

231 else: 

232 raise TypeError("parity weight_adapter must return a candidate-to-reference mapping or None") 

233 candidate_trainable = { 

234 name for name, parameter in candidate.named_parameters() if parameter.requires_grad 

235 } 

236 reference_names = dict(reference.named_parameters()) 

237 missing = sorted(candidate_trainable - set(mapping)) 

238 invalid = sorted(set(mapping.values()) - set(reference_names)) 

239 if missing or invalid: 

240 raise ValueError( 

241 "parity weight coverage is incomplete: " 

242 f"missing_candidate={missing}, unknown_reference={invalid}" 

243 ) 

244 for pattern in case.required_parameter_patterns: 

245 if not any(fnmatch.fnmatchcase(name, pattern) for name in candidate_trainable): 

246 raise ValueError(f"required parity parameter pattern matched nothing: {pattern!r}") 

247 return mapping 

248 

249 

250def _input_gradients(args: tuple[Any, ...], kwargs: dict[str, Any]) -> list[torch.Tensor | None]: 

251 return [ 

252 value.grad if torch.is_tensor(value) and value.requires_grad else None 

253 for value in tree_flatten((args, kwargs))[0] 

254 ] 

255 

256 

257def _build_parity_modules(case: ModuleParityCase, context: ParityContext) -> tuple[Any, Any]: 

258 """Instantiate reference/candidate and reject aliased implementations.""" 

259 candidate_builder = case.candidate_builder 

260 reference_builder = case.reference_builder 

261 input_builder = case.input_builder 

262 if not all(callable(value) for value in (candidate_builder, reference_builder, input_builder)): 

263 raise TypeError("in-process parity case has an incomplete callable contract") 

264 candidate_tree = candidate_builder(context) 

265 candidate = _select_module(candidate_tree, case.candidate_selector) 

266 reference = reference_builder(context) 

267 if candidate is reference or type(candidate) is type(reference): 

268 raise ValueError( 

269 "reference and candidate must be independent implementations; " 

270 f"got {type(candidate).__module__}.{type(candidate).__qualname__}" 

271 ) 

272 return candidate, reference 

273 

274 

275def _resolve_parity_tolerance( 

276 context: ParityContext, 

277) -> tuple[float, float, float | None]: 

278 """Read the scalar tolerance contract from the parity context options.""" 

279 tolerance = context.options.get("tolerance", {}) 

280 atol = tolerance_value(tolerance, "atol", "max_abs") 

281 rtol = tolerance_value(tolerance, "rtol") 

282 relative_l2_limit = tolerance.get("relative_l2") 

283 if relative_l2_limit is not None: 

284 relative_l2_limit = float(relative_l2_limit) 

285 return atol, rtol, relative_l2_limit 

286 

287 

288def _observation_metrics( 

289 case: ModuleParityCase, 

290 reference_execution: ParityExecution, 

291 candidate_execution: ParityExecution, 

292 reference_output: Any, 

293 candidate_output: Any, 

294 atol: float, 

295 rtol: float, 

296 relative_l2_limit: float | None, 

297) -> list[ComparisonMetric]: 

298 """Compare every declared observation between reference and candidate.""" 

299 metrics = [] 

300 for observation in case.observations: 

301 reference_value = ( 

302 observation.reference_getter(reference_execution) 

303 if observation.reference_getter is not None 

304 else reference_output 

305 ) 

306 candidate_value = ( 

307 observation.candidate_getter(candidate_execution) 

308 if observation.candidate_getter is not None 

309 else candidate_output 

310 ) 

311 if reference_value is None or candidate_value is None: 

312 if observation.required: 

313 raise ValueError( 

314 f"required parity observation {observation.name!r} was not produced: " 

315 f"reference={reference_value is not None}, " 

316 f"candidate={candidate_value is not None}" 

317 ) 

318 if reference_value is None and candidate_value is None: 

319 continue 

320 metrics.extend( 

321 _compare_tree( 

322 observation.name, 

323 reference_value, 

324 candidate_value, 

325 exact=observation.comparison == "exact", 

326 atol=observation.atol if observation.atol is not None else atol, 

327 rtol=observation.rtol if observation.rtol is not None else rtol, 

328 relative_l2_limit=relative_l2_limit, 

329 ) 

330 ) 

331 return metrics 

332 

333 

334def _input_gradient_metrics( 

335 reference_args: tuple[Any, ...], 

336 reference_kwargs: dict[str, Any], 

337 candidate_args: tuple[Any, ...], 

338 candidate_kwargs: dict[str, Any], 

339 atol: float, 

340 rtol: float, 

341 relative_l2_limit: float | None, 

342) -> list[ComparisonMetric]: 

343 """Compare flattened input gradients pairwise after backward.""" 

344 metrics = [] 

345 reference_input_grads = _input_gradients(reference_args, reference_kwargs) 

346 candidate_input_grads = _input_gradients(candidate_args, candidate_kwargs) 

347 for index, (reference_grad, candidate_grad) in enumerate( 

348 zip(reference_input_grads, candidate_input_grads) 

349 ): 

350 if reference_grad is None and candidate_grad is None: 

351 continue 

352 if reference_grad is None or candidate_grad is None: 

353 raise ValueError(f"input gradient presence differs at flattened input {index}") 

354 metrics.append( 

355 _metric( 

356 f"input_grad.{index}", 

357 reference_grad, 

358 candidate_grad, 

359 exact=False, 

360 atol=atol, 

361 rtol=rtol, 

362 relative_l2_limit=relative_l2_limit, 

363 ) 

364 ) 

365 return metrics 

366 

367 

368def _parameter_gradient_metrics( 

369 mapping: dict[str, str], 

370 reference: Any, 

371 candidate: Any, 

372 atol: float, 

373 rtol: float, 

374 relative_l2_limit: float | None, 

375) -> list[ComparisonMetric]: 

376 """Compare mapped parameter gradients pairwise after backward.""" 

377 metrics = [] 

378 reference_parameters = dict(reference.named_parameters()) 

379 candidate_parameters = dict(candidate.named_parameters()) 

380 for candidate_name, reference_name in sorted(mapping.items()): 

381 reference_grad = reference_parameters[reference_name].grad 

382 candidate_grad = candidate_parameters[candidate_name].grad 

383 if reference_grad is None and candidate_grad is None: 

384 continue 

385 if reference_grad is None or candidate_grad is None: 

386 raise ValueError( 

387 f"parameter gradient presence differs: {candidate_name} -> {reference_name}" 

388 ) 

389 metrics.append( 

390 _metric( 

391 f"parameter_grad.{candidate_name}", 

392 reference_grad, 

393 candidate_grad, 

394 exact=False, 

395 atol=atol, 

396 rtol=rtol, 

397 relative_l2_limit=relative_l2_limit, 

398 ) 

399 ) 

400 return metrics 

401 

402 

403def _run_in_process_case(case: ModuleParityCase, context: ParityContext) -> dict[str, Any]: 

404 candidate, reference = _build_parity_modules(case, context) 

405 mapping = _parameter_mapping(case, reference, candidate, context) 

406 inputs = case.input_builder(context) 

407 reference_args, reference_kwargs = _clone_call_inputs(inputs) 

408 candidate_args, candidate_kwargs = _clone_call_inputs(inputs) 

409 reference_output = reference(*reference_args, **reference_kwargs) 

410 candidate_output = candidate(*candidate_args, **candidate_kwargs) 

411 atol, rtol, relative_l2_limit = _resolve_parity_tolerance(context) 

412 metrics = _compare_tree( 

413 "output", 

414 reference_output, 

415 candidate_output, 

416 exact=False, 

417 atol=atol, 

418 rtol=rtol, 

419 relative_l2_limit=relative_l2_limit, 

420 ) 

421 reference_execution = ParityExecution(reference, reference_output, reference_args, reference_kwargs) 

422 candidate_execution = ParityExecution(candidate, candidate_output, candidate_args, candidate_kwargs) 

423 metrics.extend( 

424 _observation_metrics( 

425 case, 

426 reference_execution, 

427 candidate_execution, 

428 reference_output, 

429 candidate_output, 

430 atol, 

431 rtol, 

432 relative_l2_limit, 

433 ) 

434 ) 

435 objective = case.objective or _default_objective 

436 objective(reference_output).backward() 

437 objective(candidate_output).backward() 

438 metrics.extend( 

439 _input_gradient_metrics( 

440 reference_args, 

441 reference_kwargs, 

442 candidate_args, 

443 candidate_kwargs, 

444 atol, 

445 rtol, 

446 relative_l2_limit, 

447 ) 

448 ) 

449 metrics.extend( 

450 _parameter_gradient_metrics( 

451 mapping, 

452 reference, 

453 candidate, 

454 atol, 

455 rtol, 

456 relative_l2_limit, 

457 ) 

458 ) 

459 status = "PASS" if all(metric.status == "PASS" for metric in metrics) else "FAIL" 

460 return { 

461 "case": case.name, 

462 "status": status, 

463 "candidate_type": f"{type(candidate).__module__}.{type(candidate).__qualname__}", 

464 "reference_type": f"{type(reference).__module__}.{type(reference).__qualname__}", 

465 "weight_mapping": mapping, 

466 "metrics": [metric.to_dict() for metric in metrics], 

467 } 

468 

469 

470def run_module_parity( 

471 validation_spec: ModelValidationSpec, 

472 context: ParityContext, 

473) -> dict[str, Any]: 

474 """Execute every declared parity case and persist per-case evidence.""" 

475 reports = [] 

476 for case in validation_spec.module_cases: 

477 if context.dtype not in case.required_dtypes: 

478 continue 

479 try: 

480 if case.execution == "isolated_process": 

481 report = case.isolated_runner(context) # type: ignore[misc] 

482 if not isinstance(report, dict) or report.get("status") not in ( 

483 "PASS", "FAIL", "BLOCKED" 

484 ): 

485 raise ValueError("isolated parity runner returned an invalid report") 

486 else: 

487 report = _run_in_process_case(case, context) 

488 except NotImplementedError as error: 

489 report = { 

490 "case": case.name, 

491 "status": "BLOCKED", 

492 "reason": "OPERATOR_UNSUPPORTED", 

493 "error": str(error), 

494 } 

495 except Exception as error: # pylint: disable=broad-except 

496 report = { 

497 "case": case.name, 

498 "status": "FAIL", 

499 "error_type": type(error).__name__, 

500 "error": str(error), 

501 "traceback": traceback.format_exc(), 

502 } 

503 reports.append(report) 

504 context.evidence_store.write_json( 

505 f"module_parity/{case.name}/comparison.json", 

506 report, 

507 ) 

508 statuses = [report["status"] for report in reports] 

509 status = ( 

510 "FAIL" if "FAIL" in statuses 

511 else "BLOCKED" if "BLOCKED" in statuses or not statuses 

512 else "PASS" 

513 ) 

514 aggregate = {"status": status, "cases": reports} 

515 if not reports: 

516 aggregate["reason"] = ( 

517 "no module parity case supports the requested dtype, or the adapter " 

518 "declared no authoritative case" 

519 ) 

520 context.evidence_store.write_json("module_parity/comparison.json", aggregate) 

521 return aggregate 

522 

523 

524__all__ = ["ParityContext", "ParityExecution", "run_module_parity"]