Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / tensor_probes.py: 77%
284 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Tensor summaries, input identity, and parameter probe recorders."""
17from __future__ import annotations
19import dataclasses
20import fnmatch
21import hashlib
22import json
23import math
24from typing import Any, Iterable, Mapping, Optional
26import torch
27import torch.distributed as dist
29from hyper_parallel.models.validation_spec import (
30 ModelValidationSpec,
31 ParameterProbeSpec,
32)
33from hyper_parallel.tools.model_integration.evidence_store import EvidenceStore
34from hyper_parallel.tools.model_integration.optimizer_layout import (
35 optimizer_state_for_parameter,
36 tensor_layout,
37)
39_SMALL_TENSOR_NUMEL = 8192
41def _tensor_numel(tensor: Any) -> int:
42 shape = getattr(tensor, "shape", ())
43 return math.prod(int(size) for size in shape)
46def _local_tensor(tensor: Any) -> torch.Tensor:
47 """Return one detached local shard without requesting redistribution."""
48 to_local = getattr(tensor, "to_local", None)
49 value = to_local() if callable(to_local) else tensor
50 return value.detach().float().cpu().contiguous()
53def _shard_identity(tensor: Any) -> tuple[tuple[int, int], ...]:
54 """Identify the logical shard while excluding replicated mesh axes."""
55 layout = getattr(tensor, "layout", None)
56 mesh = getattr(layout, "mesh", getattr(tensor, "device_mesh", None))
57 placements = getattr(layout, "placements", getattr(tensor, "placements", ()))
58 get_coordinate = getattr(mesh, "get_coordinate", None)
59 coordinate = get_coordinate() if callable(get_coordinate) else None
60 if coordinate is None:
61 return ()
62 shard_coordinates = []
63 for mesh_dim, placement in enumerate(placements or ()):
64 is_shard = getattr(placement, "is_shard", None)
65 is_ragged = getattr(placement, "is_ragged_shard", None)
66 if (callable(is_shard) and is_shard()) or (
67 callable(is_ragged) and is_ragged()
68 ):
69 shard_coordinates.append((mesh_dim, int(coordinate[mesh_dim])))
70 return tuple(shard_coordinates)
73def _local_shards_follow_flattened_order(tensor: Any) -> bool:
74 """Return whether rank-ordered shard values reconstruct the flattened tensor."""
75 layout = getattr(tensor, "layout", None)
76 placements = getattr(layout, "placements", getattr(tensor, "placements", ()))
77 shard_placements = []
78 for placement in placements or ():
79 is_shard = getattr(placement, "is_shard", None)
80 is_ragged = getattr(placement, "is_ragged_shard", None)
81 if (callable(is_shard) and is_shard()) or (
82 callable(is_ragged) and is_ragged()
83 ):
84 shard_placements.append(placement)
85 if not shard_placements:
86 return True
87 if len(shard_placements) != 1:
88 return False
89 placement = shard_placements[0]
90 return getattr(placement, "dim", None) == 0 and not hasattr(
91 placement,
92 "split_factor",
93 )
96def _local_tensor_summary(tensor: Any) -> dict[str, Any]:
97 """Summarize one local shard for cross-rank aggregation."""
98 value = _local_tensor(tensor).reshape(-1)
99 digest = hashlib.sha256(value.numpy().tobytes()).hexdigest()
100 return {
101 "rank": dist.get_rank() if dist.is_available() and dist.is_initialized() else 0,
102 "shard": _shard_identity(tensor),
103 "sha256": digest,
104 "numel": value.numel(),
105 "finite": bool(torch.isfinite(value).all()) if value.numel() else True,
106 "sum": float(value.double().sum()),
107 "squared_l2": float(torch.square(value.double()).sum()),
108 "min": float(value.min()) if value.numel() else None,
109 "max": float(value.max()) if value.numel() else None,
110 "values": value.tolist() if value.numel() <= _SMALL_TENSOR_NUMEL else None,
111 "global_shape": [int(size) for size in getattr(tensor, "shape", ())],
112 "dtype": str(getattr(tensor, "dtype", None)),
113 "layout": tensor_layout(tensor),
114 }
117def _gather_shard_summaries(
118 local_summary: Optional[dict[str, Any]],
119) -> tuple[list[dict[str, Any]], list[Optional[dict[str, Any]]]]:
120 """Gather optional metadata records and retain one value per logical shard."""
121 rank_summaries = [local_summary]
122 if dist.is_available() and dist.is_initialized():
123 rank_summaries = [None] * dist.get_world_size()
124 dist.all_gather_object(rank_summaries, local_summary)
125 by_shard: dict[tuple[tuple[int, int], ...], list[dict[str, Any]]] = {}
126 for summary in rank_summaries:
127 if summary is None:
128 continue
129 shard = tuple(tuple(item) for item in summary["shard"])
130 by_shard.setdefault(shard, []).append(summary)
131 shards = [
132 min(replicas, key=lambda item: item["rank"])
133 for replicas in by_shard.values()
134 ]
135 return shards, rank_summaries
138def _replica_digest_groups(
139 rank_summaries: list[Optional[dict[str, Any]]],
140) -> dict[tuple[Any, ...], set[str]]:
141 """Group per-rank payload digests by their logical shard identity."""
142 replica_groups: dict[tuple[Any, ...], set[str]] = {}
143 for rank_summary in rank_summaries:
144 if rank_summary is None:
145 continue
146 shard = tuple(tuple(item) for item in rank_summary["shard"])
147 replica_groups.setdefault(shard, set()).add(rank_summary["sha256"])
148 return replica_groups
151def _canonical_shard_digest(shards: list[dict[str, Any]]) -> str:
152 """Hash the ordered shard digests into one canonical tensor identity."""
153 return hashlib.sha256(
154 json.dumps(
155 [(summary["shard"], summary["sha256"]) for summary in shards],
156 sort_keys=True,
157 ).encode("utf-8")
158 ).hexdigest()
161def _flattened_shard_values(tensor: Any, shards: list[dict[str, Any]]) -> Optional[list[Any]]:
162 """Reconstruct full values only from flattened-order small shard sets."""
163 if not (
164 _local_shards_follow_flattened_order(tensor)
165 and sum(shard["numel"] for shard in shards) <= _SMALL_TENSOR_NUMEL
166 and all(shard["values"] is not None for shard in shards)
167 ):
168 return None
169 return [value for shard in shards for value in shard["values"]]
172def _global_shard_summary(
173 name: str,
174 tensor: Any,
175 shards: list[dict[str, Any]],
176 rank_summaries: list[Optional[dict[str, Any]]],
177) -> dict[str, Any]:
178 """Aggregate sorted shard summaries into one canonical global record."""
179 replica_groups = _replica_digest_groups(rank_summaries)
180 minima = [summary["min"] for summary in shards if summary["min"] is not None]
181 maxima = [summary["max"] for summary in shards if summary["max"] is not None]
182 reference = shards[0]
183 summary = {
184 "name": name,
185 "global_shape": reference["global_shape"],
186 "canonical_shape": reference["global_shape"],
187 "dtype": reference["dtype"],
188 "sha256": _canonical_shard_digest(shards),
189 "finite": all(shard["finite"] for shard in shards),
190 "sum": sum(shard["sum"] for shard in shards),
191 "l2": math.sqrt(sum(shard["squared_l2"] for shard in shards)),
192 "min": min(minima) if minima else None,
193 "max": max(maxima) if maxima else None,
194 "layout": reference["layout"],
195 "replica": {
196 "logical_shards": len(replica_groups),
197 "replicas_equal": all(len(digests) == 1 for digests in replica_groups.values()),
198 },
199 "missing_ranks": [
200 rank for rank, rank_summary in enumerate(rank_summaries) if rank_summary is None
201 ],
202 }
203 values = _flattened_shard_values(tensor, shards)
204 if values is not None:
205 summary["values"] = values
206 return summary
209def _tensor_summary(name: str, tensor: Any) -> Optional[dict[str, Any]]:
210 """Build one global summary while allowing rank-local lazy state absence."""
211 local_summary = None if tensor is None else _local_tensor_summary(tensor)
212 shards, rank_summaries = _gather_shard_summaries(local_summary)
213 if not shards:
214 return None
215 shards.sort(key=lambda item: item["shard"])
216 return _global_shard_summary(name, tensor, shards, rank_summaries)
219def _global_tensor_state_names(local_names: Iterable[str]) -> tuple[str, ...]:
220 """Return a rank-consistent tensor-state key order for collective probes."""
221 local_names = tuple(sorted(local_names))
222 rank_names = [local_names]
223 if dist.is_available() and dist.is_initialized():
224 rank_names = [None] * dist.get_world_size()
225 dist.all_gather_object(rank_names, local_names)
226 return tuple(sorted({name for names in rank_names for name in (names or ())}))
229class InputIdentityRecorder:
230 """Hash global step inputs from deterministic per-rank tensor payloads."""
232 def __init__(
233 self,
234 mesh_context: Any = None,
235 cp_replicated_forward_fields: Iterable[str] = (),
236 ) -> None:
237 """Configure mesh-aware canonicalization for one Trainer run."""
238 self.mesh_context = mesh_context
239 self.cp_replicated_forward_fields = set(cp_replicated_forward_fields)
240 self._entries: list[dict[str, Any]] = []
241 self._micro_steps = 0
243 def begin_step(self) -> None:
244 """Reset the current optimizer-step identity."""
245 self._entries = []
246 self._micro_steps = 0
248 @staticmethod
249 def _tensor_leaves(value: Any) -> list[Any]:
250 """Find tensors inside mappings, sequences, and metadata dataclasses."""
251 if torch.is_tensor(value):
252 return [value]
253 if dataclasses.is_dataclass(value) and not isinstance(value, type):
254 return [
255 tensor
256 for dataclass_field in dataclasses.fields(value)
257 for field_value in (getattr(value, dataclass_field.name),)
258 for tensor in InputIdentityRecorder._tensor_leaves(field_value)
259 ]
260 if isinstance(value, Mapping):
261 return [
262 tensor
263 for name in sorted(value)
264 for tensor in InputIdentityRecorder._tensor_leaves(value[name])
265 ]
266 if isinstance(value, (list, tuple)):
267 return [
268 tensor
269 for child in value
270 for tensor in InputIdentityRecorder._tensor_leaves(child)
271 ]
272 return []
274 def record(self, model_inputs: Mapping[str, Any], loss_inputs: Mapping[str, Any]) -> None:
275 """Add model/loss tensor fields without changing their device or values."""
276 for owner, values in (("model", model_inputs), ("loss", loss_inputs)):
277 for field_name in sorted(values):
278 flattened = self._tensor_leaves(values[field_name])
279 for value_index, value in enumerate(flattened):
280 local = value.detach()
281 to_local = getattr(local, "to_local", None)
282 if callable(to_local):
283 local = to_local()
284 self._entries.append(
285 {
286 "micro_step": self._micro_steps,
287 "owner": owner,
288 "field": field_name,
289 "value_index": value_index,
290 "cp_replicated": (
291 owner == "model"
292 and field_name in self.cp_replicated_forward_fields
293 ),
294 "tensor": local.cpu().contiguous(),
295 }
296 )
297 self._micro_steps += 1
299 def _coordinates(self) -> dict[str, int]:
300 return {
301 name: int(getattr(self.mesh_context, f"{name}_rank", 0))
302 for name in ("dp", "cp", "tp", "pp")
303 }
305 @staticmethod
306 def _hash_tensor(value: torch.Tensor) -> str:
307 hasher = hashlib.sha256()
308 hasher.update(f"{tuple(value.shape)}:{value.dtype}".encode())
309 byte_view = value.contiguous().view(torch.uint8)
310 hasher.update(byte_view.numpy().tobytes())
311 return hasher.hexdigest()
313 @staticmethod
314 def _require_input_payloads(rank_payloads: list[dict[str, Any]]) -> None:
315 """Reject collectives polluted by non-input rank payloads."""
316 malformed_ranks = [
317 rank
318 for rank, payload in enumerate(rank_payloads)
319 if not isinstance(payload, Mapping)
320 or "coordinates" not in payload
321 or "entries" not in payload
322 ]
323 if malformed_ranks:
324 raise RuntimeError(
325 "input-identity collective received non-input payloads from ranks "
326 f"{malformed_ranks}; parameter probes must execute a rank-consistent "
327 "collective sequence"
328 )
330 @staticmethod
331 def _group_input_entries(
332 rank_payloads: list[dict[str, Any]],
333 ) -> dict[tuple[Any, ...], list[tuple[int, torch.Tensor, bool]]]:
334 """Group entry shards by logical field across tp0/pp0 ranks."""
335 selected = [
336 payload
337 for payload in rank_payloads
338 if payload["coordinates"]["tp"] == 0 and payload["coordinates"]["pp"] == 0
339 ]
340 grouped: dict[tuple[Any, ...], list[tuple[int, torch.Tensor, bool]]] = {}
341 for payload in selected:
342 coordinates = payload["coordinates"]
343 for entry in payload["entries"]:
344 key = (
345 entry["micro_step"],
346 coordinates["dp"],
347 entry["owner"],
348 entry["field"],
349 entry["value_index"],
350 )
351 grouped.setdefault(key, []).append(
352 (
353 coordinates["cp"],
354 entry["tensor"],
355 bool(entry.get("cp_replicated", False)),
356 )
357 )
358 return grouped
360 @staticmethod
361 def _merge_entry_shards(
362 shards: list[tuple[int, torch.Tensor, bool]],
363 ) -> torch.Tensor:
364 """Rebuild one logical field tensor from ordered CP shards."""
365 ordered_shards = sorted(shards, key=lambda item: item[0])
366 ordered = [tensor for _, tensor, _ in ordered_shards]
367 tensor = ordered[0]
368 cp_replicated = all(replicated for _, _, replicated in ordered_shards)
369 if cp_replicated and any(
370 not torch.equal(tensor, replica) for replica in ordered[1:]
371 ):
372 cp_replicated = False
373 if len(ordered) > 1 and tensor.ndim > 0 and not cp_replicated:
374 try:
375 tensor = torch.cat(ordered, dim=-1)
376 except RuntimeError:
377 tensor = torch.cat([value.reshape(-1) for value in ordered])
378 return tensor
380 @classmethod
381 def _entry_component_hashes(
382 cls,
383 grouped: dict[tuple[Any, ...], list[tuple[int, torch.Tensor, bool]]],
384 ) -> list[str]:
385 """Hash every logical field occurrence independently of rank order."""
386 component_hashes = []
387 for key, shards in grouped.items():
388 tensor = cls._merge_entry_shards(shards)
389 field_identity = f"{key[2]}:{key[3]}:{key[4]}"
390 if tensor.ndim > 0:
391 values = tensor.reshape(tensor.shape[0], -1)
392 component_hashes.extend(
393 f"{field_identity}:{cls._hash_tensor(value)}" for value in values
394 )
395 else:
396 component_hashes.append(f"{field_identity}:{cls._hash_tensor(tensor)}")
397 return component_hashes
399 @classmethod
400 def _canonical_global_hash(cls, rank_payloads: list[dict[str, Any]]) -> str:
401 cls._require_input_payloads(rank_payloads)
402 grouped = cls._group_input_entries(rank_payloads)
403 component_hashes = cls._entry_component_hashes(grouped)
404 hasher = hashlib.sha256()
405 for component_hash in sorted(component_hashes):
406 hasher.update(component_hash.encode())
407 return hasher.hexdigest()
409 def finish_step(self) -> dict[str, Any]:
410 """Return rank and global digests for the optimizer step."""
411 local_payload = {
412 "coordinates": self._coordinates(),
413 "entries": self._entries,
414 }
415 rank_payloads = [local_payload]
416 if dist.is_available() and dist.is_initialized():
417 rank_payloads = [None] * dist.get_world_size()
418 dist.all_gather_object(rank_payloads, local_payload)
419 return {
420 "global_sha256": self._canonical_global_hash(rank_payloads),
421 "rank_count": len(rank_payloads),
422 "micro_steps": self._micro_steps,
423 }
426class ParameterProbeRecorder:
427 """Capture canonical parameter/main-gradient/update evidence by semantic watchlist."""
429 def __init__(
430 self,
431 model: Any,
432 optimizer: Any,
433 evidence_store: EvidenceStore,
434 *,
435 rank: int,
436 validation_spec: Optional[ModelValidationSpec] = None,
437 include_small_parameters_numel_le: int = _SMALL_TENSOR_NUMEL,
438 representative_parameter_names: Iterable[str] = (),
439 ) -> None:
440 """Resolve a stable watchlist from the final model and adapter declarations."""
441 self.model = model
442 self.optimizer = optimizer
443 self.evidence_store = evidence_store
444 self.rank = rank
445 self.validation_spec = validation_spec
446 self.include_small_parameters_numel_le = include_small_parameters_numel_le
447 self.representative_parameter_names = set(representative_parameter_names)
448 self._named_parameters = dict(model.named_parameters(remove_duplicate=False))
449 self._probes = self._resolve_probes()
450 self._validate_required_patterns()
451 self._step = 0
453 def _resolve_probes(self) -> dict[str, ParameterProbeSpec]:
454 probes = {}
455 declared = self.validation_spec.parameter_probes if self.validation_spec else ()
456 replaced_module_fqns = tuple(
457 getattr(self.model, "_hp_replaced_module_fqns", ()) or ()
458 )
459 for parameter_name, parameter in self._named_parameters.items():
460 main_parameter = getattr(parameter, "main_param", parameter)
461 replacement_parameter = any(
462 parameter_name == module_fqn
463 or parameter_name.startswith(f"{module_fqn}.")
464 for module_fqn in replaced_module_fqns
465 )
466 if (
467 replacement_parameter
468 or parameter_name in self.representative_parameter_names
469 or _tensor_numel(main_parameter) <= self.include_small_parameters_numel_le
470 ):
471 probes[parameter_name] = ParameterProbeSpec(
472 match=parameter_name,
473 required=True,
474 stages=(
475 "model_before",
476 "main_before",
477 "main_grad_before_clip",
478 "main_grad_after_clip",
479 "main_after_optimizer",
480 "model_after_copy_back",
481 "optimizer_state",
482 ),
483 )
484 for probe in declared:
485 if fnmatch.fnmatchcase(parameter_name, probe.match):
486 probes[parameter_name] = probe
487 return dict(sorted(probes.items()))
489 def _validate_required_patterns(self) -> None:
490 for probe in self.validation_spec.parameter_probes if self.validation_spec else ():
491 if probe.required and not any(
492 fnmatch.fnmatchcase(name, probe.match) for name in self._named_parameters
493 ):
494 raise ValueError(
495 f"required parameter probe {probe.match!r} matched no final parameter"
496 )
498 @property
499 def watched_parameter_names(self) -> tuple[str, ...]:
500 """Return stable final FQNs in capture order."""
501 return tuple(self._probes)
503 def begin_step(self, step: int) -> None:
504 """Start evidence for one optimizer step."""
505 self._step = step
506 self.capture("model_before")
507 self.capture("main_before")
509 @staticmethod
510 def _stage_tensor(parameter: Any, stage: str) -> Any:
511 main_parameter = getattr(parameter, "main_param", parameter)
512 if stage in ("model_before", "model_after_copy_back"):
513 return parameter
514 if stage in ("main_before", "main_after_optimizer"):
515 return main_parameter
516 if stage in ("main_grad_before_clip", "main_grad_after_clip"):
517 main_gradient = getattr(parameter, "main_grad", None)
518 return main_gradient if main_gradient is not None else getattr(main_parameter, "grad", None)
519 return None
521 def _optimizer_state(self, parameter: Any) -> dict[str, Any]:
522 main_parameter = (
523 None if parameter is None else getattr(parameter, "main_param", parameter)
524 )
525 state = (
526 {} if main_parameter is None
527 else optimizer_state_for_parameter(self.optimizer, main_parameter)
528 )
529 state_names = _global_tensor_state_names(
530 name for name, value in state.items() if torch.is_tensor(value)
531 )
532 summaries = {}
533 for name in state_names:
534 value = state.get(name)
535 summary = _tensor_summary(name, value if torch.is_tensor(value) else None)
536 if summary is not None:
537 summaries[name] = summary
538 return summaries
540 def capture(self, stage: str) -> None:
541 """Capture global summaries and persist one canonical rank-zero copy."""
542 records = []
543 for parameter_name, probe in self._probes.items():
544 if stage not in probe.stages:
545 continue
546 parameter = self._named_parameters.get(parameter_name)
547 if stage == "optimizer_state":
548 summary = self._optimizer_state(parameter)
549 if not summary:
550 continue
551 record = {"name": parameter_name, "optimizer_state": summary}
552 else:
553 tensor = None if parameter is None else self._stage_tensor(parameter, stage)
554 record = _tensor_summary(parameter_name, tensor)
555 if record is None:
556 continue
557 if not probe.check_replica_consistency:
558 record.pop("replica", None)
559 record.update(
560 {
561 "step": self._step,
562 "stage": stage,
563 "rank": self.rank,
564 "comparison": probe.compare,
565 }
566 )
567 records.append(record)
568 if self.rank != 0:
569 return
570 for record in records:
571 self.evidence_store.append_jsonl(
572 f"parameter_probes/rank{self.rank}.jsonl",
573 record,
574 )
577__all__ = [
578 "InputIdentityRecorder",
579 "ParameterProbeRecorder",
580]