Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / optimizer_layout.py: 28%
88 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Optimizer and checkpoint-layout observations for Trainer validation."""
17from __future__ import annotations
19import math
20from collections.abc import Mapping
21from typing import Any
23import torch
25from hyper_parallel.tools.model_integration.schemas import (
26 FindingSeverity,
27 ModelIntegrationFinding,
28 SCHEMA_VERSION,
29)
32def tensor_layout(value: Any) -> dict[str, Any]:
33 """Return logical/local shape and distributed layout without custom chunks."""
34 to_local = getattr(value, "to_local", None)
35 local = to_local() if callable(to_local) else value
36 layout = getattr(value, "layout", None)
37 mesh = getattr(value, "device_mesh", None)
38 placements = getattr(value, "placements", None)
39 if layout is not None:
40 mesh = getattr(layout, "mesh", mesh)
41 placements = getattr(layout, "placements", placements)
42 mesh_tensor = getattr(mesh, "mesh", None)
43 return {
44 "global_shape": [int(size) for size in getattr(value, "shape", ())],
45 "local_shape": [int(size) for size in getattr(local, "shape", ())],
46 "dtype": str(getattr(value, "dtype", None)),
47 "device": str(getattr(local, "device", None)),
48 "mesh_dim_names": list(getattr(mesh, "mesh_dim_names", ()) or ()),
49 "mesh_shape": [int(size) for size in getattr(mesh_tensor, "shape", ())],
50 "placements": [repr(placement) for placement in (placements or ())],
51 "is_dtensor": callable(to_local),
52 }
55def _same_logical_layout(first: Any, second: Any) -> bool:
56 first_layout = tensor_layout(first)
57 second_layout = tensor_layout(second)
58 return all(
59 first_layout[name] == second_layout[name]
60 for name in ("global_shape", "mesh_dim_names", "mesh_shape", "placements")
61 )
64def optimizer_leaves(optimizer: Any) -> tuple[Any, ...]:
65 """Return optimizer leaves through list, chain, and wrapper containers."""
66 chained = getattr(optimizer, "chained_optimizers", None)
67 if chained is not None:
68 return tuple(leaf for item in chained for leaf in optimizer_leaves(item))
69 inner = getattr(optimizer, "optimizer", None)
70 if inner is not None and inner is not optimizer:
71 return optimizer_leaves(inner)
72 if isinstance(optimizer, (list, tuple)):
73 return tuple(leaf for item in optimizer for leaf in optimizer_leaves(item))
74 return (optimizer,)
77def optimizer_state_for_parameter(optimizer: Any, parameter: Any) -> Mapping[str, Any]:
78 """Find one parameter's state through supported optimizer wrappers."""
79 for leaf in optimizer_leaves(optimizer):
80 state = getattr(leaf, "state", None)
81 if isinstance(state, Mapping) and parameter in state:
82 value = state[parameter]
83 return value if isinstance(value, Mapping) else {}
84 return {}
87def _layout_mismatch_finding(
88 parameter_fqn: str,
89 owner_name: str,
90 expected: Any,
91 actual: Any,
92) -> ModelIntegrationFinding:
93 return ModelIntegrationFinding(
94 phase="D6",
95 owner_fqn=parameter_fqn,
96 severity=FindingSeverity.ERROR,
97 message=f"{owner_name} does not preserve its parameter's global layout",
98 facts={
99 "parameter": tensor_layout(expected),
100 owner_name: tensor_layout(actual),
101 },
102 why_unsafe=(
103 "optimizer state and gradients must describe the same logical tensor; "
104 "a local-only or differently placed value cannot be resharded safely"
105 ),
106 remediation=(
107 "preserve the parameter DTensor global shape, mesh, and placements when "
108 "creating main gradients, main parameters, and optimizer moments"
109 ),
110 related_config=("optimizer.fp32_main_params", "checkpoint.restore_optimizer"),
111 )
114def validate_optimizer_layout(model: Any, optimizer: Any) -> list[ModelIntegrationFinding]:
115 """Validate param/main-param/main-grad/moment layouts from live objects."""
116 findings = []
117 for parameter_fqn, model_parameter in model.named_parameters(remove_duplicate=False):
118 main_parameter = getattr(model_parameter, "main_param", model_parameter)
119 if not _same_logical_layout(model_parameter, main_parameter):
120 findings.append(
121 _layout_mismatch_finding(
122 parameter_fqn,
123 "main_parameter",
124 model_parameter,
125 main_parameter,
126 )
127 )
128 continue
129 gradient = getattr(model_parameter, "main_grad", None)
130 if gradient is None:
131 gradient = getattr(main_parameter, "grad", None)
132 if gradient is not None and not _same_logical_layout(main_parameter, gradient):
133 findings.append(
134 _layout_mismatch_finding(
135 parameter_fqn,
136 "main_gradient",
137 main_parameter,
138 gradient,
139 )
140 )
141 for state_name, state_value in optimizer_state_for_parameter(
142 optimizer, main_parameter
143 ).items():
144 if torch.is_tensor(state_value) and state_value.ndim > 0:
145 if not _same_logical_layout(main_parameter, state_value):
146 findings.append(
147 _layout_mismatch_finding(
148 parameter_fqn,
149 f"optimizer_state.{state_name}",
150 main_parameter,
151 state_value,
152 )
153 )
154 layout = tensor_layout(main_parameter)
155 if (
156 math.prod(layout["global_shape"])
157 and not math.prod(layout["local_shape"])
158 and not layout["is_dtensor"]
159 ):
160 findings.append(
161 ModelIntegrationFinding(
162 phase="D8",
163 owner_fqn=parameter_fqn,
164 severity=FindingSeverity.ERROR,
165 message="empty optimizer shard has no global DTensor layout",
166 facts={"main_parameter": layout},
167 why_unsafe=(
168 "DCP cannot place an empty local tensor into its logical global tensor "
169 "without mesh and placement metadata"
170 ),
171 remediation="keep uneven FSDP shards as DTensors through optimizer state creation",
172 related_config=("parallel.fsdp", "optimizer.fp32_main_params"),
173 )
174 )
175 return findings
178def snapshot_training_layout(model: Any, optimizer: Any) -> dict[str, Any]:
179 """Build model/main-gradient/optimizer-state layout evidence."""
180 records = {}
181 for parameter_fqn, model_parameter in model.named_parameters(remove_duplicate=False):
182 main_parameter = getattr(model_parameter, "main_param", model_parameter)
183 record = {
184 "model_parameter": tensor_layout(model_parameter),
185 "main_parameter": tensor_layout(main_parameter),
186 }
187 gradient = getattr(model_parameter, "main_grad", None)
188 if gradient is None:
189 gradient = getattr(main_parameter, "grad", None)
190 if gradient is not None:
191 record["main_gradient"] = tensor_layout(gradient)
192 state = optimizer_state_for_parameter(optimizer, main_parameter)
193 record["optimizer_state"] = {
194 name: tensor_layout(value)
195 for name, value in sorted(state.items())
196 if torch.is_tensor(value)
197 }
198 records[parameter_fqn] = record
199 return {"schema_version": SCHEMA_VERSION, "parameters": records}
202def flatten_tensor_layouts(value: Any, prefix: str = "") -> dict[str, dict[str, Any]]:
203 """Flatten tensor leaves from a checkpoint payload into layout records."""
204 records = {}
205 if torch.is_tensor(value):
206 records[prefix or "<root>"] = tensor_layout(value)
207 elif isinstance(value, Mapping):
208 for name, child in value.items():
209 child_prefix = f"{prefix}.{name}" if prefix else str(name)
210 records.update(flatten_tensor_layouts(child, child_prefix))
211 elif isinstance(value, (list, tuple)):
212 for index, child in enumerate(value):
213 child_prefix = f"{prefix}.{index}" if prefix else str(index)
214 records.update(flatten_tensor_layouts(child, child_prefix))
215 return records
218__all__ = [
219 "flatten_tensor_layouts",
220 "optimizer_leaves",
221 "optimizer_state_for_parameter",
222 "snapshot_training_layout",
223 "tensor_layout",
224 "validate_optimizer_layout",
225]