Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / tools / model_integration / optimizer_layout.py: 28%

88 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Optimizer and checkpoint-layout observations for Trainer validation.""" 

16 

17from __future__ import annotations 

18 

19import math 

20from collections.abc import Mapping 

21from typing import Any 

22 

23import torch 

24 

25from hyper_parallel.tools.model_integration.schemas import ( 

26 FindingSeverity, 

27 ModelIntegrationFinding, 

28 SCHEMA_VERSION, 

29) 

30 

31 

32def tensor_layout(value: Any) -> dict[str, Any]: 

33 """Return logical/local shape and distributed layout without custom chunks.""" 

34 to_local = getattr(value, "to_local", None) 

35 local = to_local() if callable(to_local) else value 

36 layout = getattr(value, "layout", None) 

37 mesh = getattr(value, "device_mesh", None) 

38 placements = getattr(value, "placements", None) 

39 if layout is not None: 

40 mesh = getattr(layout, "mesh", mesh) 

41 placements = getattr(layout, "placements", placements) 

42 mesh_tensor = getattr(mesh, "mesh", None) 

43 return { 

44 "global_shape": [int(size) for size in getattr(value, "shape", ())], 

45 "local_shape": [int(size) for size in getattr(local, "shape", ())], 

46 "dtype": str(getattr(value, "dtype", None)), 

47 "device": str(getattr(local, "device", None)), 

48 "mesh_dim_names": list(getattr(mesh, "mesh_dim_names", ()) or ()), 

49 "mesh_shape": [int(size) for size in getattr(mesh_tensor, "shape", ())], 

50 "placements": [repr(placement) for placement in (placements or ())], 

51 "is_dtensor": callable(to_local), 

52 } 

53 

54 

55def _same_logical_layout(first: Any, second: Any) -> bool: 

56 first_layout = tensor_layout(first) 

57 second_layout = tensor_layout(second) 

58 return all( 

59 first_layout[name] == second_layout[name] 

60 for name in ("global_shape", "mesh_dim_names", "mesh_shape", "placements") 

61 ) 

62 

63 

64def optimizer_leaves(optimizer: Any) -> tuple[Any, ...]: 

65 """Return optimizer leaves through list, chain, and wrapper containers.""" 

66 chained = getattr(optimizer, "chained_optimizers", None) 

67 if chained is not None: 

68 return tuple(leaf for item in chained for leaf in optimizer_leaves(item)) 

69 inner = getattr(optimizer, "optimizer", None) 

70 if inner is not None and inner is not optimizer: 

71 return optimizer_leaves(inner) 

72 if isinstance(optimizer, (list, tuple)): 

73 return tuple(leaf for item in optimizer for leaf in optimizer_leaves(item)) 

74 return (optimizer,) 

75 

76 

77def optimizer_state_for_parameter(optimizer: Any, parameter: Any) -> Mapping[str, Any]: 

78 """Find one parameter's state through supported optimizer wrappers.""" 

79 for leaf in optimizer_leaves(optimizer): 

80 state = getattr(leaf, "state", None) 

81 if isinstance(state, Mapping) and parameter in state: 

82 value = state[parameter] 

83 return value if isinstance(value, Mapping) else {} 

84 return {} 

85 

86 

87def _layout_mismatch_finding( 

88 parameter_fqn: str, 

89 owner_name: str, 

90 expected: Any, 

91 actual: Any, 

92) -> ModelIntegrationFinding: 

93 return ModelIntegrationFinding( 

94 phase="D6", 

95 owner_fqn=parameter_fqn, 

96 severity=FindingSeverity.ERROR, 

97 message=f"{owner_name} does not preserve its parameter's global layout", 

98 facts={ 

99 "parameter": tensor_layout(expected), 

100 owner_name: tensor_layout(actual), 

101 }, 

102 why_unsafe=( 

103 "optimizer state and gradients must describe the same logical tensor; " 

104 "a local-only or differently placed value cannot be resharded safely" 

105 ), 

106 remediation=( 

107 "preserve the parameter DTensor global shape, mesh, and placements when " 

108 "creating main gradients, main parameters, and optimizer moments" 

109 ), 

110 related_config=("optimizer.fp32_main_params", "checkpoint.restore_optimizer"), 

111 ) 

112 

113 

114def validate_optimizer_layout(model: Any, optimizer: Any) -> list[ModelIntegrationFinding]: 

115 """Validate param/main-param/main-grad/moment layouts from live objects.""" 

116 findings = [] 

117 for parameter_fqn, model_parameter in model.named_parameters(remove_duplicate=False): 

118 main_parameter = getattr(model_parameter, "main_param", model_parameter) 

119 if not _same_logical_layout(model_parameter, main_parameter): 

120 findings.append( 

121 _layout_mismatch_finding( 

122 parameter_fqn, 

123 "main_parameter", 

124 model_parameter, 

125 main_parameter, 

126 ) 

127 ) 

128 continue 

129 gradient = getattr(model_parameter, "main_grad", None) 

130 if gradient is None: 

131 gradient = getattr(main_parameter, "grad", None) 

132 if gradient is not None and not _same_logical_layout(main_parameter, gradient): 

133 findings.append( 

134 _layout_mismatch_finding( 

135 parameter_fqn, 

136 "main_gradient", 

137 main_parameter, 

138 gradient, 

139 ) 

140 ) 

141 for state_name, state_value in optimizer_state_for_parameter( 

142 optimizer, main_parameter 

143 ).items(): 

144 if torch.is_tensor(state_value) and state_value.ndim > 0: 

145 if not _same_logical_layout(main_parameter, state_value): 

146 findings.append( 

147 _layout_mismatch_finding( 

148 parameter_fqn, 

149 f"optimizer_state.{state_name}", 

150 main_parameter, 

151 state_value, 

152 ) 

153 ) 

154 layout = tensor_layout(main_parameter) 

155 if ( 

156 math.prod(layout["global_shape"]) 

157 and not math.prod(layout["local_shape"]) 

158 and not layout["is_dtensor"] 

159 ): 

160 findings.append( 

161 ModelIntegrationFinding( 

162 phase="D8", 

163 owner_fqn=parameter_fqn, 

164 severity=FindingSeverity.ERROR, 

165 message="empty optimizer shard has no global DTensor layout", 

166 facts={"main_parameter": layout}, 

167 why_unsafe=( 

168 "DCP cannot place an empty local tensor into its logical global tensor " 

169 "without mesh and placement metadata" 

170 ), 

171 remediation="keep uneven FSDP shards as DTensors through optimizer state creation", 

172 related_config=("parallel.fsdp", "optimizer.fp32_main_params"), 

173 ) 

174 ) 

175 return findings 

176 

177 

178def snapshot_training_layout(model: Any, optimizer: Any) -> dict[str, Any]: 

179 """Build model/main-gradient/optimizer-state layout evidence.""" 

180 records = {} 

181 for parameter_fqn, model_parameter in model.named_parameters(remove_duplicate=False): 

182 main_parameter = getattr(model_parameter, "main_param", model_parameter) 

183 record = { 

184 "model_parameter": tensor_layout(model_parameter), 

185 "main_parameter": tensor_layout(main_parameter), 

186 } 

187 gradient = getattr(model_parameter, "main_grad", None) 

188 if gradient is None: 

189 gradient = getattr(main_parameter, "grad", None) 

190 if gradient is not None: 

191 record["main_gradient"] = tensor_layout(gradient) 

192 state = optimizer_state_for_parameter(optimizer, main_parameter) 

193 record["optimizer_state"] = { 

194 name: tensor_layout(value) 

195 for name, value in sorted(state.items()) 

196 if torch.is_tensor(value) 

197 } 

198 records[parameter_fqn] = record 

199 return {"schema_version": SCHEMA_VERSION, "parameters": records} 

200 

201 

202def flatten_tensor_layouts(value: Any, prefix: str = "") -> dict[str, dict[str, Any]]: 

203 """Flatten tensor leaves from a checkpoint payload into layout records.""" 

204 records = {} 

205 if torch.is_tensor(value): 

206 records[prefix or "<root>"] = tensor_layout(value) 

207 elif isinstance(value, Mapping): 

208 for name, child in value.items(): 

209 child_prefix = f"{prefix}.{name}" if prefix else str(name) 

210 records.update(flatten_tensor_layouts(child, child_prefix)) 

211 elif isinstance(value, (list, tuple)): 

212 for index, child in enumerate(value): 

213 child_prefix = f"{prefix}.{index}" if prefix else str(index) 

214 records.update(flatten_tensor_layouts(child, child_prefix)) 

215 return records 

216 

217 

218__all__ = [ 

219 "flatten_tensor_layouts", 

220 "optimizer_leaves", 

221 "optimizer_state_for_parameter", 

222 "snapshot_training_layout", 

223 "tensor_layout", 

224 "validate_optimizer_layout", 

225]