Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / trainer / utils / discovery.py: 0%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Model spec auto-discovery — resolve ``model.name`` to a Python package. 

16 

17import_module`` pattern in 

18```: the entry script never hard-codes any model 

19import; the spec is loaded on demand based on the YAML config. 

20 

21Naming Convention 

22----------------- 

23``model.name`` is **both** the spec key and the import path: 

24 

25- **Built-in**: ``model.name: qwen3_5`` → imports 

26 ``hyper_parallel.models.qwen3_5`` whose ``__init__.py`` must call 

27 ``register_spec("qwen3_5", ...)`` at import time. 

28- **External**: any name containing ``.`` is treated as a fully-qualified 

29 module path (e.g. ``my_org.research_models.my_moe``). The trainer 

30 imports it as-is so external packages can ship their own specs. 

31 

32Why convention-over-config 

33-------------------------- 

34- Reuses ``model.name`` as both the spec key AND the import path. No new 

35 YAML field, no name → path mapping table to maintain. 

36- Single source of truth: the package name IS the model name. 

37- Adding a new built-in model = create ``models/<name>/__init__.py`` that 

38 calls ``register_spec("<name>", ...)``. Zero changes to entry scripts 

39 or trainer code. 

40""" 

41import importlib 

42import logging 

43from pkgutil import iter_modules 

44 

45logger = logging.getLogger(__name__) 

46 

47_BUILTIN_NAMESPACE = "hyper_parallel.models" 

48_EXCLUDE_FROM_LIST = frozenset({"modules", "spec"}) # shared blocks / spec layer, not models 

49 

50 

51def discover_model_spec(name: str) -> str: 

52 """Import the package that registers ``ModelSpec`` for ``name``. 

53 

54 The import side-effect is what calls ``register_spec`` and populates 

55 the global ``_SPEC_REGISTRY``. The trainer's later ``get_spec(name)`` 

56 then succeeds. 

57 

58 Args: 

59 name: Either a built-in model name (e.g. ``"qwen3_5"``) or a 

60 fully-qualified package path (e.g. ``"my_pkg.my_model"``). 

61 A name is treated as fully-qualified iff it contains ``.``. 

62 

63 Returns: 

64 The fully-qualified module path that was imported. 

65 

66 Raises: 

67 ImportError: With the list of available built-in models when the 

68 name cannot be resolved. The message tells the user how to 

69 register an external model. 

70 """ 

71 candidate = name if "." in name else f"{_BUILTIN_NAMESPACE}.{name}" 

72 try: 

73 importlib.import_module(candidate) 

74 except ImportError as exc: 

75 available = sorted(_list_builtin_models()) 

76 raise ImportError( 

77 f"Cannot import model package '{candidate}' for " 

78 f"model.name='{name}'. Built-in models: {available}. " 

79 f"For an external model, set model.name to a fully-qualified " 

80 f"package path (e.g. 'my_org.my_model'); the package's " 

81 f"__init__.py must call register_spec(...)." 

82 ) from exc 

83 logger.info("Model spec auto-discovered: model.name=%s -> %s", name, candidate) 

84 return candidate 

85 

86 

87def _list_builtin_models() -> list[str]: 

88 """List subpackages under ``hyper_parallel.models`` that look like models. 

89 

90 Excludes ``common`` (shared building blocks, not a model). Used only 

91 for friendly error messages — no caching, called at most once per 

92 failed lookup. 

93 """ 

94 try: 

95 pkg = importlib.import_module(_BUILTIN_NAMESPACE) 

96 except ImportError: 

97 return [] 

98 return [ 

99 m.name for m in iter_modules(pkg.__path__) 

100 if m.ispkg and m.name not in _EXCLUDE_FROM_LIST 

101 ]