Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / trainer / utils / discovery.py: 0%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Model spec auto-discovery — resolve ``model.name`` to a Python package. 

16 

17import_module`` pattern in 

18```: the entry script never hard-codes any model 

19import; the spec is loaded on demand based on the YAML config. 

20 

21Naming Convention 

22----------------- 

23``model.name`` is **both** the spec key and the import path: 

24 

25- **Built-in**: ``model.name: qwen3_5`` → imports 

26 ``hyper_parallel.models.qwen3_5`` whose ``__init__.py`` must call 

27 ``register_spec("qwen3_5", ...)`` at import time. 

28- **External**: any name containing ``.`` is treated as a fully-qualified 

29 module path (e.g. ``my_org.research_models.my_moe``). The trainer 

30 imports it as-is so external packages can ship their own specs. 

31 

32Why convention-over-config 

33-------------------------- 

34- Reuses ``model.name`` as both the spec key AND the import path. No new 

35 YAML field, no name → path mapping table to maintain. 

36- Single source of truth: the package name IS the model name. 

37- Adding a new built-in model = create ``models/<name>/__init__.py`` that 

38 calls ``register_spec("<name>", ...)``. Zero changes to entry scripts 

39 or trainer code. 

40""" 

41import importlib 

42import logging 

43from pkgutil import iter_modules 

44 

45logger = logging.getLogger(__name__) 

46 

47_BUILTIN_NAMESPACE = "hyper_parallel.models" 

48_EXCLUDE_FROM_LIST = frozenset({"modules", "spec"}) # shared blocks / spec layer, not models 

49 

50def discover_model_spec(name: str) -> str: 

51 """Import the package that registers ``ModelSpec`` for ``name``. 

52 

53 The import side-effect is what calls ``register_spec`` and populates 

54 the global ``_SPEC_REGISTRY``. The trainer's later ``get_spec(name)`` 

55 then succeeds. 

56 

57 Args: 

58 name: Either a built-in model name (e.g. ``"qwen3_5"``) or a 

59 fully-qualified package path (e.g. ``"my_pkg.my_model"``). 

60 A name is treated as fully-qualified iff it contains ``.``. 

61 

62 Returns: 

63 The fully-qualified module path that was imported. 

64 

65 Raises: 

66 ImportError: With the list of available built-in models when the 

67 name cannot be resolved. The message tells the user how to 

68 register an external model. 

69 """ 

70 candidate = name if "." in name else f"{_BUILTIN_NAMESPACE}.{name}" 

71 try: 

72 importlib.import_module(candidate) 

73 except ImportError as exc: 

74 available = sorted(_list_builtin_models()) 

75 raise ImportError( 

76 f"Cannot import model package '{candidate}' for " 

77 f"model.name='{name}'. Built-in models: {available}. " 

78 f"For an external model, set model.name to a fully-qualified " 

79 f"package path (e.g. 'my_org.my_model'); the package's " 

80 f"__init__.py must call register_spec(...)." 

81 ) from exc 

82 logger.info("Model spec auto-discovered: model.name=%s -> %s", name, candidate) 

83 return candidate 

84 

85def _list_builtin_models() -> list[str]: 

86 """List subpackages under ``hyper_parallel.models`` that look like models. 

87 

88 Excludes ``common`` (shared building blocks, not a model). Used only 

89 for friendly error messages — no caching, called at most once per 

90 failed lookup. 

91 """ 

92 try: 

93 pkg = importlib.import_module(_BUILTIN_NAMESPACE) 

94 except ImportError: 

95 return [] 

96 return [ 

97 m.name for m in iter_modules(pkg.__path__) 

98 if m.ispkg and m.name not in _EXCLUDE_FROM_LIST 

99 ]