Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/trainer/config/parser.py 86.7% 47,52,84-87
hyper_parallel/trainer/config/resolver.py 65.1% 67,71,80,85,89,96,99,121-122,144,150,167,182,216,237-239,241,248,251,268-271,278-281,303,305,321,365-366,412,416,434-435,438,465,474,479,488,493,506,524,558,560,567,574,596,601,607,636,665,695,709,727,732,741,755
hyper_parallel/trainer/config/parser.py
43
44
45
46
47
48
49
50
51
52
53
54
55
                "CLI", f"expected a dotted override in '--field=value' form, got {token!r}"
            )
        option = token[2:]
        if "=" not in option:
            raise ConfigResolutionError(
                "CLI", f"expected a dotted override in '--field=value' form, got {token!r}"
            )
        path, raw_value = option.split("=", 1)
        if not path or any(not part for part in path.split(".")):
            raise ConfigResolutionError("CLI", f"invalid override path {path!r}")
        parsed.append((path, raw_value))
    return parsed

80
81
82
83
84
85
86
87
88
89
90
    """Load a YAML configuration and apply CLI overrides."""
    config_path = Path(config_file)
    try:
        raw = yaml.safe_load(config_path.read_text(encoding="utf-8"))
    except OSError as exc:
        raise ConfigResolutionError(config_path, f"could not read config file: {exc}") from exc
    except yaml.YAMLError as exc:
        raise ConfigResolutionError(config_path, f"invalid YAML: {exc}") from exc

    config = resolve_config(raw)
    return _apply_typed_overrides(config, cli_overrides)
hyper_parallel/trainer/config/resolver.py
63
64
65
66
67
68
69
70
71
72
73
74
75
        ConfigResolutionError: The path is invalid, cannot be imported, or
            does not identify a callable.
    """
    if not isinstance(target_path, str) or not target_path.strip():
        raise ConfigResolutionError(location, "_target_ must be a non-empty dotted path")

    parts = target_path.split(".")
    if any(not part for part in parts):
        raise ConfigResolutionError(location, f"invalid target path {target_path!r}")

    for split_at in range(len(parts), 0, -1):
        module_name = ".".join(parts[:split_at])
        try:
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
            target = importlib.import_module(module_name)
        except ModuleNotFoundError as exc:
            if exc.name == module_name or module_name.startswith(f"{exc.name}."):
                continue
            raise ConfigResolutionError(
                location,
                f"target {target_path!r} failed while importing dependency {exc.name!r}",
            ) from exc
        except ImportError as exc:
            raise ConfigResolutionError(location, f"target {target_path!r} could not be imported: {exc}") from exc

        for attribute in parts[split_at:]:
            if not hasattr(target, attribute):
                raise ConfigResolutionError(
                    location,
                    f"target {target_path!r} has no attribute {attribute!r}",
                )
            target = getattr(target, attribute)
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
                )
            target = getattr(target, attribute)

        if not callable(target):
            raise ConfigResolutionError(location, f"target {target_path!r} is not callable")
        return target

    raise ConfigResolutionError(location, f"target {target_path!r} could not be imported")


def _is_union(annotation: object) -> bool:
    """Return whether the annotation is a ``Union`` or a PEP 604 union."""
117
118
119
120
121
122
123
124
125
126
    """Resolve annotations for a target function or class constructor."""
    hint_source = target.__init__ if inspect.isclass(target) else target
    try:
        return get_type_hints(hint_source)
    except (NameError, TypeError) as exc:
        raise ConfigResolutionError(path, f"could not resolve target type annotations: {exc}") from exc


# Annotations whose payload is a container of other values.
_COLLECTION_TYPES: tuple[type, ...] = (list, tuple, dict, Mapping)
140
141
142
143
144
145
146
147
148
    ``item_types`` must contain one type per element. Empty ``item_types``
    produces a tuple without normalizing its items.
    """
    if not isinstance(value, (list, tuple)):
        raise ConfigResolutionError(path, f"expected {kind}, got {type(value).__name__}")
    if not item_types:
        return tuple(value)
    if uniform:
        item_types = item_types * len(value)
146
147
148
149
150
151
152
153
154
        return tuple(value)
    if uniform:
        item_types = item_types * len(value)
    if len(item_types) != len(value):
        raise ConfigResolutionError(
            path,
            f"expected {kind} of length {len(item_types)}, got {len(value)}",
        )
    normalized = [
163
164
165
166
167
168
169
170
171
    if annotation is types.NoneType or (
        _is_union(annotation) and types.NoneType in get_args(annotation)
    ):
        return None
    raise ConfigResolutionError(path, f"expected {_annotation_name(annotation)}, got None")


def _normalize_union(value: object, annotation: object, *, path: str) -> object:
    """Normalize a value against the first compatible union member."""
178
179
180
181
182
183
184
185
        try:
            return normalize_value(value, member, path=path)
        except ConfigResolutionError:
            continue
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(value).__name__}",
    )
212
213
214
215
216
217
218
219
220
                return word
    if any(type(value) is type(choice) and value == choice for choice in choices):
        return value
    expected = ", ".join(repr(choice) for choice in choices)
    raise ConfigResolutionError(path, f"expected one of ({expected}), got {value!r}")


def _normalize_collection(
    value: object,
233
234
235
236
237
238
239
240
241
242
243
244
245
            kind="list",
            path=path,
        )
    if origin is tuple or annotation is tuple:
        if len(args) == 2 and args[1] is Ellipsis:
            return _normalize_sequence(value, args[:1], uniform=True, kind="tuple", path=path)
        return _normalize_sequence(value, args, uniform=False, kind="tuple", path=path)
    if not isinstance(value, Mapping):
        raise ConfigResolutionError(path, f"expected mapping, got {type(value).__name__}")
    return dict(value)


def _normalize_class(value: object, annotation: object, *, path: str) -> object:
244
245
246
247
248
249
250
251
252
253
254

def _normalize_class(value: object, annotation: object, *, path: str) -> object:
    """Validate that a value is an instance of the annotated class."""
    if not isinstance(annotation, type):
        raise ConfigResolutionError(path, f"unsupported type annotation {_annotation_name(annotation)}")
    if isinstance(value, annotation):
        return value
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(value).__name__}",
    )
264
265
266
267
268
269
270
271
272
273
274
275
    if not isinstance(value, str):
        return value
    target = annotation
    if get_origin(target) in (Union, types.UnionType):
        members = [member for member in get_args(target) if member is not types.NoneType]
        if len(members) != 1:
            return value
        target = members[0]
    if target is int:
        try:
            return int(value)
        except ValueError:
274
275
276
277
278
279
280
281
282
283
284
285
            return int(value)
        except ValueError:
            return value
    if target is float:
        try:
            return float(value)
        except ValueError:
            return value
    return value


def normalize_value(value: object, annotation: object, *, path: str) -> object:
299
300
301
302
303
304
305
306
307
308
309
    """
    if annotation in (Any, object):
        return value
    if isinstance(annotation, dataclasses.InitVar):
        return normalize_value(value, annotation.type, path=path)
    if value is None:
        return _require_none_allowed(annotation, path=path)
    if _is_union(annotation):
        return _normalize_union(value, annotation, path=path)
    if annotation in (bool, int, float, str):
        return _normalize_scalar(value, annotation, path=path)
317
318
319
320
321
322
323
324
325
    if isinstance(annotation, type) and dataclasses.is_dataclass(annotation):
        # Nested dataclass items resolve from mappings in the same way as
        # top-level dataclass components.
        return _resolve_dataclass(value, annotation, path=path)
    return _normalize_class(value, annotation, path=path)


def _suggestion(name: str, candidates: Iterable[str]) -> str:
    """Return a spelling suggestion, or an empty string if no name is close."""
361
362
363
364
365
366
367
368
369

    if len(parts) > 1:
        try:
            child = getattr(config, name)
        except AttributeError as exc:
            raise ConfigResolutionError(
                f"CLI.{full_path}", "target argument is not configured"
            ) from exc
        return config.replace(**{name: replace_override_path(child, parts[1:], value, path=full_path)})
408
409
410
411
412
413
414
415
416
417
418
419
420
        if name not in config:
            raise ConfigResolutionError(f"CLI.{path}", f"unknown mapping key {name!r}")
        if len(parts) == 1:
            return {**config, name: value}
        return {**config, name: replace_override_path(config[name], parts[1:], value, path=full_path)}

    location = f"CLI.{path}" if path else "CLI"
    if not is_dataclass(config):
        raise ConfigResolutionError(
            location, f"value of type {type(config).__name__} has no configurable fields"
        )

    config_fields = {field.name: field for field in fields(config)}
430
431
432
433
434
435
436
437
438
439
440
441
442
        )

    full_path = f"{path}.{name}" if path else name
    if len(parts) == 1:
        annotation = get_type_hints(type(config))[name]
        normalized = normalize_value(
            _normalize_override_scalar(value, annotation), annotation, path=f"CLI.{full_path}"
        )
        return replace(config, **{name: normalized})

    child = getattr(config, name)
    if child is None:
        raise ConfigResolutionError(
461
462
463
464
465
466
467
468
        try:
            return resolve_component(node, annotation=member, path=path)
        except ConfigResolutionError as exc:
            last_error = exc
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(node).__name__}",
    ) from last_error
470
471
472
473
474
475
476
477
478
479
480
481
482
483

def _resolve_dataclass(node: object, config_type: type, *, path: str) -> object:
    """Resolve YAML fields and construct a configuration dataclass."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "configuration section must be a YAML mapping")

    config_fields = {field.name: field for field in fields(config_type)}
    unknown = sorted(set(node) - set(config_fields))
    if unknown:
        raise ConfigResolutionError(path, f"unknown configuration fields: {unknown}")

    required = [
        field.name
        for field in config_fields.values()
484
485
486
487
488
489
490
491
492
493
494
495
496
497
        if field.default is MISSING and field.default_factory is MISSING
    ]
    missing = [name for name in required if name not in node]
    if missing:
        raise ConfigResolutionError(path, f"missing required configuration fields: {missing}")

    try:
        hints = get_type_hints(config_type)
    except (NameError, TypeError) as exc:
        raise ConfigResolutionError(path, f"could not resolve configuration type annotations: {exc}") from exc

    resolved = {
        name: resolve_component(
            value,
502
503
504
505
506
507
508
509
510
    }
    try:
        return config_type(**resolved)
    except TypeError as exc:
        raise ConfigResolutionError(path, f"could not construct {config_type.__name__}: {exc}") from exc


def _resolve_target_args(
    raw_args: Mapping[str, object],
520
521
522
523
524
525
526
527
528
    """
    try:
        signature.bind_partial(**raw_args)
    except TypeError as exc:
        raise ConfigResolutionError(path, f"target arguments are invalid: {exc}") from exc

    normalized = {}
    for name, value in raw_args.items():
        parameter = signature.parameters.get(name)
554
555
556
557
558
559
560
561
562
563
564

def _resolve_target(node: object, *, path: str) -> Target[Any]:
    """Resolve a YAML mapping into a ``Target`` without invoking its callable."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "target section must be a YAML mapping")
    if "_target_" not in node:
        raise ConfigResolutionError(path, "target section is missing required _target_")

    target_path = node["_target_"]
    target = import_target(target_path, location=f"{path}._target_")
    try:
563
564
565
566
567
568
569
570
571
    target = import_target(target_path, location=f"{path}._target_")
    try:
        signature = inspect.signature(target)
    except (TypeError, ValueError) as exc:
        raise ConfigResolutionError(path, f"target signature is unavailable: {exc}") from exc

    for parameter in signature.parameters.values():
        if (
            parameter.kind is inspect.Parameter.POSITIONAL_ONLY
570
571
572
573
574
575
576
577
        if (
            parameter.kind is inspect.Parameter.POSITIONAL_ONLY
            and parameter.default is inspect.Signature.empty
        ):
            raise ConfigResolutionError(
                path,
                f"target parameter {parameter.name!r} must be callable by keyword",
            )
592
593
594
595
596
597
598
599
600
601
602
603
604
605

def _resolve_dataloader_config(node: object, *, path: str) -> DataLoaderConfig:
    """Resolve a ``DataLoaderConfig`` with its collator and batch adapter."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "DataLoader configuration must be a YAML mapping")

    target_node = dict(node)
    collate_node = target_node.pop("collate_fn", None)
    get_batch_node = target_node.pop("get_batch", None)
    dataloader_type = normalize_value(
        target_node.pop("dataloader_type", "single"),
        Literal["single", "cyclic"],
        path=f"{path}.dataloader_type",
    )
603
604
605
606
607
608
609
610
611
        Literal["single", "cyclic"],
        path=f"{path}.dataloader_type",
    )
    data_rearrange_map = target_node.pop("data_rearrange_map", None)
    data_sharding = normalize_value(
        target_node.pop("data_sharding", False),
        bool,
        path=f"{path}.data_sharding",
    )
632
633
634
635
636
637
638
639
640

def _resolve_dataset_config(node: object, *, path: str) -> DatasetConfig:
    """Resolve a ``DatasetConfig`` with its model assets and sample transform."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "Dataset configuration must be a YAML mapping")

    target_node = dict(node)
    model_assets_node = target_node.pop("model_assets", {})
    data_transform_node = target_node.pop("data_transform", None)
661
662
663
664
665
666
667
668
669

def _resolve_optimizer_config(node: object, *, path: str) -> OptimizerConfig:
    """Resolve an ``OptimizerConfig`` with its FP32 main-parameter policy."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "Optimizer configuration must be a YAML mapping")

    target_node = dict(node)
    fp32_main_params = normalize_value(
        target_node.pop("fp32_main_params", False),
691
692
693
694
695
696
697
698
699
        ConfigResolutionError: The value or target declaration is invalid for
            the annotation.
    """
    if node is None:
        return _require_none_allowed(annotation, path=path)
    if _is_union(annotation):
        return _resolve_union(node, annotation, path=path)

    origin = get_origin(annotation)
705
706
707
708
709
710
711
712
713
        return _resolve_dataloader_config(node, path=path)
    if annotation is OptimizerConfig:
        return _resolve_optimizer_config(node, path=path)
    if isinstance(annotation, type) and dataclasses.is_dataclass(annotation):
        return _resolve_dataclass(node, annotation, path=path)
    return normalize_value(node, annotation, path=path)


def resolve_config(raw: object) -> TrainerConfig:
723
724
725
726
727
728
729
730
731
732
733
734
735
736
        ConfigResolutionError: The root is not a mapping, required fields are
            missing, or a field or target declaration is invalid.
    """
    if not isinstance(raw, Mapping):
        raise ConfigResolutionError("$", "YAML root must be a mapping")

    root_fields = {field.name: field for field in fields(TrainerConfig)}
    unknown = sorted(set(raw) - set(root_fields))
    if unknown:
        raise ConfigResolutionError("$", f"unknown configuration fields: {unknown}")

    required = [
        field.name
        for field in root_fields.values()
737
738
739
740
741
742
743
744
745
        if field.default is MISSING and field.default_factory is MISSING
    ]
    missing = [name for name in required if name not in raw]
    if missing:
        raise ConfigResolutionError("$", f"missing required configuration fields: {missing}")

    root_hints = get_type_hints(TrainerConfig)
    resolved = {
        name: resolve_component(
751
752
753
754
755
    }
    try:
        return TrainerConfig(**resolved)
    except TypeError as exc:
        raise ConfigResolutionError("$", f"could not construct TrainerConfig: {exc}") from exc