Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / components / losses / model_output.py: 50%
12 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Loss module that reads the loss produced by a model."""
17__all__ = ["ModelOutputLoss"]
19from typing import Any, Dict, Optional, Union
21# AutoModels loss components implement the Transformers/PyTorch Trainer API.
22import torch
24from hyper_parallel.data.constants import IGNORE_INDEX
27class ModelOutputLoss(torch.nn.Module):
28 """Return the loss field from a Transformers-style model output."""
30 def forward( # pylint: disable=unused-argument
31 self,
32 *,
33 model_output: Any,
34 labels: Optional[torch.Tensor],
35 ) -> Union[torch.Tensor, Dict[str, torch.Tensor]]:
36 """Read the model-computed loss.
38 Args:
39 model_output: Model output exposing a ``loss`` attribute.
40 labels: Labels associated with the output. This default loss keeps
41 the argument only to share the trainer-facing call signature
42 with replaceable loss modules.
44 Returns:
45 The loss tensor or named loss mapping from ``model_output.loss``.
46 """
47 local_loss = model_output.loss
48 if labels is None or not isinstance(local_loss, torch.Tensor):
49 return local_loss
51 # Causal LM loss shifts labels by one position. A CP-local slice may
52 # therefore contain no trainable target even when other CP ranks do.
53 has_valid_labels = labels[..., 1:].ne(IGNORE_INDEX).any()
54 local_loss = torch.where(has_valid_labels, local_loss, torch.zeros_like(local_loss))
56 return local_loss