Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / batching / get_batch.py: 91%
209 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Build one model and loss batch from a DataLoader iterator."""
17from __future__ import annotations
19from collections.abc import Mapping
20from typing import Any
22import torch
24from hyper_parallel.data.batching.runtime_input import (
25 AttentionRuntime,
26 IndexedBoundaryResolver,
27 OnlineBoundaryResolver,
28 RuntimeInputAdapter,
29)
30from hyper_parallel.data.constants import (
31 OMNI_CP_TOKEN_FIELDS,
32 OMNI_INTERNAL_FIELDS,
33 OMNI_LOSS_INPUT_FIELDS,
34)
35from hyper_parallel.data.dataset_logging import get_dataset_logger
36from hyper_parallel.data.indexed.indexed_data_config import resolve_dataset_type
37from hyper_parallel.data.indexed.indexed_pretrain_dataset import SFTFromMRDataset
38from hyper_parallel.data.parallel import (
39 CPBatchSharder,
40 DataLoaderParallelContext,
41 TPBatchBroadcaster,
42 create_dataloader_parallel_context,
43)
45logger = get_dataset_logger(__name__)
48class OmniParallelBatch:
49 """Validate collate-aligned Omni batches, shard over CP, and broadcast over TP."""
51 def __init__(
52 self,
53 *,
54 mesh_context: Any,
55 device: Any,
56 pp_shared_data: bool = False,
57 runtime_input_adapter: RuntimeInputAdapter | None = None,
58 dataloader_context: DataLoaderParallelContext | None = None,
59 data_config: Mapping[str, Any] | None = None,
60 ) -> None:
61 """Create the DataLoader ownership and TP/CP transport context.
63 Args:
64 mesh_context: Trainer mesh exposing TP, CP, and PP sizes.
65 device: Destination model device.
66 pp_shared_data: Whether pipeline stages share the source batch.
67 runtime_input_adapter: Provider of additional forward inputs.
68 data_config: Dataset cache settings used by DataLoader ownership.
69 dataloader_context: Shared ownership and transport policy from the Trainer.
70 """
71 # if encoder_dp:
72 # raise NotImplementedError("OmniParallelBatch encoder_dp image-bucket distribution is not implemented")
74 if runtime_input_adapter is not None and not isinstance(runtime_input_adapter, RuntimeInputAdapter):
75 raise TypeError("OmniParallelBatch runtime_input_adapter must be a RuntimeInputAdapter")
77 if int(getattr(mesh_context, "pp_size", 1)) != 1 or pp_shared_data:
78 raise NotImplementedError("OmniParallelBatch does not support pipeline parallelism or pp_shared_data")
80 source_config = data_config or {}
81 self.parallel_context = dataloader_context or create_dataloader_parallel_context(
82 mesh_context,
83 data_index_cache=bool(source_config.get("data_index_cache", False)),
84 shared_storage=not bool(source_config.get("no_shared_storage", False)),
85 )
86 if not self.parallel_context.distributed_enabled and (
87 int(getattr(mesh_context, "tp_size", 1)) > 1 or int(getattr(mesh_context, "cp_size", 1)) > 1
88 ):
89 raise ValueError("OmniParallelBatch requires an initialized process group for TP or CP")
91 self.device = device
92 self.sequence_parallel = bool(mesh_context.sequence_parallel)
93 self.runtime_input_adapter = runtime_input_adapter
94 self.cp_sharder = CPBatchSharder(self.parallel_context, token_fields=OMNI_CP_TOKEN_FIELDS)
95 self.tp_broadcaster = TPBatchBroadcaster(self.parallel_context, device)
96 self.batch_transport = None
97 self._batch_index = 0
99 def __call__(
100 self,
101 data_iterator: Any,
102 *,
103 external_batch: Mapping[str, Any] | None = None,
104 ) -> tuple[dict[str, Any], dict[str, Any]]:
105 """Read on source owners, slice for CP, and distribute only if needed."""
106 self._batch_index += 1
107 local_batch = None
108 if self.parallel_context.build_on_rank():
109 logger.debug("Omni get_batch: batch=%d source_read_begin", self._batch_index, enabled=True)
110 source_batch = external_batch if external_batch is not None else next(data_iterator)
111 if not isinstance(source_batch, Mapping):
112 raise TypeError("Omni DataLoader must yield a mapping batch")
114 normalized_batch = self._normalize_source_batch(source_batch)
115 local_batch = self.cp_sharder.shard(normalized_batch)
116 logger.debug(
117 "Omni get_batch: batch=%d source_ready global_shape=%s local_shape=%s tp=%d cp=%d cp_rank=%d",
118 self._batch_index, tuple(normalized_batch["input_ids"].shape),
119 tuple(local_batch["input_ids"].shape), self.parallel_context.tp_world_size,
120 self.parallel_context.cp_world_size, self.parallel_context.cp_rank, enabled=True,
121 )
123 transport_inputs = {}
124 if self.batch_transport is None:
125 device_batch = self.tp_broadcaster.broadcast(local_batch, broadcast_all_fields=True)
126 else:
127 device_batch, transport_inputs = self.batch_transport(local_batch, self.tp_broadcaster)
128 runtime_inputs = self._build_runtime_inputs(device_batch)
129 model_inputs, loss_inputs = self._split_model_and_loss_inputs(device_batch)
130 if set(transport_inputs).intersection((*device_batch, *runtime_inputs)):
131 raise ValueError("batch transport cannot replace Omni model or runtime inputs")
132 model_inputs.update(runtime_inputs)
133 model_inputs.update(transport_inputs)
134 # Shapes are host metadata; inspecting device tensor values here can stall collectives.
135 logger.debug(
136 "Omni get_batch: batch=%d transport_ready fields=%s runtime_fields=%s",
137 self._batch_index,
138 {field: tuple(value.shape) for field, value in device_batch.items() if torch.is_tensor(value)},
139 list(runtime_inputs), enabled=True,
140 )
142 return model_inputs, loss_inputs
144 @staticmethod
145 def _normalize_source_batch(source_batch: Mapping[str, Any]) -> dict[str, Any]:
146 """Validate required fields and derive the default loss mask."""
147 batch = dict(source_batch)
148 if "input_ids" not in batch:
149 raise ValueError("Omni batch must contain input_ids")
151 if "labels" not in batch:
152 raise ValueError("Omni batch must contain labels")
154 if "loss_mask" not in batch:
155 batch["loss_mask"] = batch["labels"] >= 0
156 if "cu_seq_lens" in batch:
157 batch["cu_seq_lens"] = batch["cu_seq_lens"].reshape(-1)
158 return batch
160 def _build_runtime_inputs(self, batch: Mapping[str, Any]) -> dict[str, Any]:
161 """Call the configured model hook after TP/CP batch transport."""
162 if self.runtime_input_adapter is None:
163 return {}
165 runtime_inputs = self.runtime_input_adapter.build(
166 batch=batch, parallel_context=self.parallel_context,
167 options={"sequence_parallel": self.sequence_parallel},
168 )
169 collisions = set(batch).intersection(runtime_inputs)
170 if collisions:
171 raise ValueError(
172 "runtime inputs cannot replace Omni model inputs: "
173 f"{sorted(collisions)}"
174 )
176 return runtime_inputs
178 @staticmethod
179 def _split_model_and_loss_inputs(
180 batch: Mapping[str, Any],
181 ) -> tuple[dict[str, Any], dict[str, Any]]:
182 """Keep processor fields as model inputs and isolate internal metadata."""
183 model_inputs = {}
184 loss_inputs = {}
185 for field, value in batch.items():
186 if field not in OMNI_INTERNAL_FIELDS:
187 model_inputs[field] = value
188 if field in OMNI_LOSS_INPUT_FIELDS:
189 loss_inputs[field] = value
190 return model_inputs, loss_inputs
193class TextParallelBatch:
194 """Define the DataLoader-to-forward-batch processing boundary.
196 The runtime reads data only on TP rank zero at every CP coordinate, keeps
197 global sequence boundaries, shards token fields for CP, broadcasts the
198 local fields across TP, and then builds common attention inputs.
199 """
201 def __init__(
202 self,
203 mesh_context: Any,
204 device: Any,
205 tokenizer: Any,
206 data_config: Mapping[str, Any],
207 pp_shared_data: bool,
208 *,
209 source_type: str,
210 attention_mode: str = "dense",
211 causal: bool = True,
212 sliding_window: int | None = None,
213 reset_position_ids: bool = False,
214 reset_attention_mask: bool = False,
215 eod_mask_loss: bool = False,
216 runtime_input_adapter: RuntimeInputAdapter | None = None,
217 dataloader_context: DataLoaderParallelContext | None = None,
218 ) -> None:
219 """Initialize the batch runtime and its parallel execution context.
221 Args:
222 mesh_context: Trainer mesh used to create DataLoader parallel state.
223 device: Device receiving the DataLoader batch.
224 tokenizer: Tokenizer providing EOD and padding token semantics.
225 data_config: Dataset options used to build runtime LTR fields.
226 pp_shared_data: Whether pipeline stages share the prepared batch.
227 source_type: ``online``, ``indexed``, or ``packed_text`` batch contract.
228 Packed text provides input_ids, shifted labels, and explicit cu_seq_lens.
229 attention_mode: ``dense`` or ``compressed`` attention representation.
230 causal: Whether attention uses left-to-right causal semantics.
231 sliding_window: Sliding-window size, or ``None`` for full attention.
232 reset_position_ids: Whether positions restart at sequence boundaries.
233 reset_attention_mask: Whether EOD starts an independent attention sequence.
234 eod_mask_loss: Whether EOD tokens are excluded from the loss.
235 runtime_input_adapter: Optional model-owned forward-input extension.
236 dataloader_context: Shared ownership and transport policy from the Trainer.
237 """
238 # pylint: disable=too-many-locals
240 self.parallel_context = dataloader_context or create_dataloader_parallel_context(
241 mesh_context,
242 data_index_cache=bool(data_config.get("data_index_cache", False)),
243 shared_storage=not bool(data_config.get("no_shared_storage", False)),
244 )
245 self.device = device
246 self.tokenizer = tokenizer
247 self.data_config = dict(data_config)
248 self.source_type = source_type
249 dataset_spec = data_config.get("low_level_dataset")
250 self.is_sft_dataset = (source_type == "indexed" and dataset_spec is not None
251 and issubclass(resolve_dataset_type(dataset_spec), SFTFromMRDataset))
252 self.causal = causal
253 self.sliding_window = sliding_window
254 if runtime_input_adapter is not None and not isinstance(
255 runtime_input_adapter, RuntimeInputAdapter
256 ):
257 raise TypeError("TextParallelBatch runtime_input_adapter must be a RuntimeInputAdapter")
259 self.runtime_input_adapter = runtime_input_adapter
260 self._batch_flow_logged = False
262 self.boundary_resolver = self._build_boundary_resolver(source_type, tokenizer)
264 self.pp_shared_data = pp_shared_data
265 self.labels_are_shifted = bool(self.data_config.get("labels_are_shifted", True))
266 create_attention_mask = bool(
267 self.data_config.get("create_attention_mask_in_dataloader", attention_mode == "dense")
268 )
269 self.eod_mask_loss = eod_mask_loss or bool(self.data_config.get("eod_mask_loss", False))
270 self._validate_attention_inputs(attention_mode, create_attention_mask)
271 self.cp_sharder = CPBatchSharder(self.parallel_context)
272 self.tp_broadcaster = TPBatchBroadcaster(self.parallel_context, device)
274 self.reset_position_ids = (
275 reset_position_ids or bool(self.data_config.get("reset_position_ids", False))
276 or source_type == "packed_text"
277 )
278 resolved_reset_attention_mask = (
279 reset_attention_mask
280 or bool(self.data_config.get("reset_attention_mask", False))
281 or source_type in {"online", "packed_text"}
282 or self.is_sft_dataset
283 )
284 self.attention_runtime = AttentionRuntime(
285 mode=attention_mode,
286 create_mask=create_attention_mask,
287 reset_mask=resolved_reset_attention_mask,
288 sliding_window=sliding_window,
289 )
291 @staticmethod
292 def _build_boundary_resolver(source_type: str, tokenizer: Any) -> OnlineBoundaryResolver | IndexedBoundaryResolver:
293 """Select sequence-boundary handling for the source's input_ids batch."""
294 if source_type in {"online", "packed_text"}:
295 return OnlineBoundaryResolver()
296 if source_type == "indexed":
297 return IndexedBoundaryResolver(getattr(tokenizer, "eod", None))
298 raise ValueError(f"Unsupported batch source type: {source_type!r}")
300 def _validate_attention_inputs(self, attention_mode: str, create_attention_mask: bool) -> None:
301 """Require shifted labels and document isolation for prepared text records."""
302 if self.source_type == "packed_text":
303 if not self.labels_are_shifted:
304 raise ValueError("Packed text batches require shifted labels; labels_are_shifted must be True")
305 if not create_attention_mask or (attention_mode == "compressed" and self.runtime_input_adapter is None):
306 raise ValueError(
307 "Packed text batches require document-boundary attention: enable "
308 "create_attention_mask_in_dataloader and supply a runtime_input_adapter for compressed attention"
309 )
310 if self.is_sft_dataset:
311 if not self.labels_are_shifted or self.eod_mask_loss or not self.causal:
312 raise ValueError("Prepared records require shifted labels, causal attention and eod_mask_loss=False")
313 if (attention_mode == "compressed" or not create_attention_mask) and self.runtime_input_adapter is None:
314 raise ValueError("Prepared records without a dense mask require a boundary-aware runtime adapter")
316 def __call__(
317 self,
318 data_iterator: Any,
319 ) -> tuple[Mapping[str, Any], Mapping[str, Any]]:
320 """Build model and loss inputs for one forward-backward step.
322 Args:
323 data_iterator: Iterator yielding one collated Indexed or Online batch.
325 Returns:
326 Model inputs and loss inputs for one forward-backward step.
327 """
328 canonical_batch = self._read_source_batch(data_iterator)
329 cu_seq_lens = self._resolve_sequence_boundaries(canonical_batch)
330 cp_local_batch = self.cp_sharder.shard(canonical_batch)
331 parallel_batch = self.tp_broadcaster.broadcast(
332 cp_local_batch,
333 cu_seq_lens,
334 )
335 self._log_batch_flow(canonical_batch, parallel_batch)
337 position_ids = self._build_local_position_ids(
338 parallel_batch["input_ids"],
339 parallel_batch["cu_seq_lens"],
340 )
341 parallel_batch["position_ids"] = position_ids
343 loss_mask = self._build_loss_mask(parallel_batch)
344 parallel_batch["loss_mask"] = loss_mask
345 runtime_inputs = self._build_runtime_inputs(parallel_batch)
347 model_inputs, loss_inputs = self._split_model_and_loss_inputs(
348 parallel_batch,
349 runtime_inputs,
350 )
352 return model_inputs, loss_inputs
354 def _build_runtime_inputs(
355 self,
356 parallel_batch: Mapping[str, Any],
357 ) -> dict[str, Any]:
358 """Build common attention inputs plus a model-owned extension."""
359 runtime_inputs = self.attention_runtime.build(
360 batch=parallel_batch,
361 parallel_context=self.parallel_context,
362 )
363 if self.runtime_input_adapter is None:
364 return runtime_inputs
366 model_runtime_inputs = self.runtime_input_adapter.build(
367 batch=parallel_batch,
368 parallel_context=self.parallel_context,
369 options={
370 "source_type": self.source_type,
371 "attention_mode": self.attention_runtime.mode,
372 "causal": self.causal,
373 "sliding_window": self.sliding_window,
374 },
375 )
376 collisions = set(runtime_inputs).intersection(model_runtime_inputs)
377 if collisions:
378 raise ValueError(
379 "model runtime inputs cannot replace common attention inputs: "
380 f"{sorted(collisions)}"
381 )
383 runtime_inputs.update(model_runtime_inputs)
384 return runtime_inputs
386 def _log_batch_flow(
387 self,
388 canonical_batch: Mapping[str, Any] | None,
389 parallel_batch: Mapping[str, Any],
390 ) -> None:
391 """Log the resolved source and parallel batch shapes once."""
392 if not self._batch_flow_logged:
393 source_shape = None if canonical_batch is None else tuple(canonical_batch["input_ids"].shape)
394 local_shape = tuple(parallel_batch["input_ids"].shape)
395 num_boundaries = 0 if parallel_batch["cu_seq_lens"] is None else parallel_batch["cu_seq_lens"].numel()
396 logger.debug(
397 "Parallel batch flow: source=%s, tp_rank=%d/%d, cp_rank=%d/%d, "
398 "source_owner=%s, source_shape=%s, local_shape=%s, global_boundaries=%d",
399 self.source_type,
400 self.parallel_context.tp_rank,
401 self.parallel_context.tp_world_size,
402 self.parallel_context.cp_rank,
403 self.parallel_context.cp_world_size,
404 canonical_batch is not None,
405 source_shape,
406 local_shape,
407 num_boundaries,
408 enabled=True,
409 )
410 self._batch_flow_logged = True
412 def _read_source_batch(self, data_iterator: Any) -> Mapping[str, Any] | None:
413 """Advance the source only on ranks selected by the shared context."""
414 if self.parallel_context.build_on_rank():
415 source_batch = next(data_iterator)
416 else:
417 source_batch = None
419 return source_batch
421 def _resolve_sequence_boundaries(
422 self,
423 canonical_batch: Mapping[str, Any] | None,
424 ) -> Any:
425 """Resolve global leading-zero cumulative sequence boundaries."""
426 if canonical_batch is None:
427 cu_seq_lens = None
428 else:
429 cu_seq_lens = self.boundary_resolver.resolve(canonical_batch)
431 return cu_seq_lens
433 def _build_local_position_ids(
434 self,
435 input_ids: Any,
436 cu_seq_lens: Any,
437 ) -> Any:
438 """Build position IDs for the current CP slice on every TP rank.
440 Packed boundaries define the position semantics. CP only determines
441 which global sequence interval is materialized by this rank.
442 """
443 # pylint: disable=too-many-locals
444 batch_size, local_seq_len = input_ids.shape
445 cp_size = self.parallel_context.cp_world_size
446 global_seq_len = local_seq_len * cp_size
447 cp_rank = self.parallel_context.cp_rank
448 cp_seq_start = cp_rank * local_seq_len
449 cp_seq_end = cp_seq_start + local_seq_len
451 # This produces the same values as building global [B, S] positions
452 # and then taking the CP slice, without allocating the global tensor.
453 position_ids = torch.arange(
454 cp_seq_start, cp_seq_end, dtype=torch.int64, device=input_ids.device
455 )
456 local_position_ids = position_ids.unsqueeze(0).expand(batch_size, -1)
458 if self.reset_position_ids:
459 local_position_ids = local_position_ids.clone()
460 boundaries = cu_seq_lens.tolist()
461 for seq_start, seq_end in zip(boundaries[:-1], boundaries[1:]):
462 seq_len = seq_end - seq_start
463 batch_idx, seq_start_in_batch = divmod(seq_start, global_seq_len)
464 seq_end_in_batch = seq_start_in_batch + seq_len
465 # Intersect the sequence with the current CP slice.
466 slice_start = max(seq_start_in_batch, cp_seq_start)
467 slice_end = min(seq_end_in_batch, cp_seq_end)
468 if slice_start >= slice_end:
469 continue
471 local_start = slice_start - cp_seq_start
472 local_end = slice_end - cp_seq_start
473 local_position_ids[batch_idx, local_start:local_end] -= seq_start_in_batch
475 local_position_ids = local_position_ids.contiguous()
477 return local_position_ids
479 def _build_loss_mask(self, parallel_batch: Mapping[str, Any]) -> Any:
480 """Build the local loss mask from labels and input IDs."""
481 loss_mask = (parallel_batch["labels"] >= 0).to(dtype=torch.int64)
483 if self.eod_mask_loss:
484 eod_token_id = getattr(self.tokenizer, "eod", None)
485 loss_mask = loss_mask.masked_fill(parallel_batch["input_ids"] == eod_token_id, 0)
487 return loss_mask
489 def _split_model_and_loss_inputs(
490 self,
491 parallel_batch: Mapping[str, Any],
492 runtime_inputs: Mapping[str, Any] | None = None,
493 ) -> tuple[Mapping[str, Any], Mapping[str, Any]]:
494 """Split forward fields from loss and token-accounting fields."""
495 model_inputs = {
496 "input_ids": parallel_batch["input_ids"],
497 "labels": parallel_batch["labels"],
498 "position_ids": parallel_batch["position_ids"],
499 }
500 if self.labels_are_shifted:
501 model_inputs["shift_labels"] = parallel_batch["labels"]
503 runtime_inputs = runtime_inputs or {}
504 collisions = set(model_inputs).intersection(runtime_inputs)
505 if collisions:
506 raise ValueError(
507 "runtime inputs cannot replace framework-owned model inputs: "
508 f"{sorted(collisions)}"
509 )
511 model_inputs.update(runtime_inputs)
513 loss_inputs = {
514 "labels": parallel_batch["labels"],
515 "loss_mask": parallel_batch["loss_mask"],
516 }
517 if self.labels_are_shifted:
518 loss_inputs["shift_labels"] = parallel_batch["labels"]
520 return model_inputs, loss_inputs