Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / batching / get_batch.py: 91%

209 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Build one model and loss batch from a DataLoader iterator.""" 

16 

17from __future__ import annotations 

18 

19from collections.abc import Mapping 

20from typing import Any 

21 

22import torch 

23 

24from hyper_parallel.data.batching.runtime_input import ( 

25 AttentionRuntime, 

26 IndexedBoundaryResolver, 

27 OnlineBoundaryResolver, 

28 RuntimeInputAdapter, 

29) 

30from hyper_parallel.data.constants import ( 

31 OMNI_CP_TOKEN_FIELDS, 

32 OMNI_INTERNAL_FIELDS, 

33 OMNI_LOSS_INPUT_FIELDS, 

34) 

35from hyper_parallel.data.dataset_logging import get_dataset_logger 

36from hyper_parallel.data.indexed.indexed_data_config import resolve_dataset_type 

37from hyper_parallel.data.indexed.indexed_pretrain_dataset import SFTFromMRDataset 

38from hyper_parallel.data.parallel import ( 

39 CPBatchSharder, 

40 DataLoaderParallelContext, 

41 TPBatchBroadcaster, 

42 create_dataloader_parallel_context, 

43) 

44 

45logger = get_dataset_logger(__name__) 

46 

47 

48class OmniParallelBatch: 

49 """Validate collate-aligned Omni batches, shard over CP, and broadcast over TP.""" 

50 

51 def __init__( 

52 self, 

53 *, 

54 mesh_context: Any, 

55 device: Any, 

56 pp_shared_data: bool = False, 

57 runtime_input_adapter: RuntimeInputAdapter | None = None, 

58 dataloader_context: DataLoaderParallelContext | None = None, 

59 data_config: Mapping[str, Any] | None = None, 

60 ) -> None: 

61 """Create the DataLoader ownership and TP/CP transport context. 

62 

63 Args: 

64 mesh_context: Trainer mesh exposing TP, CP, and PP sizes. 

65 device: Destination model device. 

66 pp_shared_data: Whether pipeline stages share the source batch. 

67 runtime_input_adapter: Provider of additional forward inputs. 

68 data_config: Dataset cache settings used by DataLoader ownership. 

69 dataloader_context: Shared ownership and transport policy from the Trainer. 

70 """ 

71 # if encoder_dp: 

72 # raise NotImplementedError("OmniParallelBatch encoder_dp image-bucket distribution is not implemented") 

73 

74 if runtime_input_adapter is not None and not isinstance(runtime_input_adapter, RuntimeInputAdapter): 

75 raise TypeError("OmniParallelBatch runtime_input_adapter must be a RuntimeInputAdapter") 

76 

77 if int(getattr(mesh_context, "pp_size", 1)) != 1 or pp_shared_data: 

78 raise NotImplementedError("OmniParallelBatch does not support pipeline parallelism or pp_shared_data") 

79 

80 source_config = data_config or {} 

81 self.parallel_context = dataloader_context or create_dataloader_parallel_context( 

82 mesh_context, 

83 data_index_cache=bool(source_config.get("data_index_cache", False)), 

84 shared_storage=not bool(source_config.get("no_shared_storage", False)), 

85 ) 

86 if not self.parallel_context.distributed_enabled and ( 

87 int(getattr(mesh_context, "tp_size", 1)) > 1 or int(getattr(mesh_context, "cp_size", 1)) > 1 

88 ): 

89 raise ValueError("OmniParallelBatch requires an initialized process group for TP or CP") 

90 

91 self.device = device 

92 self.sequence_parallel = bool(mesh_context.sequence_parallel) 

93 self.runtime_input_adapter = runtime_input_adapter 

94 self.cp_sharder = CPBatchSharder(self.parallel_context, token_fields=OMNI_CP_TOKEN_FIELDS) 

95 self.tp_broadcaster = TPBatchBroadcaster(self.parallel_context, device) 

96 self.batch_transport = None 

97 self._batch_index = 0 

98 

99 def __call__( 

100 self, 

101 data_iterator: Any, 

102 *, 

103 external_batch: Mapping[str, Any] | None = None, 

104 ) -> tuple[dict[str, Any], dict[str, Any]]: 

105 """Read on source owners, slice for CP, and distribute only if needed.""" 

106 self._batch_index += 1 

107 local_batch = None 

108 if self.parallel_context.build_on_rank(): 

109 logger.debug("Omni get_batch: batch=%d source_read_begin", self._batch_index, enabled=True) 

110 source_batch = external_batch if external_batch is not None else next(data_iterator) 

111 if not isinstance(source_batch, Mapping): 

112 raise TypeError("Omni DataLoader must yield a mapping batch") 

113 

114 normalized_batch = self._normalize_source_batch(source_batch) 

115 local_batch = self.cp_sharder.shard(normalized_batch) 

116 logger.debug( 

117 "Omni get_batch: batch=%d source_ready global_shape=%s local_shape=%s tp=%d cp=%d cp_rank=%d", 

118 self._batch_index, tuple(normalized_batch["input_ids"].shape), 

119 tuple(local_batch["input_ids"].shape), self.parallel_context.tp_world_size, 

120 self.parallel_context.cp_world_size, self.parallel_context.cp_rank, enabled=True, 

121 ) 

122 

123 transport_inputs = {} 

124 if self.batch_transport is None: 

125 device_batch = self.tp_broadcaster.broadcast(local_batch, broadcast_all_fields=True) 

126 else: 

127 device_batch, transport_inputs = self.batch_transport(local_batch, self.tp_broadcaster) 

128 runtime_inputs = self._build_runtime_inputs(device_batch) 

129 model_inputs, loss_inputs = self._split_model_and_loss_inputs(device_batch) 

130 if set(transport_inputs).intersection((*device_batch, *runtime_inputs)): 

131 raise ValueError("batch transport cannot replace Omni model or runtime inputs") 

132 model_inputs.update(runtime_inputs) 

133 model_inputs.update(transport_inputs) 

134 # Shapes are host metadata; inspecting device tensor values here can stall collectives. 

135 logger.debug( 

136 "Omni get_batch: batch=%d transport_ready fields=%s runtime_fields=%s", 

137 self._batch_index, 

138 {field: tuple(value.shape) for field, value in device_batch.items() if torch.is_tensor(value)}, 

139 list(runtime_inputs), enabled=True, 

140 ) 

141 

142 return model_inputs, loss_inputs 

143 

144 @staticmethod 

145 def _normalize_source_batch(source_batch: Mapping[str, Any]) -> dict[str, Any]: 

146 """Validate required fields and derive the default loss mask.""" 

147 batch = dict(source_batch) 

148 if "input_ids" not in batch: 

149 raise ValueError("Omni batch must contain input_ids") 

150 

151 if "labels" not in batch: 

152 raise ValueError("Omni batch must contain labels") 

153 

154 if "loss_mask" not in batch: 

155 batch["loss_mask"] = batch["labels"] >= 0 

156 if "cu_seq_lens" in batch: 

157 batch["cu_seq_lens"] = batch["cu_seq_lens"].reshape(-1) 

158 return batch 

159 

160 def _build_runtime_inputs(self, batch: Mapping[str, Any]) -> dict[str, Any]: 

161 """Call the configured model hook after TP/CP batch transport.""" 

162 if self.runtime_input_adapter is None: 

163 return {} 

164 

165 runtime_inputs = self.runtime_input_adapter.build( 

166 batch=batch, parallel_context=self.parallel_context, 

167 options={"sequence_parallel": self.sequence_parallel}, 

168 ) 

169 collisions = set(batch).intersection(runtime_inputs) 

170 if collisions: 

171 raise ValueError( 

172 "runtime inputs cannot replace Omni model inputs: " 

173 f"{sorted(collisions)}" 

174 ) 

175 

176 return runtime_inputs 

177 

178 @staticmethod 

179 def _split_model_and_loss_inputs( 

180 batch: Mapping[str, Any], 

181 ) -> tuple[dict[str, Any], dict[str, Any]]: 

182 """Keep processor fields as model inputs and isolate internal metadata.""" 

183 model_inputs = {} 

184 loss_inputs = {} 

185 for field, value in batch.items(): 

186 if field not in OMNI_INTERNAL_FIELDS: 

187 model_inputs[field] = value 

188 if field in OMNI_LOSS_INPUT_FIELDS: 

189 loss_inputs[field] = value 

190 return model_inputs, loss_inputs 

191 

192 

193class TextParallelBatch: 

194 """Define the DataLoader-to-forward-batch processing boundary. 

195 

196 The runtime reads data only on TP rank zero at every CP coordinate, keeps 

197 global sequence boundaries, shards token fields for CP, broadcasts the 

198 local fields across TP, and then builds common attention inputs. 

199 """ 

200 

201 def __init__( 

202 self, 

203 mesh_context: Any, 

204 device: Any, 

205 tokenizer: Any, 

206 data_config: Mapping[str, Any], 

207 pp_shared_data: bool, 

208 *, 

209 source_type: str, 

210 attention_mode: str = "dense", 

211 causal: bool = True, 

212 sliding_window: int | None = None, 

213 reset_position_ids: bool = False, 

214 reset_attention_mask: bool = False, 

215 eod_mask_loss: bool = False, 

216 runtime_input_adapter: RuntimeInputAdapter | None = None, 

217 dataloader_context: DataLoaderParallelContext | None = None, 

218 ) -> None: 

219 """Initialize the batch runtime and its parallel execution context. 

220 

221 Args: 

222 mesh_context: Trainer mesh used to create DataLoader parallel state. 

223 device: Device receiving the DataLoader batch. 

224 tokenizer: Tokenizer providing EOD and padding token semantics. 

225 data_config: Dataset options used to build runtime LTR fields. 

226 pp_shared_data: Whether pipeline stages share the prepared batch. 

227 source_type: ``online``, ``indexed``, or ``packed_text`` batch contract. 

228 Packed text provides input_ids, shifted labels, and explicit cu_seq_lens. 

229 attention_mode: ``dense`` or ``compressed`` attention representation. 

230 causal: Whether attention uses left-to-right causal semantics. 

231 sliding_window: Sliding-window size, or ``None`` for full attention. 

232 reset_position_ids: Whether positions restart at sequence boundaries. 

233 reset_attention_mask: Whether EOD starts an independent attention sequence. 

234 eod_mask_loss: Whether EOD tokens are excluded from the loss. 

235 runtime_input_adapter: Optional model-owned forward-input extension. 

236 dataloader_context: Shared ownership and transport policy from the Trainer. 

237 """ 

238 # pylint: disable=too-many-locals 

239 

240 self.parallel_context = dataloader_context or create_dataloader_parallel_context( 

241 mesh_context, 

242 data_index_cache=bool(data_config.get("data_index_cache", False)), 

243 shared_storage=not bool(data_config.get("no_shared_storage", False)), 

244 ) 

245 self.device = device 

246 self.tokenizer = tokenizer 

247 self.data_config = dict(data_config) 

248 self.source_type = source_type 

249 dataset_spec = data_config.get("low_level_dataset") 

250 self.is_sft_dataset = (source_type == "indexed" and dataset_spec is not None 

251 and issubclass(resolve_dataset_type(dataset_spec), SFTFromMRDataset)) 

252 self.causal = causal 

253 self.sliding_window = sliding_window 

254 if runtime_input_adapter is not None and not isinstance( 

255 runtime_input_adapter, RuntimeInputAdapter 

256 ): 

257 raise TypeError("TextParallelBatch runtime_input_adapter must be a RuntimeInputAdapter") 

258 

259 self.runtime_input_adapter = runtime_input_adapter 

260 self._batch_flow_logged = False 

261 

262 self.boundary_resolver = self._build_boundary_resolver(source_type, tokenizer) 

263 

264 self.pp_shared_data = pp_shared_data 

265 self.labels_are_shifted = bool(self.data_config.get("labels_are_shifted", True)) 

266 create_attention_mask = bool( 

267 self.data_config.get("create_attention_mask_in_dataloader", attention_mode == "dense") 

268 ) 

269 self.eod_mask_loss = eod_mask_loss or bool(self.data_config.get("eod_mask_loss", False)) 

270 self._validate_attention_inputs(attention_mode, create_attention_mask) 

271 self.cp_sharder = CPBatchSharder(self.parallel_context) 

272 self.tp_broadcaster = TPBatchBroadcaster(self.parallel_context, device) 

273 

274 self.reset_position_ids = ( 

275 reset_position_ids or bool(self.data_config.get("reset_position_ids", False)) 

276 or source_type == "packed_text" 

277 ) 

278 resolved_reset_attention_mask = ( 

279 reset_attention_mask 

280 or bool(self.data_config.get("reset_attention_mask", False)) 

281 or source_type in {"online", "packed_text"} 

282 or self.is_sft_dataset 

283 ) 

284 self.attention_runtime = AttentionRuntime( 

285 mode=attention_mode, 

286 create_mask=create_attention_mask, 

287 reset_mask=resolved_reset_attention_mask, 

288 sliding_window=sliding_window, 

289 ) 

290 

291 @staticmethod 

292 def _build_boundary_resolver(source_type: str, tokenizer: Any) -> OnlineBoundaryResolver | IndexedBoundaryResolver: 

293 """Select sequence-boundary handling for the source's input_ids batch.""" 

294 if source_type in {"online", "packed_text"}: 

295 return OnlineBoundaryResolver() 

296 if source_type == "indexed": 

297 return IndexedBoundaryResolver(getattr(tokenizer, "eod", None)) 

298 raise ValueError(f"Unsupported batch source type: {source_type!r}") 

299 

300 def _validate_attention_inputs(self, attention_mode: str, create_attention_mask: bool) -> None: 

301 """Require shifted labels and document isolation for prepared text records.""" 

302 if self.source_type == "packed_text": 

303 if not self.labels_are_shifted: 

304 raise ValueError("Packed text batches require shifted labels; labels_are_shifted must be True") 

305 if not create_attention_mask or (attention_mode == "compressed" and self.runtime_input_adapter is None): 

306 raise ValueError( 

307 "Packed text batches require document-boundary attention: enable " 

308 "create_attention_mask_in_dataloader and supply a runtime_input_adapter for compressed attention" 

309 ) 

310 if self.is_sft_dataset: 

311 if not self.labels_are_shifted or self.eod_mask_loss or not self.causal: 

312 raise ValueError("Prepared records require shifted labels, causal attention and eod_mask_loss=False") 

313 if (attention_mode == "compressed" or not create_attention_mask) and self.runtime_input_adapter is None: 

314 raise ValueError("Prepared records without a dense mask require a boundary-aware runtime adapter") 

315 

316 def __call__( 

317 self, 

318 data_iterator: Any, 

319 ) -> tuple[Mapping[str, Any], Mapping[str, Any]]: 

320 """Build model and loss inputs for one forward-backward step. 

321 

322 Args: 

323 data_iterator: Iterator yielding one collated Indexed or Online batch. 

324 

325 Returns: 

326 Model inputs and loss inputs for one forward-backward step. 

327 """ 

328 canonical_batch = self._read_source_batch(data_iterator) 

329 cu_seq_lens = self._resolve_sequence_boundaries(canonical_batch) 

330 cp_local_batch = self.cp_sharder.shard(canonical_batch) 

331 parallel_batch = self.tp_broadcaster.broadcast( 

332 cp_local_batch, 

333 cu_seq_lens, 

334 ) 

335 self._log_batch_flow(canonical_batch, parallel_batch) 

336 

337 position_ids = self._build_local_position_ids( 

338 parallel_batch["input_ids"], 

339 parallel_batch["cu_seq_lens"], 

340 ) 

341 parallel_batch["position_ids"] = position_ids 

342 

343 loss_mask = self._build_loss_mask(parallel_batch) 

344 parallel_batch["loss_mask"] = loss_mask 

345 runtime_inputs = self._build_runtime_inputs(parallel_batch) 

346 

347 model_inputs, loss_inputs = self._split_model_and_loss_inputs( 

348 parallel_batch, 

349 runtime_inputs, 

350 ) 

351 

352 return model_inputs, loss_inputs 

353 

354 def _build_runtime_inputs( 

355 self, 

356 parallel_batch: Mapping[str, Any], 

357 ) -> dict[str, Any]: 

358 """Build common attention inputs plus a model-owned extension.""" 

359 runtime_inputs = self.attention_runtime.build( 

360 batch=parallel_batch, 

361 parallel_context=self.parallel_context, 

362 ) 

363 if self.runtime_input_adapter is None: 

364 return runtime_inputs 

365 

366 model_runtime_inputs = self.runtime_input_adapter.build( 

367 batch=parallel_batch, 

368 parallel_context=self.parallel_context, 

369 options={ 

370 "source_type": self.source_type, 

371 "attention_mode": self.attention_runtime.mode, 

372 "causal": self.causal, 

373 "sliding_window": self.sliding_window, 

374 }, 

375 ) 

376 collisions = set(runtime_inputs).intersection(model_runtime_inputs) 

377 if collisions: 

378 raise ValueError( 

379 "model runtime inputs cannot replace common attention inputs: " 

380 f"{sorted(collisions)}" 

381 ) 

382 

383 runtime_inputs.update(model_runtime_inputs) 

384 return runtime_inputs 

385 

386 def _log_batch_flow( 

387 self, 

388 canonical_batch: Mapping[str, Any] | None, 

389 parallel_batch: Mapping[str, Any], 

390 ) -> None: 

391 """Log the resolved source and parallel batch shapes once.""" 

392 if not self._batch_flow_logged: 

393 source_shape = None if canonical_batch is None else tuple(canonical_batch["input_ids"].shape) 

394 local_shape = tuple(parallel_batch["input_ids"].shape) 

395 num_boundaries = 0 if parallel_batch["cu_seq_lens"] is None else parallel_batch["cu_seq_lens"].numel() 

396 logger.debug( 

397 "Parallel batch flow: source=%s, tp_rank=%d/%d, cp_rank=%d/%d, " 

398 "source_owner=%s, source_shape=%s, local_shape=%s, global_boundaries=%d", 

399 self.source_type, 

400 self.parallel_context.tp_rank, 

401 self.parallel_context.tp_world_size, 

402 self.parallel_context.cp_rank, 

403 self.parallel_context.cp_world_size, 

404 canonical_batch is not None, 

405 source_shape, 

406 local_shape, 

407 num_boundaries, 

408 enabled=True, 

409 ) 

410 self._batch_flow_logged = True 

411 

412 def _read_source_batch(self, data_iterator: Any) -> Mapping[str, Any] | None: 

413 """Advance the source only on ranks selected by the shared context.""" 

414 if self.parallel_context.build_on_rank(): 

415 source_batch = next(data_iterator) 

416 else: 

417 source_batch = None 

418 

419 return source_batch 

420 

421 def _resolve_sequence_boundaries( 

422 self, 

423 canonical_batch: Mapping[str, Any] | None, 

424 ) -> Any: 

425 """Resolve global leading-zero cumulative sequence boundaries.""" 

426 if canonical_batch is None: 

427 cu_seq_lens = None 

428 else: 

429 cu_seq_lens = self.boundary_resolver.resolve(canonical_batch) 

430 

431 return cu_seq_lens 

432 

433 def _build_local_position_ids( 

434 self, 

435 input_ids: Any, 

436 cu_seq_lens: Any, 

437 ) -> Any: 

438 """Build position IDs for the current CP slice on every TP rank. 

439 

440 Packed boundaries define the position semantics. CP only determines 

441 which global sequence interval is materialized by this rank. 

442 """ 

443 # pylint: disable=too-many-locals 

444 batch_size, local_seq_len = input_ids.shape 

445 cp_size = self.parallel_context.cp_world_size 

446 global_seq_len = local_seq_len * cp_size 

447 cp_rank = self.parallel_context.cp_rank 

448 cp_seq_start = cp_rank * local_seq_len 

449 cp_seq_end = cp_seq_start + local_seq_len 

450 

451 # This produces the same values as building global [B, S] positions 

452 # and then taking the CP slice, without allocating the global tensor. 

453 position_ids = torch.arange( 

454 cp_seq_start, cp_seq_end, dtype=torch.int64, device=input_ids.device 

455 ) 

456 local_position_ids = position_ids.unsqueeze(0).expand(batch_size, -1) 

457 

458 if self.reset_position_ids: 

459 local_position_ids = local_position_ids.clone() 

460 boundaries = cu_seq_lens.tolist() 

461 for seq_start, seq_end in zip(boundaries[:-1], boundaries[1:]): 

462 seq_len = seq_end - seq_start 

463 batch_idx, seq_start_in_batch = divmod(seq_start, global_seq_len) 

464 seq_end_in_batch = seq_start_in_batch + seq_len 

465 # Intersect the sequence with the current CP slice. 

466 slice_start = max(seq_start_in_batch, cp_seq_start) 

467 slice_end = min(seq_end_in_batch, cp_seq_end) 

468 if slice_start >= slice_end: 

469 continue 

470 

471 local_start = slice_start - cp_seq_start 

472 local_end = slice_end - cp_seq_start 

473 local_position_ids[batch_idx, local_start:local_end] -= seq_start_in_batch 

474 

475 local_position_ids = local_position_ids.contiguous() 

476 

477 return local_position_ids 

478 

479 def _build_loss_mask(self, parallel_batch: Mapping[str, Any]) -> Any: 

480 """Build the local loss mask from labels and input IDs.""" 

481 loss_mask = (parallel_batch["labels"] >= 0).to(dtype=torch.int64) 

482 

483 if self.eod_mask_loss: 

484 eod_token_id = getattr(self.tokenizer, "eod", None) 

485 loss_mask = loss_mask.masked_fill(parallel_batch["input_ids"] == eod_token_id, 0) 

486 

487 return loss_mask 

488 

489 def _split_model_and_loss_inputs( 

490 self, 

491 parallel_batch: Mapping[str, Any], 

492 runtime_inputs: Mapping[str, Any] | None = None, 

493 ) -> tuple[Mapping[str, Any], Mapping[str, Any]]: 

494 """Split forward fields from loss and token-accounting fields.""" 

495 model_inputs = { 

496 "input_ids": parallel_batch["input_ids"], 

497 "labels": parallel_batch["labels"], 

498 "position_ids": parallel_batch["position_ids"], 

499 } 

500 if self.labels_are_shifted: 

501 model_inputs["shift_labels"] = parallel_batch["labels"] 

502 

503 runtime_inputs = runtime_inputs or {} 

504 collisions = set(model_inputs).intersection(runtime_inputs) 

505 if collisions: 

506 raise ValueError( 

507 "runtime inputs cannot replace framework-owned model inputs: " 

508 f"{sorted(collisions)}" 

509 ) 

510 

511 model_inputs.update(runtime_inputs) 

512 

513 loss_inputs = { 

514 "labels": parallel_batch["labels"], 

515 "loss_mask": parallel_batch["loss_mask"], 

516 } 

517 if self.labels_are_shifted: 

518 loss_inputs["shift_labels"] = parallel_batch["labels"] 

519 

520 return model_inputs, loss_inputs