Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed / encoder_parallel / transport.py: 96%

83 statements  

« prev     ^ index     » next       coverage.py v7.13.1, created at 2026-11-19 05:29 +0800

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""Transport CPU image shards without staging complete images on a device.""" 

16 

17from __future__ import annotations 

18 

19from typing import Any, Mapping 

20 

21import torch 

22import torch.distributed as dist 

23 

24from hyper_parallel.data.parallel import TPBatchBroadcaster 

25from hyper_parallel.distributed.encoder_parallel.contracts import EncoderLocalInputs 

26from hyper_parallel.distributed.encoder_parallel.planning import DecoderSequenceLayout, build_segments 

27from hyper_parallel.distributed.encoder_parallel.runtime import EncoderParallelPlan, EncoderParallelRuntime 

28 

29 

30_METADATA_FIELD = "_encoder_transport_metadata" 

31_FORWARD_FIELDS = ("encoder_parallel_plan", "encoder_parallel_inputs") 

32_CHUNK_BYTES = 4 * 1024 * 1024 

33 

34 

35class EncoderParallelBatchTransport: 

36 """Plan from small metadata and send only each destination's CPU patches. 

37 

38 Every decoder CP coordinate already has a complete CPU batch on TP0. TP 

39 peers may belong to different encoder DP buckets or CP shards, so their 

40 payloads are transported individually with bounded device staging. 

41 """ 

42 

43 def __init__(self, *, runtime: EncoderParallelRuntime, chunk_bytes: int = _CHUNK_BYTES) -> None: 

44 """Set the runtime and maximum device staging size in bytes.""" 

45 if isinstance(chunk_bytes, bool) or not isinstance(chunk_bytes, int) or chunk_bytes < 1: 

46 raise ValueError("encoder transport chunk_bytes must be a positive integer") 

47 self.runtime = runtime 

48 self.chunk_bytes = chunk_bytes 

49 

50 def _prepare_source(self, batch: Mapping[str, Any]) -> dict[str, Any]: 

51 """Validate CPU images and keep tensor payloads out of TP broadcasting.""" 

52 bridge = self.runtime.bridge 

53 prepared = {name: value for name, value in batch.items() if name not in bridge.input_fields} 

54 error = None 

55 items = () 

56 dtype = torch.float32 

57 try: 

58 if set(batch).intersection((_METADATA_FIELD, *_FORWARD_FIELDS)): 

59 raise ValueError("encoder transport fields cannot be supplied by the DataLoader") 

60 for name in bridge.input_fields: 

61 value = batch.get(name) 

62 if torch.is_tensor(value) and value.device.type != "cpu": 

63 raise ValueError(f"encoder source field {name} must remain on the CPU") 

64 items = bridge.parse_images(batch) 

65 pixels = batch.get(bridge.payload_field) 

66 if pixels is not None: 

67 dtype = pixels.dtype 

68 except (ValueError, TypeError, KeyError) as exc: 

69 error = str(exc) 

70 prepared[_METADATA_FIELD] = (items, dtype, error) 

71 return prepared 

72 

73 def _check_source_errors(self, error: str | None, device: Any) -> None: 

74 """Make all FSDP participants fail before plan or payload collectives.""" 

75 invalid = torch.tensor(int(error is not None), device=device) 

76 if dist.is_initialized() and self.runtime.global_mesh.size() > 1: 

77 dist.all_reduce(invalid, op=dist.ReduceOp.MAX, group=self.runtime.global_mesh.get_group()) 

78 if bool(invalid.item()): 

79 raise ValueError(f"invalid encoder CPU input: {error or 'another source rank rejected its batch'}") 

80 

81 def __call__(self, batch: Mapping[str, Any] | None, 

82 broadcaster: TPBatchBroadcaster) -> tuple[dict[str, Any], dict[str, Any]]: 

83 """Broadcast ordinary inputs, agree on a plan, and materialize local patches.""" 

84 source = self._prepare_source(batch) if batch is not None else None 

85 device_batch = broadcaster.broadcast(source, broadcast_all_fields=True) 

86 items, dtype, error = device_batch.pop(_METADATA_FIELD) 

87 self._check_source_errors(error, broadcaster.device) 

88 layout = DecoderSequenceLayout.build( 

89 mesh_context=self.runtime.mesh_context, cp_local_length=int(device_batch["input_ids"].shape[1]), 

90 ) 

91 plan = self.runtime.finalize_plan(items, layout) 

92 inputs = {"encoder_parallel_plan": plan} 

93 if plan.active: 

94 inputs["encoder_parallel_inputs"] = EncoderLocalInputs( 

95 self._transport_patches(batch, plan, dtype, broadcaster), 

96 ) 

97 return device_batch, inputs 

98 

99 def _transport_patches(self, batch: Mapping[str, Any] | None, plan: EncoderParallelPlan, 

100 dtype: torch.dtype, broadcaster: TPBatchBroadcaster) -> torch.Tensor: 

101 """Pack on the CPU and reuse a chunk-sized buffer across TP destinations.""" 

102 context = broadcaster.parallel_context 

103 local_count = sum(segment.local_padded_length for segment in plan.local_segments) 

104 local = torch.empty((local_count, *self.runtime.bridge.empty_payload_shape), 

105 dtype=dtype, device=broadcaster.device) 

106 chunk_elements = max(1, self.chunk_bytes // local.element_size()) 

107 target_segments = [] 

108 for target in range(context.tp_world_size): 

109 parent_rank = context.cp_rank * context.tp_world_size + target 

110 bucket, cp_rank = divmod(parent_rank, self.runtime.config.encoder_cp_size) 

111 target_segments.append(build_segments(plan.buckets[bucket], cp_rank, self.runtime.config.encoder_cp_size)) 

112 patch_elements = local[0].numel() 

113 max_elements = max((sum(segment.local_padded_length for segment in segments) * patch_elements 

114 for segments in target_segments[1:]), default=0) 

115 staging = torch.empty(min(chunk_elements, max_elements), dtype=dtype, device=broadcaster.device) 

116 for target, segments in enumerate(target_segments): 

117 packed = self.runtime.bridge.pack_payload(batch, segments) if context.tp_rank == 0 else None 

118 if target == 0: 

119 if context.tp_rank == 0: 

120 local.copy_(packed) 

121 continue 

122 count = sum(segment.local_padded_length for segment in segments) 

123 elements = count * patch_elements 

124 for start in range(0, elements, chunk_elements): 

125 length = min(chunk_elements, elements - start) 

126 buffer = (local.view(-1)[start:start + length] if context.tp_rank == target 

127 else staging[:length]) 

128 if context.tp_rank == 0: 

129 buffer.copy_(packed.view(-1)[start:start + length]) 

130 dist.broadcast(buffer, group=context.tp_group, group_src=0) 

131 return local