Coverage for / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / distributed / encoder_parallel / transport.py: 96%
83 statements
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
« prev ^ index » next coverage.py v7.13.1, created at 2026-11-19 05:29 +0800
1# Copyright 2026 Huawei Technologies Co., Ltd
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ============================================================================
15"""Transport CPU image shards without staging complete images on a device."""
17from __future__ import annotations
19from typing import Any, Mapping
21import torch
22import torch.distributed as dist
24from hyper_parallel.data.parallel import TPBatchBroadcaster
25from hyper_parallel.distributed.encoder_parallel.contracts import EncoderLocalInputs
26from hyper_parallel.distributed.encoder_parallel.planning import DecoderSequenceLayout, build_segments
27from hyper_parallel.distributed.encoder_parallel.runtime import EncoderParallelPlan, EncoderParallelRuntime
30_METADATA_FIELD = "_encoder_transport_metadata"
31_FORWARD_FIELDS = ("encoder_parallel_plan", "encoder_parallel_inputs")
32_CHUNK_BYTES = 4 * 1024 * 1024
35class EncoderParallelBatchTransport:
36 """Plan from small metadata and send only each destination's CPU patches.
38 Every decoder CP coordinate already has a complete CPU batch on TP0. TP
39 peers may belong to different encoder DP buckets or CP shards, so their
40 payloads are transported individually with bounded device staging.
41 """
43 def __init__(self, *, runtime: EncoderParallelRuntime, chunk_bytes: int = _CHUNK_BYTES) -> None:
44 """Set the runtime and maximum device staging size in bytes."""
45 if isinstance(chunk_bytes, bool) or not isinstance(chunk_bytes, int) or chunk_bytes < 1:
46 raise ValueError("encoder transport chunk_bytes must be a positive integer")
47 self.runtime = runtime
48 self.chunk_bytes = chunk_bytes
50 def _prepare_source(self, batch: Mapping[str, Any]) -> dict[str, Any]:
51 """Validate CPU images and keep tensor payloads out of TP broadcasting."""
52 bridge = self.runtime.bridge
53 prepared = {name: value for name, value in batch.items() if name not in bridge.input_fields}
54 error = None
55 items = ()
56 dtype = torch.float32
57 try:
58 if set(batch).intersection((_METADATA_FIELD, *_FORWARD_FIELDS)):
59 raise ValueError("encoder transport fields cannot be supplied by the DataLoader")
60 for name in bridge.input_fields:
61 value = batch.get(name)
62 if torch.is_tensor(value) and value.device.type != "cpu":
63 raise ValueError(f"encoder source field {name} must remain on the CPU")
64 items = bridge.parse_images(batch)
65 pixels = batch.get(bridge.payload_field)
66 if pixels is not None:
67 dtype = pixels.dtype
68 except (ValueError, TypeError, KeyError) as exc:
69 error = str(exc)
70 prepared[_METADATA_FIELD] = (items, dtype, error)
71 return prepared
73 def _check_source_errors(self, error: str | None, device: Any) -> None:
74 """Make all FSDP participants fail before plan or payload collectives."""
75 invalid = torch.tensor(int(error is not None), device=device)
76 if dist.is_initialized() and self.runtime.global_mesh.size() > 1:
77 dist.all_reduce(invalid, op=dist.ReduceOp.MAX, group=self.runtime.global_mesh.get_group())
78 if bool(invalid.item()):
79 raise ValueError(f"invalid encoder CPU input: {error or 'another source rank rejected its batch'}")
81 def __call__(self, batch: Mapping[str, Any] | None,
82 broadcaster: TPBatchBroadcaster) -> tuple[dict[str, Any], dict[str, Any]]:
83 """Broadcast ordinary inputs, agree on a plan, and materialize local patches."""
84 source = self._prepare_source(batch) if batch is not None else None
85 device_batch = broadcaster.broadcast(source, broadcast_all_fields=True)
86 items, dtype, error = device_batch.pop(_METADATA_FIELD)
87 self._check_source_errors(error, broadcaster.device)
88 layout = DecoderSequenceLayout.build(
89 mesh_context=self.runtime.mesh_context, cp_local_length=int(device_batch["input_ids"].shape[1]),
90 )
91 plan = self.runtime.finalize_plan(items, layout)
92 inputs = {"encoder_parallel_plan": plan}
93 if plan.active:
94 inputs["encoder_parallel_inputs"] = EncoderLocalInputs(
95 self._transport_patches(batch, plan, dtype, broadcaster),
96 )
97 return device_batch, inputs
99 def _transport_patches(self, batch: Mapping[str, Any] | None, plan: EncoderParallelPlan,
100 dtype: torch.dtype, broadcaster: TPBatchBroadcaster) -> torch.Tensor:
101 """Pack on the CPU and reuse a chunk-sized buffer across TP destinations."""
102 context = broadcaster.parallel_context
103 local_count = sum(segment.local_padded_length for segment in plan.local_segments)
104 local = torch.empty((local_count, *self.runtime.bridge.empty_payload_shape),
105 dtype=dtype, device=broadcaster.device)
106 chunk_elements = max(1, self.chunk_bytes // local.element_size())
107 target_segments = []
108 for target in range(context.tp_world_size):
109 parent_rank = context.cp_rank * context.tp_world_size + target
110 bucket, cp_rank = divmod(parent_rank, self.runtime.config.encoder_cp_size)
111 target_segments.append(build_segments(plan.buckets[bucket], cp_rank, self.runtime.config.encoder_cp_size))
112 patch_elements = local[0].numel()
113 max_elements = max((sum(segment.local_padded_length for segment in segments) * patch_elements
114 for segments in target_segments[1:]), default=0)
115 staging = torch.empty(min(chunk_elements, max_elements), dtype=dtype, device=broadcaster.device)
116 for target, segments in enumerate(target_segments):
117 packed = self.runtime.bridge.pack_payload(batch, segments) if context.tp_rank == 0 else None
118 if target == 0:
119 if context.tp_rank == 0:
120 local.copy_(packed)
121 continue
122 count = sum(segment.local_padded_length for segment in segments)
123 elements = count * patch_elements
124 for start in range(0, elements, chunk_elements):
125 length = min(chunk_elements, elements - start)
126 buffer = (local.view(-1)[start:start + length] if context.tp_rank == target
127 else staging[:length])
128 if context.tp_rank == 0:
129 buffer.copy_(packed.view(-1)[start:start + length])
130 dist.broadcast(buffer, group=context.tp_group, group_src=0)
131 return local