[Refactor][EC Connector] Add backend extension points to ECCPUWorker (#54033)

Signed-off-by: Akine-Ko <2923601304@qq.com>
This commit is contained in:
Akine-Ko
2026-09-08 20:13:47 -07:00
committed by GitHub
parent f1aeff6007
commit 41541636fd
2 changed files with 117 additions and 9 deletions
@@ -38,7 +38,10 @@ from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import (
ECCPUTransferDirection,
ECCPUWorker,
)
from vllm.platforms import current_platform
# ── shape constants ──────────────────────────────────────────────────────────
@@ -199,6 +202,71 @@ def _warm_up_stream_pools(worker: ECCPUWorker) -> None:
assert worker._stream_pool, "warm-up did not recycle a stream"
# ── backend extension points ────────────────────────────────────────────────
def test_save_path_delegates_transfer_submission(make_worker):
worker = make_worker()
stream = Mock()
worker._acquire_stream = Mock(return_value=stream)
worker._acquire_event = Mock(side_effect=[Mock(), Mock()])
worker._submit_transfer = Mock()
src = MagicMock()
src.numel.return_value = _HIDDEN_DIM
src.element_size.return_value = _DTYPE.itemsize
src.view.return_value = src
src.data_ptr.return_value = 1000
platform = Mock()
platform.current_stream.return_value = Mock()
platform.stream.return_value = contextlib.nullcontext()
with patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.current_platform",
platform,
):
worker.save_caches({"h": src}, "h", _meta(saves={"h": [3]}))
worker.flush_saves()
worker._submit_transfer.assert_called_once()
_, count, direction = worker._submit_transfer.call_args.args
assert count == 1
assert direction == ECCPUTransferDirection.DEVICE_TO_HOST
def test_load_path_delegates_transfer_submission(make_worker):
worker = make_worker()
stream = Mock()
compute_stream = Mock()
worker._acquire_stream = Mock(return_value=stream)
worker._acquire_event = Mock(side_effect=[Mock(), Mock()])
worker._submit_transfer = Mock()
dst_buf = MagicMock()
dst_buf.data_ptr.return_value = 1000
platform = Mock()
platform.device_type = "cpu"
platform.current_stream.return_value = compute_stream
platform.stream.return_value = contextlib.nullcontext()
with (
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.current_platform",
platform,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.torch.empty",
return_value=dst_buf,
),
):
worker.start_load_caches({}, _meta(loads={"h": [3]}))
worker._submit_transfer.assert_called_once()
_, count, direction = worker._submit_transfer.call_args.args
assert count == 1
assert direction == ECCPUTransferDirection.HOST_TO_DEVICE
# ── save_caches ──────────────────────────────────────────────────────────────
@@ -792,8 +860,10 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
worker._inflight_loads = deque()
worker._stream_pool = []
worker._event_pool = []
worker._shutdown_transfer_backend = MagicMock()
worker.shutdown()
worker._shutdown_transfer_backend.assert_called_once()
mock_region.cleanup.assert_called_once()
mock_region.cleanup.side_effect = RuntimeError("boom")
@@ -10,6 +10,7 @@ blocks to copy in each direction.
from collections import deque
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum, auto
from itertools import chain
from typing import TYPE_CHECKING
@@ -40,6 +41,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class ECCPUTransferDirection(Enum):
HOST_TO_DEVICE = auto()
DEVICE_TO_HOST = auto()
def _coalesce_runs(
block_ids: Sequence[int],
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
@@ -111,8 +117,7 @@ class ECCPUWorker:
# Model dtype; used to reinterpret raw int8 blocks on load.
self._dtype = vllm_config.model_config.dtype
if is_pin_memory_available():
self._region.pin_memory()
self._pin_shared_region()
# All TP/PCP ranks hold identical encoder output. Only one rank
# per mmap needs to write — saves host memory bandwidth.
@@ -188,6 +193,34 @@ class ECCPUWorker:
self._event_pool.append(transfer.end_event)
return done
def _pin_shared_region(self) -> None:
"""Prepare the shared region for device transfers."""
if is_pin_memory_available():
self._region.pin_memory()
def _submit_transfer(
self,
bufs: DescriptorBuffers,
count: int,
direction: ECCPUTransferDirection,
) -> None:
"""Submit a transfer on the current backend stream."""
src_ptrs = bufs.src_ptrs[:count]
dst_ptrs = bufs.dst_ptrs[:count]
sizes = bufs.sizes[:count]
if direction == ECCPUTransferDirection.HOST_TO_DEVICE:
swap_blocks_batch(
src_ptrs,
dst_ptrs,
sizes,
is_src_access_order_any=True,
)
else:
swap_blocks_batch(src_ptrs, dst_ptrs, sizes)
def _shutdown_transfer_backend(self) -> None:
"""Release backend-specific transfer resources."""
def save_caches(
self,
encoder_cache: dict[str, torch.Tensor],
@@ -269,7 +302,6 @@ class ECCPUWorker:
bufs = self._save_bufs
stream = self._save_stream
assert bufs is not None and stream is not None
src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes
n = self._save_count
num_bytes = self._save_bytes
@@ -280,7 +312,11 @@ class ECCPUWorker:
end_event = self._acquire_event()
with current_platform.stream(stream):
start_event.record(stream)
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
self._submit_transfer(
bufs,
n,
ECCPUTransferDirection.DEVICE_TO_HOST,
)
end_event.record(stream)
self._inflight_saves.append(
@@ -357,12 +393,13 @@ class ECCPUWorker:
num_blocks * block_size,
)
op_idx = slots.size
src_ptrs = bufs.src_ptrs[:op_idx]
dst_ptrs = bufs.dst_ptrs[:op_idx]
sizes = bufs.sizes[:op_idx]
start_event.record(stream)
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
self._submit_transfer(
bufs,
op_idx,
ECCPUTransferDirection.HOST_TO_DEVICE,
)
end_event.record(stream)
self._inflight_loads.append(
Transfer(
@@ -404,6 +441,7 @@ class ECCPUWorker:
def shutdown(self) -> None:
for transfer in (*self._inflight_saves, *self._inflight_loads):
transfer.end_event.synchronize()
self._shutdown_transfer_backend()
self._save_bufs = None
self._save_stream = None
self._save_count = 0