[Refactor][EC Connector] Add backend extension points to ECCPUWorker (#54033)
Signed-off-by: Akine-Ko <2923601304@qq.com>
This commit is contained in:
@@ -38,7 +38,10 @@ from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import (
|
||||
ECCPUTransferDirection,
|
||||
ECCPUWorker,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# ── shape constants ──────────────────────────────────────────────────────────
|
||||
@@ -199,6 +202,71 @@ def _warm_up_stream_pools(worker: ECCPUWorker) -> None:
|
||||
assert worker._stream_pool, "warm-up did not recycle a stream"
|
||||
|
||||
|
||||
# ── backend extension points ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_save_path_delegates_transfer_submission(make_worker):
|
||||
worker = make_worker()
|
||||
stream = Mock()
|
||||
worker._acquire_stream = Mock(return_value=stream)
|
||||
worker._acquire_event = Mock(side_effect=[Mock(), Mock()])
|
||||
worker._submit_transfer = Mock()
|
||||
|
||||
src = MagicMock()
|
||||
src.numel.return_value = _HIDDEN_DIM
|
||||
src.element_size.return_value = _DTYPE.itemsize
|
||||
src.view.return_value = src
|
||||
src.data_ptr.return_value = 1000
|
||||
|
||||
platform = Mock()
|
||||
platform.current_stream.return_value = Mock()
|
||||
platform.stream.return_value = contextlib.nullcontext()
|
||||
with patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.current_platform",
|
||||
platform,
|
||||
):
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [3]}))
|
||||
worker.flush_saves()
|
||||
|
||||
worker._submit_transfer.assert_called_once()
|
||||
_, count, direction = worker._submit_transfer.call_args.args
|
||||
assert count == 1
|
||||
assert direction == ECCPUTransferDirection.DEVICE_TO_HOST
|
||||
|
||||
|
||||
def test_load_path_delegates_transfer_submission(make_worker):
|
||||
worker = make_worker()
|
||||
stream = Mock()
|
||||
compute_stream = Mock()
|
||||
worker._acquire_stream = Mock(return_value=stream)
|
||||
worker._acquire_event = Mock(side_effect=[Mock(), Mock()])
|
||||
worker._submit_transfer = Mock()
|
||||
|
||||
dst_buf = MagicMock()
|
||||
dst_buf.data_ptr.return_value = 1000
|
||||
|
||||
platform = Mock()
|
||||
platform.device_type = "cpu"
|
||||
platform.current_stream.return_value = compute_stream
|
||||
platform.stream.return_value = contextlib.nullcontext()
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.current_platform",
|
||||
platform,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.torch.empty",
|
||||
return_value=dst_buf,
|
||||
),
|
||||
):
|
||||
worker.start_load_caches({}, _meta(loads={"h": [3]}))
|
||||
|
||||
worker._submit_transfer.assert_called_once()
|
||||
_, count, direction = worker._submit_transfer.call_args.args
|
||||
assert count == 1
|
||||
assert direction == ECCPUTransferDirection.HOST_TO_DEVICE
|
||||
|
||||
|
||||
# ── save_caches ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -792,8 +860,10 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
|
||||
worker._inflight_loads = deque()
|
||||
worker._stream_pool = []
|
||||
worker._event_pool = []
|
||||
worker._shutdown_transfer_backend = MagicMock()
|
||||
|
||||
worker.shutdown()
|
||||
worker._shutdown_transfer_backend.assert_called_once()
|
||||
mock_region.cleanup.assert_called_once()
|
||||
|
||||
mock_region.cleanup.side_effect = RuntimeError("boom")
|
||||
|
||||
@@ -10,6 +10,7 @@ blocks to copy in each direction.
|
||||
from collections import deque
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -40,6 +41,11 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUTransferDirection(Enum):
|
||||
HOST_TO_DEVICE = auto()
|
||||
DEVICE_TO_HOST = auto()
|
||||
|
||||
|
||||
def _coalesce_runs(
|
||||
block_ids: Sequence[int],
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
@@ -111,8 +117,7 @@ class ECCPUWorker:
|
||||
# Model dtype; used to reinterpret raw int8 blocks on load.
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
|
||||
if is_pin_memory_available():
|
||||
self._region.pin_memory()
|
||||
self._pin_shared_region()
|
||||
|
||||
# All TP/PCP ranks hold identical encoder output. Only one rank
|
||||
# per mmap needs to write — saves host memory bandwidth.
|
||||
@@ -188,6 +193,34 @@ class ECCPUWorker:
|
||||
self._event_pool.append(transfer.end_event)
|
||||
return done
|
||||
|
||||
def _pin_shared_region(self) -> None:
|
||||
"""Prepare the shared region for device transfers."""
|
||||
if is_pin_memory_available():
|
||||
self._region.pin_memory()
|
||||
|
||||
def _submit_transfer(
|
||||
self,
|
||||
bufs: DescriptorBuffers,
|
||||
count: int,
|
||||
direction: ECCPUTransferDirection,
|
||||
) -> None:
|
||||
"""Submit a transfer on the current backend stream."""
|
||||
src_ptrs = bufs.src_ptrs[:count]
|
||||
dst_ptrs = bufs.dst_ptrs[:count]
|
||||
sizes = bufs.sizes[:count]
|
||||
if direction == ECCPUTransferDirection.HOST_TO_DEVICE:
|
||||
swap_blocks_batch(
|
||||
src_ptrs,
|
||||
dst_ptrs,
|
||||
sizes,
|
||||
is_src_access_order_any=True,
|
||||
)
|
||||
else:
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes)
|
||||
|
||||
def _shutdown_transfer_backend(self) -> None:
|
||||
"""Release backend-specific transfer resources."""
|
||||
|
||||
def save_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
@@ -269,7 +302,6 @@ class ECCPUWorker:
|
||||
bufs = self._save_bufs
|
||||
stream = self._save_stream
|
||||
assert bufs is not None and stream is not None
|
||||
src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes
|
||||
n = self._save_count
|
||||
num_bytes = self._save_bytes
|
||||
|
||||
@@ -280,7 +312,11 @@ class ECCPUWorker:
|
||||
end_event = self._acquire_event()
|
||||
with current_platform.stream(stream):
|
||||
start_event.record(stream)
|
||||
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
|
||||
self._submit_transfer(
|
||||
bufs,
|
||||
n,
|
||||
ECCPUTransferDirection.DEVICE_TO_HOST,
|
||||
)
|
||||
end_event.record(stream)
|
||||
|
||||
self._inflight_saves.append(
|
||||
@@ -357,12 +393,13 @@ class ECCPUWorker:
|
||||
num_blocks * block_size,
|
||||
)
|
||||
op_idx = slots.size
|
||||
src_ptrs = bufs.src_ptrs[:op_idx]
|
||||
dst_ptrs = bufs.dst_ptrs[:op_idx]
|
||||
sizes = bufs.sizes[:op_idx]
|
||||
|
||||
start_event.record(stream)
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
|
||||
self._submit_transfer(
|
||||
bufs,
|
||||
op_idx,
|
||||
ECCPUTransferDirection.HOST_TO_DEVICE,
|
||||
)
|
||||
end_event.record(stream)
|
||||
self._inflight_loads.append(
|
||||
Transfer(
|
||||
@@ -404,6 +441,7 @@ class ECCPUWorker:
|
||||
def shutdown(self) -> None:
|
||||
for transfer in (*self._inflight_saves, *self._inflight_loads):
|
||||
transfer.end_event.synchronize()
|
||||
self._shutdown_transfer_backend()
|
||||
self._save_bufs = None
|
||||
self._save_stream = None
|
||||
self._save_count = 0
|
||||
|
||||
Reference in New Issue
Block a user