diff --git a/tests/v1/ec_connector/unit/cpu/worker/test_worker.py b/tests/v1/ec_connector/unit/cpu/worker/test_worker.py index abc3864c39..f4f5f03f69 100644 --- a/tests/v1/ec_connector/unit/cpu/worker/test_worker.py +++ b/tests/v1/ec_connector/unit/cpu/worker/test_worker.py @@ -38,7 +38,10 @@ from vllm.distributed.ec_transfer.ec_connector.cpu.common import ( from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( ECSharedRegion, ) -from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker +from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ( + ECCPUTransferDirection, + ECCPUWorker, +) from vllm.platforms import current_platform # ── shape constants ────────────────────────────────────────────────────────── @@ -199,6 +202,71 @@ def _warm_up_stream_pools(worker: ECCPUWorker) -> None: assert worker._stream_pool, "warm-up did not recycle a stream" +# ── backend extension points ──────────────────────────────────────────────── + + +def test_save_path_delegates_transfer_submission(make_worker): + worker = make_worker() + stream = Mock() + worker._acquire_stream = Mock(return_value=stream) + worker._acquire_event = Mock(side_effect=[Mock(), Mock()]) + worker._submit_transfer = Mock() + + src = MagicMock() + src.numel.return_value = _HIDDEN_DIM + src.element_size.return_value = _DTYPE.itemsize + src.view.return_value = src + src.data_ptr.return_value = 1000 + + platform = Mock() + platform.current_stream.return_value = Mock() + platform.stream.return_value = contextlib.nullcontext() + with patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.current_platform", + platform, + ): + worker.save_caches({"h": src}, "h", _meta(saves={"h": [3]})) + worker.flush_saves() + + worker._submit_transfer.assert_called_once() + _, count, direction = worker._submit_transfer.call_args.args + assert count == 1 + assert direction == ECCPUTransferDirection.DEVICE_TO_HOST + + +def test_load_path_delegates_transfer_submission(make_worker): + worker = make_worker() + stream = Mock() + compute_stream = Mock() + worker._acquire_stream = Mock(return_value=stream) + worker._acquire_event = Mock(side_effect=[Mock(), Mock()]) + worker._submit_transfer = Mock() + + dst_buf = MagicMock() + dst_buf.data_ptr.return_value = 1000 + + platform = Mock() + platform.device_type = "cpu" + platform.current_stream.return_value = compute_stream + platform.stream.return_value = contextlib.nullcontext() + with ( + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.current_platform", + platform, + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.worker.torch.empty", + return_value=dst_buf, + ), + ): + worker.start_load_caches({}, _meta(loads={"h": [3]})) + + worker._submit_transfer.assert_called_once() + _, count, direction = worker._submit_transfer.call_args.args + assert count == 1 + assert direction == ECCPUTransferDirection.HOST_TO_DEVICE + + # ── save_caches ────────────────────────────────────────────────────────────── @@ -792,8 +860,10 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm): worker._inflight_loads = deque() worker._stream_pool = [] worker._event_pool = [] + worker._shutdown_transfer_backend = MagicMock() worker.shutdown() + worker._shutdown_transfer_backend.assert_called_once() mock_region.cleanup.assert_called_once() mock_region.cleanup.side_effect = RuntimeError("boom") diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py index 58f29f3dbd..ef7ff0f751 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py @@ -10,6 +10,7 @@ blocks to copy in each direction. from collections import deque from collections.abc import Sequence from dataclasses import dataclass +from enum import Enum, auto from itertools import chain from typing import TYPE_CHECKING @@ -40,6 +41,11 @@ if TYPE_CHECKING: logger = init_logger(__name__) +class ECCPUTransferDirection(Enum): + HOST_TO_DEVICE = auto() + DEVICE_TO_HOST = auto() + + def _coalesce_runs( block_ids: Sequence[int], ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: @@ -111,8 +117,7 @@ class ECCPUWorker: # Model dtype; used to reinterpret raw int8 blocks on load. self._dtype = vllm_config.model_config.dtype - if is_pin_memory_available(): - self._region.pin_memory() + self._pin_shared_region() # All TP/PCP ranks hold identical encoder output. Only one rank # per mmap needs to write — saves host memory bandwidth. @@ -188,6 +193,34 @@ class ECCPUWorker: self._event_pool.append(transfer.end_event) return done + def _pin_shared_region(self) -> None: + """Prepare the shared region for device transfers.""" + if is_pin_memory_available(): + self._region.pin_memory() + + def _submit_transfer( + self, + bufs: DescriptorBuffers, + count: int, + direction: ECCPUTransferDirection, + ) -> None: + """Submit a transfer on the current backend stream.""" + src_ptrs = bufs.src_ptrs[:count] + dst_ptrs = bufs.dst_ptrs[:count] + sizes = bufs.sizes[:count] + if direction == ECCPUTransferDirection.HOST_TO_DEVICE: + swap_blocks_batch( + src_ptrs, + dst_ptrs, + sizes, + is_src_access_order_any=True, + ) + else: + swap_blocks_batch(src_ptrs, dst_ptrs, sizes) + + def _shutdown_transfer_backend(self) -> None: + """Release backend-specific transfer resources.""" + def save_caches( self, encoder_cache: dict[str, torch.Tensor], @@ -269,7 +302,6 @@ class ECCPUWorker: bufs = self._save_bufs stream = self._save_stream assert bufs is not None and stream is not None - src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes n = self._save_count num_bytes = self._save_bytes @@ -280,7 +312,11 @@ class ECCPUWorker: end_event = self._acquire_event() with current_platform.stream(stream): start_event.record(stream) - swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n]) + self._submit_transfer( + bufs, + n, + ECCPUTransferDirection.DEVICE_TO_HOST, + ) end_event.record(stream) self._inflight_saves.append( @@ -357,12 +393,13 @@ class ECCPUWorker: num_blocks * block_size, ) op_idx = slots.size - src_ptrs = bufs.src_ptrs[:op_idx] - dst_ptrs = bufs.dst_ptrs[:op_idx] - sizes = bufs.sizes[:op_idx] start_event.record(stream) - swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True) + self._submit_transfer( + bufs, + op_idx, + ECCPUTransferDirection.HOST_TO_DEVICE, + ) end_event.record(stream) self._inflight_loads.append( Transfer( @@ -404,6 +441,7 @@ class ECCPUWorker: def shutdown(self) -> None: for transfer in (*self._inflight_saves, *self._inflight_loads): transfer.end_event.synchronize() + self._shutdown_transfer_backend() self._save_bufs = None self._save_stream = None self._save_count = 0