[XPU] Support EC connector KV Offloading on XPU (#49532)

Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
Chaojun Zhang
2026-08-19 10:25:23 +08:00
committed by GitHub
parent b1d9337e92
commit f485081e8b
5 changed files with 93 additions and 58 deletions
+4 -2
View File
@@ -8,7 +8,7 @@ steps:
agent_tags:
label: production
gpu: 1+
mem: 16+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -22,6 +22,7 @@ steps:
- tests/v1/kv_offload
- tests/v1/worker
- tests/v1/kv_connector/unit
- tests/v1/ec_connector/unit
- tests/v1/metrics
- tests/entrypoints/openai/correctness/test_lmeval.py
commands:
@@ -30,7 +31,8 @@ steps:
'pip install -r requirements/kv_connectors.txt &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
cd tests &&
pytest -v -s v1/executor'
pytest -v -s v1/executor &&
VLLM_BATCH_INVARIANT=1 pytest -v -s -m 'not cpu_test' v1/ec_connector/unit'
- label: V1 Sample + Logits
timeout_in_minutes: 90
@@ -2,10 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ECCPUWorker.
The byte-level tests exercise real CUDA stream/event coordination
against a real ``ECSharedRegion`` mmap and are skipped on hosts without
CUDA. The lifecycle tests don't fire any ``torch.cuda.*`` primitives and
run anywhere.
The byte-level tests exercise real accelerator (CUDA/XPU/...) stream/event
coordination against a real ``ECSharedRegion`` mmap and are skipped on hosts
without an accelerator. The lifecycle tests don't fire any
``current_platform.*`` device primitives and run anywhere.
Mocking policy
--------------
@@ -37,6 +37,7 @@ from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker
from vllm.platforms import current_platform
# ── shape constants ──────────────────────────────────────────────────────────
@@ -47,9 +48,11 @@ _DTYPE = torch.float16
_BLOCK_SIZE_BYTES = _HIDDEN_DIM * _DTYPE.itemsize
_NUM_BLOCKS = 8
_requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(),
reason="exercises real CUDA stream/event coordination in ECCPUWorker",
DEVICE_TYPE = current_platform.device_type
_requires_accelerator = pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_xpu()),
reason="exercises real accelerator stream/event coordination in ECCPUWorker",
)
@@ -136,7 +139,7 @@ def make_worker():
# ── save_caches ──────────────────────────────────────────────────────────────
@_requires_cuda
@_requires_accelerator
@pytest.mark.parametrize(
"n_elements,n_blocks",
[
@@ -158,7 +161,7 @@ def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks
sentinel = 0x5A
worker._region.blocks.fill_(sentinel)
src = torch.arange(n_elements, dtype=_DTYPE, device="cuda")
src = torch.arange(n_elements, dtype=_DTYPE, device=DEVICE_TYPE)
expected_bytes = src.cpu().reshape(-1).view(torch.uint8)
total_bytes = n_elements * _DTYPE.itemsize
@@ -228,15 +231,15 @@ def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
@_requires_cuda
@_requires_accelerator
def test_save_caches_batches_multiple_hashes(make_worker):
"""Multiple save_caches calls are batched into a single flush."""
worker = make_worker()
sentinel = 0x5A
worker._region.blocks.fill_(sentinel)
src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device="cuda")
src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
cache = {"a": src_a, "b": src_b}
worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]}))
@@ -258,7 +261,7 @@ def test_save_caches_batches_multiple_hashes(make_worker):
# ── start_load_caches ────────────────────────────────────────────────────────
@_requires_cuda
@_requires_accelerator
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
"""Single batched load across all hashes with correct byte→dtype→shape."""
worker = make_worker()
@@ -276,20 +279,22 @@ def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
out = encoder_cache["h"]
assert out.is_cuda, "consumer worker must place the tensor on the GPU"
assert out.device.type == DEVICE_TYPE, (
"consumer worker must place the tensor on the accelerator"
)
assert out.shape == (n_blocks, _HIDDEN_DIM)
assert out.dtype == _DTYPE
assert torch.equal(out.cpu(), src_orig)
@_requires_cuda
@_requires_accelerator
def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
"""If ``encoder_cache`` already holds the ``mm_hash``, the worker must
not overwrite it."""
worker = make_worker()
worker._region.blocks[0].fill_(0x42)
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device="cuda")
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device=DEVICE_TYPE)
encoder_cache = {"h": sentinel}
worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]}))
@@ -298,7 +303,7 @@ def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
)
@_requires_cuda
@_requires_accelerator
def test_start_load_caches_noop_when_loads_is_empty(make_worker):
"""When ``meta.loads`` is empty the early-return must fire."""
worker = make_worker()
@@ -308,7 +313,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
assert encoder_cache == {}
@_requires_cuda
@_requires_accelerator
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
"""Cached entries are preserved while new ones are loaded."""
worker = make_worker()
@@ -321,7 +326,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
for i, idx in enumerate(new_block_ids):
worker._region.blocks[idx].copy_(src_int8[i])
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device="cuda")
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device=DEVICE_TYPE)
encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor}
worker.start_load_caches(
encoder_cache,
@@ -336,7 +341,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
assert torch.equal(out.cpu(), src_orig)
@_requires_cuda
@_requires_accelerator
@pytest.mark.parametrize(
"tp_rank,pcp_rank",
[(0, 0), (1, 0), (0, 1), (1, 1)],
@@ -358,23 +363,23 @@ def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
out = encoder_cache["h"]
assert out.is_cuda
assert out.device.type == DEVICE_TYPE
assert torch.equal(out.cpu(), src_orig)
# ── round-trip ───────────────────────────────────────────────────────────────
@_requires_cuda
@_requires_accelerator
def test_save_then_load_round_trips_bytes(make_worker):
"""Full producer→mmap→consumer byte path in one shot."""
worker = make_worker()
n_blocks = 3
block_ids = [5, 1, 6]
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
n_blocks, _HIDDEN_DIM
)
src = torch.arange(
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
).reshape(n_blocks, _HIDDEN_DIM)
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
worker.flush_saves()
@@ -390,12 +395,12 @@ def test_save_then_load_round_trips_bytes(make_worker):
# ── buffer recycling ────────────────────────────────────────────────────────
@_requires_cuda
@_requires_accelerator
def test_buffer_pool_is_reused_across_save_steps(make_worker):
"""After flush_saves, descriptor buffers are returned to the pool and
reused on the next flush — no reallocation."""
worker = make_worker()
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]}))
worker.flush_saves()
@@ -411,7 +416,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
@_requires_cuda
@_requires_accelerator
def test_buffer_pool_is_reused_across_load_steps(make_worker):
"""After start_load_caches, descriptor buffers are returned to the pool
and reused on the next call."""
@@ -435,11 +440,11 @@ def test_buffer_pool_is_reused_across_load_steps(make_worker):
# ── stream management ────────────────────────────────────────────────────────
@_requires_cuda
@_requires_accelerator
def test_stream_initialized_at_construction(make_worker):
"""``_load_stream`` must be a fully initialized CUDA stream."""
"""``_load_stream`` must be a fully initialized accelerator stream."""
worker = make_worker()
assert isinstance(worker._load_stream, torch.cuda.Stream)
assert isinstance(worker._load_stream, current_platform.Stream)
# ── lifecycle ────────────────────────────────────────────────────────────────
@@ -518,14 +523,14 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
@_requires_cuda
@_requires_accelerator
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
"""Full pipeline: scheduler allocates blocks, worker saves GPU tensor to
mmap via flush_saves, scheduler marks ready after step delay, worker
loads from mmap back to GPU, and the result matches the original.
Exercises the real scheduler + worker cooperation through a shared
ECSharedRegion, with real CUDA transfers and stream coordination.
ECSharedRegion, with real accelerator transfers and stream coordination.
"""
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
@@ -551,9 +556,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
# -- Step 1: scheduler allocates, worker saves --
n_blocks = 3
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
n_blocks, _HIDDEN_DIM
)
src = torch.arange(
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
).reshape(n_blocks, _HIDDEN_DIM)
class _Pos:
offset = 0
@@ -590,7 +595,7 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
worker.start_load_caches(load_cache, meta_load)
out = load_cache["img_001"]
assert out.is_cuda
assert out.device.type == DEVICE_TYPE
assert out.shape == src.shape
assert out.dtype == src.dtype
assert torch.equal(out.cpu(), src.cpu())
@@ -7,7 +7,7 @@ Verifies:
- Accuracy: outputs from EC CPU cache match fresh encoder computation.
- Latency: loading from EC CPU cache is faster than a cold encoder run.
Requires a CUDA GPU and the Qwen2-VL-2B-Instruct model.
Requires a CUDA or XPU GPU and the Qwen2-VL-2B-Instruct model.
"""
import time
@@ -57,8 +57,8 @@ def _wait_for_ec_ready(llm: LLM) -> None:
def _latency_test(llm: LLM) -> None:
"""Verify EC CPU cache hit is faster than cold encoder computation."""
if not current_platform.is_cuda():
pytest.skip("Latency test requires CUDA")
if not (current_platform.is_cuda() or current_platform.is_xpu()):
pytest.skip("Latency test requires an accelerator (CUDA or XPU)")
sampling_params = SamplingParams(max_tokens=1, temperature=0)
base_image = ImageAsset("stop_sign").pil_image
@@ -134,7 +134,10 @@ def _accuracy_test(llm: LLM) -> None:
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
@pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_xpu()),
reason="Requires an accelerator (CUDA or XPU)",
)
def test_ec_cpu_offloading() -> None:
"""Tests ECCPUConnector accuracy and latency with a VLM model."""
ec_transfer_config = ECTransferConfig(
@@ -102,16 +102,15 @@ class ECCPUWorker:
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
src_ptrs, dst_ptrs, sizes = self._save_bufs
bufs = self._save_bufs
src_base = src.view(-1).view(torch.uint8).data_ptr()
dst_base = self._region.blocks.data_ptr()
idx = self._save_count
for i, block_idx in enumerate(block_ids):
start = i * block_size
src_ptrs[idx] = src_base + start
dst_ptrs[idx] = dst_base + block_idx * block_size
sizes[idx] = min(block_size, total_bytes - start)
bufs.set_ptrs(idx, src_base + start, dst_base + block_idx * block_size)
bufs.sizes[idx] = min(block_size, total_bytes - start)
idx += 1
self._save_count = idx
@@ -123,7 +122,7 @@ class ECCPUWorker:
bufs = self._save_bufs
assert bufs is not None
src_ptrs, dst_ptrs, sizes = bufs
src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes
n = self._save_count
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
@@ -173,8 +172,11 @@ class ECCPUWorker:
op_idx = 0
for block_ids in load_items.values():
for block_idx in block_ids:
src_ptrs[op_idx] = src_base + block_idx * block_size
dst_ptrs[op_idx] = dst_buf_base + op_idx * block_size
bufs.set_ptrs(
op_idx,
src_base + block_idx * block_size,
dst_buf_base + op_idx * block_size,
)
op_idx += 1
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) int64 tensor triples.
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) tensor triples.
Used by ECCPUWorker to batch swap_blocks_batch descriptors without
per-step allocation overhead.
@@ -8,20 +8,46 @@ per-step allocation overhead.
from typing import NamedTuple
import numpy as np
import torch
from vllm.platforms import current_platform
# CUDA/ROCm cache_kernels.cu requires int64 pointers; the XPU DMA engine
# requires uint64 (see vllm._custom_ops.swap_blocks_batch).
_PTR_DTYPE = torch.uint64 if current_platform.is_xpu() else torch.int64
class DescriptorBuffers(NamedTuple):
src_ptrs: torch.Tensor
dst_ptrs: torch.Tensor
sizes: torch.Tensor
# Numpy aliases of src_ptrs/dst_ptrs, written via set_ptrs().
src_np: np.ndarray
dst_np: np.ndarray
def set_ptrs(self, idx: int, src: int, dst: int) -> None:
"""Record the source and destination address of descriptor *idx*.
TODO(torch>=2.14): drop this indirection and assign the tensors
directly once the minimum supported torch is 2.14. The numpy detour
exists only because torch's setitem unpacks the value as a signed
long long before 2.14 (pytorch#191458), rejecting XPU USM pointers
>= 2**63, while the two's-complement rewrite those versions accept is
in turn rejected by uint64 from 2.14 on. Numpy casts against the
array dtype and so works on either. `sizes` holds byte counts and
needs no such care.
"""
self.src_np[idx] = src
self.dst_np[idx] = dst
class DescriptorBufferPool:
"""Pool of descriptor buffer triples for swap_blocks_batch.
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D int64
tensors of equal length. Buffers are recycled across steps; if a
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D tensors
(dtype `_PTR_DTYPE`, platform-dependent) of equal length, paired with
numpy aliases used to fill them. Buffers are recycled across steps; if a
returned buffer is too small it is discarded and a fresh one allocated.
"""
@@ -35,11 +61,8 @@ class DescriptorBufferPool:
bufs = self._pool.pop()
if bufs.src_ptrs.numel() >= n:
return bufs
return DescriptorBuffers(
torch.empty(n, dtype=torch.int64),
torch.empty(n, dtype=torch.int64),
torch.empty(n, dtype=torch.int64),
)
src, dst, sizes = (torch.empty(n, dtype=_PTR_DTYPE) for _ in range(3))
return DescriptorBuffers(src, dst, sizes, src.numpy(), dst.numpy())
def release(self, bufs: DescriptorBuffers) -> None:
"""Return a buffer triple to the pool for reuse."""