[XPU] Support EC connector KV Offloading on XPU (#49532)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
@@ -8,7 +8,7 @@ steps:
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 16+
|
||||
mem: 24+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
@@ -22,6 +22,7 @@ steps:
|
||||
- tests/v1/kv_offload
|
||||
- tests/v1/worker
|
||||
- tests/v1/kv_connector/unit
|
||||
- tests/v1/ec_connector/unit
|
||||
- tests/v1/metrics
|
||||
- tests/entrypoints/openai/correctness/test_lmeval.py
|
||||
commands:
|
||||
@@ -30,7 +31,8 @@ steps:
|
||||
'pip install -r requirements/kv_connectors.txt &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
cd tests &&
|
||||
pytest -v -s v1/executor'
|
||||
pytest -v -s v1/executor &&
|
||||
VLLM_BATCH_INVARIANT=1 pytest -v -s -m 'not cpu_test' v1/ec_connector/unit'
|
||||
|
||||
- label: V1 Sample + Logits
|
||||
timeout_in_minutes: 90
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ECCPUWorker.
|
||||
|
||||
The byte-level tests exercise real CUDA stream/event coordination
|
||||
against a real ``ECSharedRegion`` mmap and are skipped on hosts without
|
||||
CUDA. The lifecycle tests don't fire any ``torch.cuda.*`` primitives and
|
||||
run anywhere.
|
||||
The byte-level tests exercise real accelerator (CUDA/XPU/...) stream/event
|
||||
coordination against a real ``ECSharedRegion`` mmap and are skipped on hosts
|
||||
without an accelerator. The lifecycle tests don't fire any
|
||||
``current_platform.*`` device primitives and run anywhere.
|
||||
|
||||
Mocking policy
|
||||
--------------
|
||||
@@ -37,6 +37,7 @@ from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# ── shape constants ──────────────────────────────────────────────────────────
|
||||
|
||||
@@ -47,9 +48,11 @@ _DTYPE = torch.float16
|
||||
_BLOCK_SIZE_BYTES = _HIDDEN_DIM * _DTYPE.itemsize
|
||||
_NUM_BLOCKS = 8
|
||||
|
||||
_requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="exercises real CUDA stream/event coordination in ECCPUWorker",
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
_requires_accelerator = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu()),
|
||||
reason="exercises real accelerator stream/event coordination in ECCPUWorker",
|
||||
)
|
||||
|
||||
|
||||
@@ -136,7 +139,7 @@ def make_worker():
|
||||
# ── save_caches ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
@pytest.mark.parametrize(
|
||||
"n_elements,n_blocks",
|
||||
[
|
||||
@@ -158,7 +161,7 @@ def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks
|
||||
sentinel = 0x5A
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
src = torch.arange(n_elements, dtype=_DTYPE, device="cuda")
|
||||
src = torch.arange(n_elements, dtype=_DTYPE, device=DEVICE_TYPE)
|
||||
expected_bytes = src.cpu().reshape(-1).view(torch.uint8)
|
||||
total_bytes = n_elements * _DTYPE.itemsize
|
||||
|
||||
@@ -228,15 +231,15 @@ def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_save_caches_batches_multiple_hashes(make_worker):
|
||||
"""Multiple save_caches calls are batched into a single flush."""
|
||||
worker = make_worker()
|
||||
sentinel = 0x5A
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
|
||||
src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
|
||||
|
||||
cache = {"a": src_a, "b": src_b}
|
||||
worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]}))
|
||||
@@ -258,7 +261,7 @@ def test_save_caches_batches_multiple_hashes(make_worker):
|
||||
# ── start_load_caches ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
|
||||
"""Single batched load across all hashes with correct byte→dtype→shape."""
|
||||
worker = make_worker()
|
||||
@@ -276,20 +279,22 @@ def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
|
||||
|
||||
out = encoder_cache["h"]
|
||||
assert out.is_cuda, "consumer worker must place the tensor on the GPU"
|
||||
assert out.device.type == DEVICE_TYPE, (
|
||||
"consumer worker must place the tensor on the accelerator"
|
||||
)
|
||||
assert out.shape == (n_blocks, _HIDDEN_DIM)
|
||||
assert out.dtype == _DTYPE
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
|
||||
"""If ``encoder_cache`` already holds the ``mm_hash``, the worker must
|
||||
not overwrite it."""
|
||||
worker = make_worker()
|
||||
worker._region.blocks[0].fill_(0x42)
|
||||
|
||||
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device="cuda")
|
||||
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device=DEVICE_TYPE)
|
||||
encoder_cache = {"h": sentinel}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]}))
|
||||
|
||||
@@ -298,7 +303,7 @@ def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
|
||||
)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_start_load_caches_noop_when_loads_is_empty(make_worker):
|
||||
"""When ``meta.loads`` is empty the early-return must fire."""
|
||||
worker = make_worker()
|
||||
@@ -308,7 +313,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
|
||||
assert encoder_cache == {}
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
"""Cached entries are preserved while new ones are loaded."""
|
||||
worker = make_worker()
|
||||
@@ -321,7 +326,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
for i, idx in enumerate(new_block_ids):
|
||||
worker._region.blocks[idx].copy_(src_int8[i])
|
||||
|
||||
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device="cuda")
|
||||
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device=DEVICE_TYPE)
|
||||
encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor}
|
||||
worker.start_load_caches(
|
||||
encoder_cache,
|
||||
@@ -336,7 +341,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
@pytest.mark.parametrize(
|
||||
"tp_rank,pcp_rank",
|
||||
[(0, 0), (1, 0), (0, 1), (1, 1)],
|
||||
@@ -358,23 +363,23 @@ def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
|
||||
|
||||
out = encoder_cache["h"]
|
||||
assert out.is_cuda
|
||||
assert out.device.type == DEVICE_TYPE
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
# ── round-trip ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_save_then_load_round_trips_bytes(make_worker):
|
||||
"""Full producer→mmap→consumer byte path in one shot."""
|
||||
worker = make_worker()
|
||||
n_blocks = 3
|
||||
block_ids = [5, 1, 6]
|
||||
|
||||
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
src = torch.arange(
|
||||
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
|
||||
).reshape(n_blocks, _HIDDEN_DIM)
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
|
||||
worker.flush_saves()
|
||||
|
||||
@@ -390,12 +395,12 @@ def test_save_then_load_round_trips_bytes(make_worker):
|
||||
# ── buffer recycling ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_buffer_pool_is_reused_across_save_steps(make_worker):
|
||||
"""After flush_saves, descriptor buffers are returned to the pool and
|
||||
reused on the next flush — no reallocation."""
|
||||
worker = make_worker()
|
||||
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE)
|
||||
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]}))
|
||||
worker.flush_saves()
|
||||
@@ -411,7 +416,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
|
||||
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_buffer_pool_is_reused_across_load_steps(make_worker):
|
||||
"""After start_load_caches, descriptor buffers are returned to the pool
|
||||
and reused on the next call."""
|
||||
@@ -435,11 +440,11 @@ def test_buffer_pool_is_reused_across_load_steps(make_worker):
|
||||
# ── stream management ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_stream_initialized_at_construction(make_worker):
|
||||
"""``_load_stream`` must be a fully initialized CUDA stream."""
|
||||
"""``_load_stream`` must be a fully initialized accelerator stream."""
|
||||
worker = make_worker()
|
||||
assert isinstance(worker._load_stream, torch.cuda.Stream)
|
||||
assert isinstance(worker._load_stream, current_platform.Stream)
|
||||
|
||||
|
||||
# ── lifecycle ────────────────────────────────────────────────────────────────
|
||||
@@ -518,14 +523,14 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
|
||||
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@_requires_accelerator
|
||||
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
"""Full pipeline: scheduler allocates blocks, worker saves GPU tensor to
|
||||
mmap via flush_saves, scheduler marks ready after step delay, worker
|
||||
loads from mmap back to GPU, and the result matches the original.
|
||||
|
||||
Exercises the real scheduler + worker cooperation through a shared
|
||||
ECSharedRegion, with real CUDA transfers and stream coordination.
|
||||
ECSharedRegion, with real accelerator transfers and stream coordination.
|
||||
"""
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
|
||||
@@ -551,9 +556,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
|
||||
# -- Step 1: scheduler allocates, worker saves --
|
||||
n_blocks = 3
|
||||
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
src = torch.arange(
|
||||
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
|
||||
).reshape(n_blocks, _HIDDEN_DIM)
|
||||
|
||||
class _Pos:
|
||||
offset = 0
|
||||
@@ -590,7 +595,7 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
worker.start_load_caches(load_cache, meta_load)
|
||||
|
||||
out = load_cache["img_001"]
|
||||
assert out.is_cuda
|
||||
assert out.device.type == DEVICE_TYPE
|
||||
assert out.shape == src.shape
|
||||
assert out.dtype == src.dtype
|
||||
assert torch.equal(out.cpu(), src.cpu())
|
||||
|
||||
@@ -7,7 +7,7 @@ Verifies:
|
||||
- Accuracy: outputs from EC CPU cache match fresh encoder computation.
|
||||
- Latency: loading from EC CPU cache is faster than a cold encoder run.
|
||||
|
||||
Requires a CUDA GPU and the Qwen2-VL-2B-Instruct model.
|
||||
Requires a CUDA or XPU GPU and the Qwen2-VL-2B-Instruct model.
|
||||
"""
|
||||
|
||||
import time
|
||||
@@ -57,8 +57,8 @@ def _wait_for_ec_ready(llm: LLM) -> None:
|
||||
|
||||
def _latency_test(llm: LLM) -> None:
|
||||
"""Verify EC CPU cache hit is faster than cold encoder computation."""
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Latency test requires CUDA")
|
||||
if not (current_platform.is_cuda() or current_platform.is_xpu()):
|
||||
pytest.skip("Latency test requires an accelerator (CUDA or XPU)")
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=1, temperature=0)
|
||||
base_image = ImageAsset("stop_sign").pil_image
|
||||
@@ -134,7 +134,10 @@ def _accuracy_test(llm: LLM) -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu()),
|
||||
reason="Requires an accelerator (CUDA or XPU)",
|
||||
)
|
||||
def test_ec_cpu_offloading() -> None:
|
||||
"""Tests ECCPUConnector accuracy and latency with a VLM model."""
|
||||
ec_transfer_config = ECTransferConfig(
|
||||
|
||||
@@ -102,16 +102,15 @@ class ECCPUWorker:
|
||||
|
||||
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
|
||||
|
||||
src_ptrs, dst_ptrs, sizes = self._save_bufs
|
||||
bufs = self._save_bufs
|
||||
src_base = src.view(-1).view(torch.uint8).data_ptr()
|
||||
dst_base = self._region.blocks.data_ptr()
|
||||
idx = self._save_count
|
||||
|
||||
for i, block_idx in enumerate(block_ids):
|
||||
start = i * block_size
|
||||
src_ptrs[idx] = src_base + start
|
||||
dst_ptrs[idx] = dst_base + block_idx * block_size
|
||||
sizes[idx] = min(block_size, total_bytes - start)
|
||||
bufs.set_ptrs(idx, src_base + start, dst_base + block_idx * block_size)
|
||||
bufs.sizes[idx] = min(block_size, total_bytes - start)
|
||||
idx += 1
|
||||
|
||||
self._save_count = idx
|
||||
@@ -123,7 +122,7 @@ class ECCPUWorker:
|
||||
|
||||
bufs = self._save_bufs
|
||||
assert bufs is not None
|
||||
src_ptrs, dst_ptrs, sizes = bufs
|
||||
src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes
|
||||
n = self._save_count
|
||||
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
|
||||
|
||||
@@ -173,8 +172,11 @@ class ECCPUWorker:
|
||||
op_idx = 0
|
||||
for block_ids in load_items.values():
|
||||
for block_idx in block_ids:
|
||||
src_ptrs[op_idx] = src_base + block_idx * block_size
|
||||
dst_ptrs[op_idx] = dst_buf_base + op_idx * block_size
|
||||
bufs.set_ptrs(
|
||||
op_idx,
|
||||
src_base + block_idx * block_size,
|
||||
dst_buf_base + op_idx * block_size,
|
||||
)
|
||||
op_idx += 1
|
||||
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) int64 tensor triples.
|
||||
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) tensor triples.
|
||||
|
||||
Used by ECCPUWorker to batch swap_blocks_batch descriptors without
|
||||
per-step allocation overhead.
|
||||
@@ -8,20 +8,46 @@ per-step allocation overhead.
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# CUDA/ROCm cache_kernels.cu requires int64 pointers; the XPU DMA engine
|
||||
# requires uint64 (see vllm._custom_ops.swap_blocks_batch).
|
||||
_PTR_DTYPE = torch.uint64 if current_platform.is_xpu() else torch.int64
|
||||
|
||||
|
||||
class DescriptorBuffers(NamedTuple):
|
||||
src_ptrs: torch.Tensor
|
||||
dst_ptrs: torch.Tensor
|
||||
sizes: torch.Tensor
|
||||
# Numpy aliases of src_ptrs/dst_ptrs, written via set_ptrs().
|
||||
src_np: np.ndarray
|
||||
dst_np: np.ndarray
|
||||
|
||||
def set_ptrs(self, idx: int, src: int, dst: int) -> None:
|
||||
"""Record the source and destination address of descriptor *idx*.
|
||||
|
||||
TODO(torch>=2.14): drop this indirection and assign the tensors
|
||||
directly once the minimum supported torch is 2.14. The numpy detour
|
||||
exists only because torch's setitem unpacks the value as a signed
|
||||
long long before 2.14 (pytorch#191458), rejecting XPU USM pointers
|
||||
>= 2**63, while the two's-complement rewrite those versions accept is
|
||||
in turn rejected by uint64 from 2.14 on. Numpy casts against the
|
||||
array dtype and so works on either. `sizes` holds byte counts and
|
||||
needs no such care.
|
||||
"""
|
||||
self.src_np[idx] = src
|
||||
self.dst_np[idx] = dst
|
||||
|
||||
|
||||
class DescriptorBufferPool:
|
||||
"""Pool of descriptor buffer triples for swap_blocks_batch.
|
||||
|
||||
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D int64
|
||||
tensors of equal length. Buffers are recycled across steps; if a
|
||||
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D tensors
|
||||
(dtype `_PTR_DTYPE`, platform-dependent) of equal length, paired with
|
||||
numpy aliases used to fill them. Buffers are recycled across steps; if a
|
||||
returned buffer is too small it is discarded and a fresh one allocated.
|
||||
"""
|
||||
|
||||
@@ -35,11 +61,8 @@ class DescriptorBufferPool:
|
||||
bufs = self._pool.pop()
|
||||
if bufs.src_ptrs.numel() >= n:
|
||||
return bufs
|
||||
return DescriptorBuffers(
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
)
|
||||
src, dst, sizes = (torch.empty(n, dtype=_PTR_DTYPE) for _ in range(3))
|
||||
return DescriptorBuffers(src, dst, sizes, src.numpy(), dst.numpy())
|
||||
|
||||
def release(self, bufs: DescriptorBuffers) -> None:
|
||||
"""Return a buffer triple to the pool for reuse."""
|
||||
|
||||
Reference in New Issue
Block a user