diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml index 679e626b3e..897ae7d228 100644 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ b/.buildkite/intel_jobs/misc_intel.yaml @@ -8,7 +8,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 16+ + mem: 24+ no_plugin: true working_dir: "." env: @@ -22,6 +22,7 @@ steps: - tests/v1/kv_offload - tests/v1/worker - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit - tests/v1/metrics - tests/entrypoints/openai/correctness/test_lmeval.py commands: @@ -30,7 +31,8 @@ steps: 'pip install -r requirements/kv_connectors.txt && export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && - pytest -v -s v1/executor' + pytest -v -s v1/executor && + VLLM_BATCH_INVARIANT=1 pytest -v -s -m 'not cpu_test' v1/ec_connector/unit' - label: V1 Sample + Logits timeout_in_minutes: 90 diff --git a/tests/v1/ec_connector/unit/cpu/worker/test_worker.py b/tests/v1/ec_connector/unit/cpu/worker/test_worker.py index 4668abf251..3daf4ee7de 100644 --- a/tests/v1/ec_connector/unit/cpu/worker/test_worker.py +++ b/tests/v1/ec_connector/unit/cpu/worker/test_worker.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for ECCPUWorker. -The byte-level tests exercise real CUDA stream/event coordination -against a real ``ECSharedRegion`` mmap and are skipped on hosts without -CUDA. The lifecycle tests don't fire any ``torch.cuda.*`` primitives and -run anywhere. +The byte-level tests exercise real accelerator (CUDA/XPU/...) stream/event +coordination against a real ``ECSharedRegion`` mmap and are skipped on hosts +without an accelerator. The lifecycle tests don't fire any +``current_platform.*`` device primitives and run anywhere. Mocking policy -------------- @@ -37,6 +37,7 @@ from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( ECSharedRegion, ) from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker +from vllm.platforms import current_platform # ── shape constants ────────────────────────────────────────────────────────── @@ -47,9 +48,11 @@ _DTYPE = torch.float16 _BLOCK_SIZE_BYTES = _HIDDEN_DIM * _DTYPE.itemsize _NUM_BLOCKS = 8 -_requires_cuda = pytest.mark.skipif( - not torch.cuda.is_available(), - reason="exercises real CUDA stream/event coordination in ECCPUWorker", +DEVICE_TYPE = current_platform.device_type + +_requires_accelerator = pytest.mark.skipif( + not (current_platform.is_cuda() or current_platform.is_xpu()), + reason="exercises real accelerator stream/event coordination in ECCPUWorker", ) @@ -136,7 +139,7 @@ def make_worker(): # ── save_caches ────────────────────────────────────────────────────────────── -@_requires_cuda +@_requires_accelerator @pytest.mark.parametrize( "n_elements,n_blocks", [ @@ -158,7 +161,7 @@ def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks sentinel = 0x5A worker._region.blocks.fill_(sentinel) - src = torch.arange(n_elements, dtype=_DTYPE, device="cuda") + src = torch.arange(n_elements, dtype=_DTYPE, device=DEVICE_TYPE) expected_bytes = src.cpu().reshape(-1).view(torch.uint8) total_bytes = n_elements * _DTYPE.itemsize @@ -228,15 +231,15 @@ def test_save_caches_raises_when_allocated_blocks_too_small(make_worker): worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]})) -@_requires_cuda +@_requires_accelerator def test_save_caches_batches_multiple_hashes(make_worker): """Multiple save_caches calls are batched into a single flush.""" worker = make_worker() sentinel = 0x5A worker._region.blocks.fill_(sentinel) - src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda") - src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device="cuda") + src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE) + src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE) cache = {"a": src_a, "b": src_b} worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]})) @@ -258,7 +261,7 @@ def test_save_caches_batches_multiple_hashes(make_worker): # ── start_load_caches ──────────────────────────────────────────────────────── -@_requires_cuda +@_requires_accelerator def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker): """Single batched load across all hashes with correct byte→dtype→shape.""" worker = make_worker() @@ -276,20 +279,22 @@ def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids})) out = encoder_cache["h"] - assert out.is_cuda, "consumer worker must place the tensor on the GPU" + assert out.device.type == DEVICE_TYPE, ( + "consumer worker must place the tensor on the accelerator" + ) assert out.shape == (n_blocks, _HIDDEN_DIM) assert out.dtype == _DTYPE assert torch.equal(out.cpu(), src_orig) -@_requires_cuda +@_requires_accelerator def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker): """If ``encoder_cache`` already holds the ``mm_hash``, the worker must not overwrite it.""" worker = make_worker() worker._region.blocks[0].fill_(0x42) - sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device="cuda") + sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device=DEVICE_TYPE) encoder_cache = {"h": sentinel} worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]})) @@ -298,7 +303,7 @@ def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker): ) -@_requires_cuda +@_requires_accelerator def test_start_load_caches_noop_when_loads_is_empty(make_worker): """When ``meta.loads`` is empty the early-return must fire.""" worker = make_worker() @@ -308,7 +313,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker): assert encoder_cache == {} -@_requires_cuda +@_requires_accelerator def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker): """Cached entries are preserved while new ones are loaded.""" worker = make_worker() @@ -321,7 +326,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker): for i, idx in enumerate(new_block_ids): worker._region.blocks[idx].copy_(src_int8[i]) - cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device="cuda") + cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device=DEVICE_TYPE) encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor} worker.start_load_caches( encoder_cache, @@ -336,7 +341,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker): assert torch.equal(out.cpu(), src_orig) -@_requires_cuda +@_requires_accelerator @pytest.mark.parametrize( "tp_rank,pcp_rank", [(0, 0), (1, 0), (0, 1), (1, 1)], @@ -358,23 +363,23 @@ def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank): worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids})) out = encoder_cache["h"] - assert out.is_cuda + assert out.device.type == DEVICE_TYPE assert torch.equal(out.cpu(), src_orig) # ── round-trip ─────────────────────────────────────────────────────────────── -@_requires_cuda +@_requires_accelerator def test_save_then_load_round_trips_bytes(make_worker): """Full producer→mmap→consumer byte path in one shot.""" worker = make_worker() n_blocks = 3 block_ids = [5, 1, 6] - src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape( - n_blocks, _HIDDEN_DIM - ) + src = torch.arange( + n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE + ).reshape(n_blocks, _HIDDEN_DIM) worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids})) worker.flush_saves() @@ -390,12 +395,12 @@ def test_save_then_load_round_trips_bytes(make_worker): # ── buffer recycling ──────────────────────────────────────────────────────── -@_requires_cuda +@_requires_accelerator def test_buffer_pool_is_reused_across_save_steps(make_worker): """After flush_saves, descriptor buffers are returned to the pool and reused on the next flush — no reallocation.""" worker = make_worker() - src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda") + src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE) worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]})) worker.flush_saves() @@ -411,7 +416,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker): assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id -@_requires_cuda +@_requires_accelerator def test_buffer_pool_is_reused_across_load_steps(make_worker): """After start_load_caches, descriptor buffers are returned to the pool and reused on the next call.""" @@ -435,11 +440,11 @@ def test_buffer_pool_is_reused_across_load_steps(make_worker): # ── stream management ──────────────────────────────────────────────────────── -@_requires_cuda +@_requires_accelerator def test_stream_initialized_at_construction(make_worker): - """``_load_stream`` must be a fully initialized CUDA stream.""" + """``_load_stream`` must be a fully initialized accelerator stream.""" worker = make_worker() - assert isinstance(worker._load_stream, torch.cuda.Stream) + assert isinstance(worker._load_stream, current_platform.Stream) # ── lifecycle ──────────────────────────────────────────────────────────────── @@ -518,14 +523,14 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm): # ── e2e: scheduler + worker pipeline ──────────────────────────────────────── -@_requires_cuda +@_requires_accelerator def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch): """Full pipeline: scheduler allocates blocks, worker saves GPU tensor to mmap via flush_saves, scheduler marks ready after step delay, worker loads from mmap back to GPU, and the result matches the original. Exercises the real scheduler + worker cooperation through a shared - ECSharedRegion, with real CUDA transfers and stream coordination. + ECSharedRegion, with real accelerator transfers and stream coordination. """ import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ( @@ -551,9 +556,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch): # -- Step 1: scheduler allocates, worker saves -- n_blocks = 3 - src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape( - n_blocks, _HIDDEN_DIM - ) + src = torch.arange( + n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE + ).reshape(n_blocks, _HIDDEN_DIM) class _Pos: offset = 0 @@ -590,7 +595,7 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch): worker.start_load_caches(load_cache, meta_load) out = load_cache["img_001"] - assert out.is_cuda + assert out.device.type == DEVICE_TYPE assert out.shape == src.shape assert out.dtype == src.dtype assert torch.equal(out.cpu(), src.cpu()) diff --git a/tests/v1/ec_connector/unit/test_ec_cpu_connector.py b/tests/v1/ec_connector/unit/test_ec_cpu_connector.py index 3a2600860a..6a1ccf8571 100644 --- a/tests/v1/ec_connector/unit/test_ec_cpu_connector.py +++ b/tests/v1/ec_connector/unit/test_ec_cpu_connector.py @@ -7,7 +7,7 @@ Verifies: - Accuracy: outputs from EC CPU cache match fresh encoder computation. - Latency: loading from EC CPU cache is faster than a cold encoder run. -Requires a CUDA GPU and the Qwen2-VL-2B-Instruct model. +Requires a CUDA or XPU GPU and the Qwen2-VL-2B-Instruct model. """ import time @@ -57,8 +57,8 @@ def _wait_for_ec_ready(llm: LLM) -> None: def _latency_test(llm: LLM) -> None: """Verify EC CPU cache hit is faster than cold encoder computation.""" - if not current_platform.is_cuda(): - pytest.skip("Latency test requires CUDA") + if not (current_platform.is_cuda() or current_platform.is_xpu()): + pytest.skip("Latency test requires an accelerator (CUDA or XPU)") sampling_params = SamplingParams(max_tokens=1, temperature=0) base_image = ImageAsset("stop_sign").pil_image @@ -134,7 +134,10 @@ def _accuracy_test(llm: LLM) -> None: ) -@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +@pytest.mark.skipif( + not (current_platform.is_cuda() or current_platform.is_xpu()), + reason="Requires an accelerator (CUDA or XPU)", +) def test_ec_cpu_offloading() -> None: """Tests ECCPUConnector accuracy and latency with a VLM model.""" ec_transfer_config = ECTransferConfig( diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py index b923a2218d..5a957c4966 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py @@ -102,16 +102,15 @@ class ECCPUWorker: assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel() - src_ptrs, dst_ptrs, sizes = self._save_bufs + bufs = self._save_bufs src_base = src.view(-1).view(torch.uint8).data_ptr() dst_base = self._region.blocks.data_ptr() idx = self._save_count for i, block_idx in enumerate(block_ids): start = i * block_size - src_ptrs[idx] = src_base + start - dst_ptrs[idx] = dst_base + block_idx * block_size - sizes[idx] = min(block_size, total_bytes - start) + bufs.set_ptrs(idx, src_base + start, dst_base + block_idx * block_size) + bufs.sizes[idx] = min(block_size, total_bytes - start) idx += 1 self._save_count = idx @@ -123,7 +122,7 @@ class ECCPUWorker: bufs = self._save_bufs assert bufs is not None - src_ptrs, dst_ptrs, sizes = bufs + src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes n = self._save_count swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n]) @@ -173,8 +172,11 @@ class ECCPUWorker: op_idx = 0 for block_ids in load_items.values(): for block_idx in block_ids: - src_ptrs[op_idx] = src_base + block_idx * block_size - dst_ptrs[op_idx] = dst_buf_base + op_idx * block_size + bufs.set_ptrs( + op_idx, + src_base + block_idx * block_size, + dst_buf_base + op_idx * block_size, + ) op_idx += 1 swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True) diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/descriptor_buffers.py b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/descriptor_buffers.py index a75d99ee81..a42d6ebda4 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/descriptor_buffers.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/descriptor_buffers.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Reusable pool of (src_ptrs, dst_ptrs, sizes) int64 tensor triples. +"""Reusable pool of (src_ptrs, dst_ptrs, sizes) tensor triples. Used by ECCPUWorker to batch swap_blocks_batch descriptors without per-step allocation overhead. @@ -8,20 +8,46 @@ per-step allocation overhead. from typing import NamedTuple +import numpy as np import torch +from vllm.platforms import current_platform + +# CUDA/ROCm cache_kernels.cu requires int64 pointers; the XPU DMA engine +# requires uint64 (see vllm._custom_ops.swap_blocks_batch). +_PTR_DTYPE = torch.uint64 if current_platform.is_xpu() else torch.int64 + class DescriptorBuffers(NamedTuple): src_ptrs: torch.Tensor dst_ptrs: torch.Tensor sizes: torch.Tensor + # Numpy aliases of src_ptrs/dst_ptrs, written via set_ptrs(). + src_np: np.ndarray + dst_np: np.ndarray + + def set_ptrs(self, idx: int, src: int, dst: int) -> None: + """Record the source and destination address of descriptor *idx*. + + TODO(torch>=2.14): drop this indirection and assign the tensors + directly once the minimum supported torch is 2.14. The numpy detour + exists only because torch's setitem unpacks the value as a signed + long long before 2.14 (pytorch#191458), rejecting XPU USM pointers + >= 2**63, while the two's-complement rewrite those versions accept is + in turn rejected by uint64 from 2.14 on. Numpy casts against the + array dtype and so works on either. `sizes` holds byte counts and + needs no such care. + """ + self.src_np[idx] = src + self.dst_np[idx] = dst class DescriptorBufferPool: """Pool of descriptor buffer triples for swap_blocks_batch. - Each buffer is a `DescriptorBuffers` namedtuple of three 1-D int64 - tensors of equal length. Buffers are recycled across steps; if a + Each buffer is a `DescriptorBuffers` namedtuple of three 1-D tensors + (dtype `_PTR_DTYPE`, platform-dependent) of equal length, paired with + numpy aliases used to fill them. Buffers are recycled across steps; if a returned buffer is too small it is discarded and a fresh one allocated. """ @@ -35,11 +61,8 @@ class DescriptorBufferPool: bufs = self._pool.pop() if bufs.src_ptrs.numel() >= n: return bufs - return DescriptorBuffers( - torch.empty(n, dtype=torch.int64), - torch.empty(n, dtype=torch.int64), - torch.empty(n, dtype=torch.int64), - ) + src, dst, sizes = (torch.empty(n, dtype=_PTR_DTYPE) for _ in range(3)) + return DescriptorBuffers(src, dst, sizes, src.numpy(), dst.numpy()) def release(self, bufs: DescriptorBuffers) -> None: """Return a buffer triple to the pool for reuse."""