diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml index 864128bb53..8ff4a8e6f0 100644 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ b/.buildkite/intel_jobs/misc_intel.yaml @@ -53,3 +53,26 @@ steps: pytest -v -s v1/test_oracle.py && pytest -v -s v1/test_request.py && pytest -v -s v1/test_outputs.py' + +- label: XPU CPU Offload + timeout_in_minutes: 60 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - vllm/v1/kv_offload/ + - vllm/v1/kv_connector/ + - tests/v1/kv_offload/ + - tests/v1/kv_connector/unit/test_offloading_connector.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && + cd tests && + pytest -v -s v1/kv_offload && + pytest -v -s v1/kv_connector/unit/test_offloading_connector.py' diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py index 36294632bb..833d4fe0a4 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_worker.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_worker.py @@ -30,6 +30,7 @@ BLOCK_SIZE = 16 NUM_KV_HEADS = 4 HEAD_SIZE = 64 DTYPE = torch.float16 +DEVICE_TYPE = current_platform.device_type # Attention backends to test ATTN_BACKENDS: list[str] = [] @@ -42,6 +43,8 @@ if current_platform.is_cuda(): ] elif current_platform.is_rocm(): ATTN_BACKENDS = ["TRITON_ATTN"] +elif current_platform.is_xpu(): + ATTN_BACKENDS = ["TRITON_ATTN", "FLASH_ATTN"] # --------------------------------------------------------------------------- # Helpers @@ -270,7 +273,7 @@ def test_register_kv_caches(backend): kv_caches = _allocate_and_reshape_kv_caches( kv_cache_config, attn_groups, - device=torch.device("cuda:0"), + device=torch.device(f"{DEVICE_TYPE}:0"), ) worker, spec = _make_worker(kv_cache_config) @@ -413,7 +416,7 @@ def test_register_kv_caches_uniform_type(backend): kv_caches = _allocate_and_reshape_kv_caches( kv_cache_config, attn_groups, - device=torch.device("cuda:0"), + device=torch.device(f"{DEVICE_TYPE}:0"), ) worker, spec = _make_worker(kv_cache_config) diff --git a/tests/v1/kv_connector/unit/test_offloading_connector.py b/tests/v1/kv_connector/unit/test_offloading_connector.py index d123cc520a..e3faa2cf58 100644 --- a/tests/v1/kv_connector/unit/test_offloading_connector.py +++ b/tests/v1/kv_connector/unit/test_offloading_connector.py @@ -14,11 +14,14 @@ from vllm.config import KVEventsConfig, KVTransferConfig from vllm.distributed.kv_events import BlockStored, KVEventBatch from vllm.platforms import current_platform +CPU_BLOCK_SIZES: int = 64 if current_platform.is_xpu() else 48 _ATTN_BACKENDS: list[str] = [] if current_platform.is_cuda(): _ATTN_BACKENDS = ["FLASH_ATTN", "FLASHINFER", "TRITON_ATTN"] elif current_platform.is_rocm(): _ATTN_BACKENDS = ["TRITON_ATTN"] +elif current_platform.is_xpu(): + _ATTN_BACKENDS = ["FLASH_ATTN", "TRITON_ATTN"] # (model, attn_backend | None, block_size | None, uses_hma) # @@ -30,14 +33,14 @@ elif current_platform.is_rocm(): # After page-size unification the mamba and attention groups have # different block sizes. MODEL_PARAMS: list[tuple[str, str | None, int | None, bool]] = [ - ("meta-llama/Llama-3.2-1B-Instruct", backend, 48, False) + ("meta-llama/Llama-3.2-1B-Instruct", backend, CPU_BLOCK_SIZES, False) for backend in _ATTN_BACKENDS ] # HMA / Mamba models are only tested on CUDA (not ROCm). if current_platform.is_cuda(): MODEL_PARAMS += [ - ("google/gemma-3-1b-it", None, 48, True), - ("state-spaces/mamba-130m-hf", None, 48, True), + ("google/gemma-3-1b-it", None, CPU_BLOCK_SIZES, True), + ("state-spaces/mamba-130m-hf", None, CPU_BLOCK_SIZES, True), # Falcon-H1: parallel hybrid (every layer has both attention and SSM). # The mamba and attention groups end up with different GPU block sizes # after page-size unification, so we leave cpu_block_size=None @@ -304,7 +307,7 @@ def test_tiering_offloading() -> None: """Tests OffloadingConnector with TieringOffloadingSpec.""" extra_config: dict = { "cpu_bytes_to_use": 500 << 20, - "block_size": 48, + "block_size": CPU_BLOCK_SIZES, "spec_name": "TieringOffloadingSpec", "secondary_tiers": [{"type": "example"}], } @@ -350,7 +353,7 @@ def test_fs_tiering_offloading(tmp_path) -> None: + fs secondary tier.""" extra_config: dict = { "cpu_bytes_to_use": 1 << 30, - "block_size": 48, + "block_size": CPU_BLOCK_SIZES, "spec_name": "TieringOffloadingSpec", "secondary_tiers": [{"type": "fs", "root_dir": str(tmp_path)}], } diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 7eaa10b141..2c0684c27f 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2812,7 +2812,9 @@ def swap_blocks_batch( Batch version of swap_blocks: submit all copies in a single driver call. Each entry specifies a raw pointer copy: src_ptrs[i] -> dst_ptrs[i] - of sizes[i] bytes. All three tensors must be int64 CPU tensors. + of sizes[i] bytes. All three tensors must be CPU tensors with the + platform-appropriate pointer dtype: int64 on CUDA/ROCm (required by + cache_kernels.cu) and uint64 on XPU (required by the XPU DMA engine). On CUDA 12.8+ this uses cuMemcpyBatchAsync for minimal submission overhead; on older CUDA it falls back to a loop of cudaMemcpyAsync. @@ -2822,9 +2824,12 @@ def swap_blocks_batch( writing to the source. Defaults to False (STREAM ordering), which is always safe. """ - torch.ops._C_cache_ops.swap_blocks_batch( - src_ptrs, dst_ptrs, sizes, is_src_access_order_any - ) + if current_platform.is_xpu(): + torch.ops._C_cache_ops.swap_blocks_batch(src_ptrs, dst_ptrs, sizes) + else: + torch.ops._C_cache_ops.swap_blocks_batch( + src_ptrs, dst_ptrs, sizes, is_src_access_order_any + ) def convert_fp8( diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index 4fbda71d9e..81545281b6 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -11,6 +11,7 @@ from typing_extensions import override from vllm import _custom_ops as ops from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON, triton from vllm.utils.math_utils import cdiv from vllm.utils.platform_utils import is_pin_memory_available @@ -43,8 +44,10 @@ def _select_swap_blocks_fn( if gpu_to_cpu: return ops.swap_blocks_batch # Fall back to the C++ DMA path on platforms where Triton isn't usable - # (e.g. ROCm builds without Triton). - if not HAS_TRITON: + # (e.g. ROCm builds without Triton) or where GPU kernels cannot directly + # dereference CPU pointers (XPU lacks CUDA's unified virtual address space, + # so the Triton kernel's tl.load(cpu_ptr) is invalid on XPU). + if not HAS_TRITON or current_platform.is_xpu(): return ops.swap_blocks_batch page_sizes = [r.page_size_bytes for g in kv_cache_groups_data_refs for r in g] # Triton wins only on small, 8-byte-aligned payloads. @@ -92,7 +95,7 @@ def compute_sub_block_ptrs( Args: block_ids: array of block IDs at the tensor's native granularity. block_size_factor: number of sub-blocks per block. - output: pre-allocated int64 array to write pointers into. + output: pre-allocated pointer array to write pointers into. tensor: the source or destination tensor. skip_count: sub-blocks to skip in the first block. """ @@ -104,16 +107,16 @@ def compute_sub_block_ptrs( if block_size_factor == 1: # Fast path: 1:1 mapping, no sub-block expansion needed. - output[:] = base_ptr + block_ids[:num_sub_blocks] * row_stride + output[:] = base_ptr + block_ids.astype(np.uint64)[:num_sub_blocks] * row_stride return # Vectorized expansion for block_size_factor > 1. assert tensor.shape[1] % block_size_factor == 0 sub_block_size = tensor.shape[1] // block_size_factor - sub_offsets = np.arange(block_size_factor, dtype=np.int64) * sub_block_size + sub_offsets = np.arange(block_size_factor, dtype=np.uint64) * sub_block_size # (num_blocks, 1) + (1, block_size_factor) -> (num_blocks, block_size_factor) all_ptrs = ( - base_ptr + block_ids.astype(np.int64)[:, np.newaxis] * row_stride + base_ptr + block_ids.astype(np.uint64)[:, np.newaxis] * row_stride ) + sub_offsets[np.newaxis, :] # Flatten and apply skip_count / truncation flat = all_ptrs.ravel() @@ -122,6 +125,14 @@ def compute_sub_block_ptrs( def pin_mmap_region(region: SharedOffloadRegion) -> None: """Register the entire mmap as CUDA pinned memory via cudaHostRegister.""" + if not current_platform.is_cuda_alike(): + logger.info( + "Skipping mmap host registration on %s; cudaHostRegister is only " + "available on CUDA/ROCm.", + current_platform.device_name, + ) + return + rank = region.rank base_ptr = region._base.data_ptr() @@ -146,10 +157,12 @@ def _new_descriptor_buffers( num_copy_ops: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: pin = is_pin_memory_available() + # CUDA cache_kernels.cu requires int64; XPU DMA engine requires uint64. + ptr_dtype = torch.uint64 if current_platform.is_xpu() else torch.int64 return ( - torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin), - torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin), - torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin), + torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin), + torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin), + torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin), ) @@ -190,7 +203,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): for gpu_tensor, cpu_tensor in zip(gpu_tensors, cpu_tensors): assert gpu_tensor.dtype == torch.int8 assert gpu_tensor.ndim == 2 - assert gpu_tensor.is_cuda + assert gpu_tensor.is_cuda or gpu_tensor.is_xpu assert cpu_tensor.dtype == torch.int8 assert cpu_tensor.ndim == 2 assert cpu_tensor.device.type == "cpu" @@ -355,7 +368,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): assert dst_offset == num_dst_blocks assert op_idx == num_copy_ops - stream = self._stream_pool.pop() if self._stream_pool else torch.cuda.Stream() + stream = ( + self._stream_pool.pop() if self._stream_pool else current_platform.Stream() + ) start_event = ( self._event_pool.pop() if self._event_pool @@ -369,7 +384,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): if self.gpu_to_cpu: # wait for model computation to finish before offloading - stream.wait_stream(torch.cuda.current_stream()) + stream.wait_stream(current_platform.current_stream()) if self._transfers: last_transfer: Transfer = self._transfers[-1] last_event = last_transfer.end_event @@ -382,7 +397,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): # writing; we must keep STREAM ordering so source reads are gated # by the transfer stream's wait_stream(compute) barrier. is_src_access_order_any = not self.gpu_to_cpu - with torch.cuda.stream(stream): + with current_platform.stream(stream): start_event.record(stream) if num_copy_ops > 0: self._swap_blocks_batch( diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index b9b415f12d..d5400e0ca7 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -7,6 +7,7 @@ import time import torch from vllm.logger import init_logger +from vllm.platforms import current_platform logger = init_logger(__name__) @@ -171,12 +172,15 @@ class SharedOffloadRegion: def cleanup(self) -> None: if self.is_pinned and self._base is not None: - base_ptr = self._base.data_ptr() - result = torch.cuda.cudart().cudaHostUnregister(base_ptr) - if result.value != 0: - logger.warning( - "cudaHostUnregister failed for rank=%d (code=%d)", self.rank, result - ) + if current_platform.is_cuda_alike(): + base_ptr = self._base.data_ptr() + result = torch.cuda.cudart().cudaHostUnregister(base_ptr) + if result.value != 0: + logger.warning( + "cudaHostUnregister failed for rank=%d (code=%d)", + self.rank, + result, + ) self.is_pinned = False # Release views before _base: each view holds a _base reference and a # direct StorageImpl reference. Freeing views first lets both refcounts diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 8791ff5d39..732891553a 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -107,9 +107,10 @@ class CPUOffloadingSpec(OffloadingSpec): self, kv_caches: CanonicalKVCaches ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]: if not self._handlers: - if not current_platform.is_cuda_alike(): + if not (current_platform.is_cuda_alike() or current_platform.is_xpu()): raise Exception( - "CPU Offloading is currently only supported on CUDA-alike GPUs" + "CPU Offloading is currently only supported on CUDA-alike " + "and XPU GPUs" ) self._handlers = self.create_handlers(kv_caches)