[XPU] Support cpu kv offloading and tiering offloading on XPU platform (#36423)

Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
Chaojun Zhang
2026-06-07 09:59:28 +08:00
committed by GitHub
parent 2a983c79ac
commit 810966453a
7 changed files with 86 additions and 32 deletions
+23
View File
@@ -53,3 +53,26 @@ steps:
pytest -v -s v1/test_oracle.py &&
pytest -v -s v1/test_request.py &&
pytest -v -s v1/test_outputs.py'
- label: XPU CPU Offload
timeout_in_minutes: 60
device: intel_gpu
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- vllm/v1/kv_offload/
- vllm/v1/kv_connector/
- tests/v1/kv_offload/
- tests/v1/kv_connector/unit/test_offloading_connector.py
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
cd tests &&
pytest -v -s v1/kv_offload &&
pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
@@ -30,6 +30,7 @@ BLOCK_SIZE = 16
NUM_KV_HEADS = 4
HEAD_SIZE = 64
DTYPE = torch.float16
DEVICE_TYPE = current_platform.device_type
# Attention backends to test
ATTN_BACKENDS: list[str] = []
@@ -42,6 +43,8 @@ if current_platform.is_cuda():
]
elif current_platform.is_rocm():
ATTN_BACKENDS = ["TRITON_ATTN"]
elif current_platform.is_xpu():
ATTN_BACKENDS = ["TRITON_ATTN", "FLASH_ATTN"]
# ---------------------------------------------------------------------------
# Helpers
@@ -270,7 +273,7 @@ def test_register_kv_caches(backend):
kv_caches = _allocate_and_reshape_kv_caches(
kv_cache_config,
attn_groups,
device=torch.device("cuda:0"),
device=torch.device(f"{DEVICE_TYPE}:0"),
)
worker, spec = _make_worker(kv_cache_config)
@@ -413,7 +416,7 @@ def test_register_kv_caches_uniform_type(backend):
kv_caches = _allocate_and_reshape_kv_caches(
kv_cache_config,
attn_groups,
device=torch.device("cuda:0"),
device=torch.device(f"{DEVICE_TYPE}:0"),
)
worker, spec = _make_worker(kv_cache_config)
@@ -14,11 +14,14 @@ from vllm.config import KVEventsConfig, KVTransferConfig
from vllm.distributed.kv_events import BlockStored, KVEventBatch
from vllm.platforms import current_platform
CPU_BLOCK_SIZES: int = 64 if current_platform.is_xpu() else 48
_ATTN_BACKENDS: list[str] = []
if current_platform.is_cuda():
_ATTN_BACKENDS = ["FLASH_ATTN", "FLASHINFER", "TRITON_ATTN"]
elif current_platform.is_rocm():
_ATTN_BACKENDS = ["TRITON_ATTN"]
elif current_platform.is_xpu():
_ATTN_BACKENDS = ["FLASH_ATTN", "TRITON_ATTN"]
# (model, attn_backend | None, block_size | None, uses_hma)
#
@@ -30,14 +33,14 @@ elif current_platform.is_rocm():
# After page-size unification the mamba and attention groups have
# different block sizes.
MODEL_PARAMS: list[tuple[str, str | None, int | None, bool]] = [
("meta-llama/Llama-3.2-1B-Instruct", backend, 48, False)
("meta-llama/Llama-3.2-1B-Instruct", backend, CPU_BLOCK_SIZES, False)
for backend in _ATTN_BACKENDS
]
# HMA / Mamba models are only tested on CUDA (not ROCm).
if current_platform.is_cuda():
MODEL_PARAMS += [
("google/gemma-3-1b-it", None, 48, True),
("state-spaces/mamba-130m-hf", None, 48, True),
("google/gemma-3-1b-it", None, CPU_BLOCK_SIZES, True),
("state-spaces/mamba-130m-hf", None, CPU_BLOCK_SIZES, True),
# Falcon-H1: parallel hybrid (every layer has both attention and SSM).
# The mamba and attention groups end up with different GPU block sizes
# after page-size unification, so we leave cpu_block_size=None
@@ -304,7 +307,7 @@ def test_tiering_offloading() -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec."""
extra_config: dict = {
"cpu_bytes_to_use": 500 << 20,
"block_size": 48,
"block_size": CPU_BLOCK_SIZES,
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [{"type": "example"}],
}
@@ -350,7 +353,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
+ fs secondary tier."""
extra_config: dict = {
"cpu_bytes_to_use": 1 << 30,
"block_size": 48,
"block_size": CPU_BLOCK_SIZES,
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [{"type": "fs", "root_dir": str(tmp_path)}],
}
+9 -4
View File
@@ -2812,7 +2812,9 @@ def swap_blocks_batch(
Batch version of swap_blocks: submit all copies in a single driver call.
Each entry specifies a raw pointer copy: src_ptrs[i] -> dst_ptrs[i]
of sizes[i] bytes. All three tensors must be int64 CPU tensors.
of sizes[i] bytes. All three tensors must be CPU tensors with the
platform-appropriate pointer dtype: int64 on CUDA/ROCm (required by
cache_kernels.cu) and uint64 on XPU (required by the XPU DMA engine).
On CUDA 12.8+ this uses cuMemcpyBatchAsync for minimal submission
overhead; on older CUDA it falls back to a loop of cudaMemcpyAsync.
@@ -2822,9 +2824,12 @@ def swap_blocks_batch(
writing to the source. Defaults to False (STREAM ordering), which
is always safe.
"""
torch.ops._C_cache_ops.swap_blocks_batch(
src_ptrs, dst_ptrs, sizes, is_src_access_order_any
)
if current_platform.is_xpu():
torch.ops._C_cache_ops.swap_blocks_batch(src_ptrs, dst_ptrs, sizes)
else:
torch.ops._C_cache_ops.swap_blocks_batch(
src_ptrs, dst_ptrs, sizes, is_src_access_order_any
)
def convert_fp8(
+28 -13
View File
@@ -11,6 +11,7 @@ from typing_extensions import override
from vllm import _custom_ops as ops
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import is_pin_memory_available
@@ -43,8 +44,10 @@ def _select_swap_blocks_fn(
if gpu_to_cpu:
return ops.swap_blocks_batch
# Fall back to the C++ DMA path on platforms where Triton isn't usable
# (e.g. ROCm builds without Triton).
if not HAS_TRITON:
# (e.g. ROCm builds without Triton) or where GPU kernels cannot directly
# dereference CPU pointers (XPU lacks CUDA's unified virtual address space,
# so the Triton kernel's tl.load(cpu_ptr) is invalid on XPU).
if not HAS_TRITON or current_platform.is_xpu():
return ops.swap_blocks_batch
page_sizes = [r.page_size_bytes for g in kv_cache_groups_data_refs for r in g]
# Triton wins only on small, 8-byte-aligned payloads.
@@ -92,7 +95,7 @@ def compute_sub_block_ptrs(
Args:
block_ids: array of block IDs at the tensor's native granularity.
block_size_factor: number of sub-blocks per block.
output: pre-allocated int64 array to write pointers into.
output: pre-allocated pointer array to write pointers into.
tensor: the source or destination tensor.
skip_count: sub-blocks to skip in the first block.
"""
@@ -104,16 +107,16 @@ def compute_sub_block_ptrs(
if block_size_factor == 1:
# Fast path: 1:1 mapping, no sub-block expansion needed.
output[:] = base_ptr + block_ids[:num_sub_blocks] * row_stride
output[:] = base_ptr + block_ids.astype(np.uint64)[:num_sub_blocks] * row_stride
return
# Vectorized expansion for block_size_factor > 1.
assert tensor.shape[1] % block_size_factor == 0
sub_block_size = tensor.shape[1] // block_size_factor
sub_offsets = np.arange(block_size_factor, dtype=np.int64) * sub_block_size
sub_offsets = np.arange(block_size_factor, dtype=np.uint64) * sub_block_size
# (num_blocks, 1) + (1, block_size_factor) -> (num_blocks, block_size_factor)
all_ptrs = (
base_ptr + block_ids.astype(np.int64)[:, np.newaxis] * row_stride
base_ptr + block_ids.astype(np.uint64)[:, np.newaxis] * row_stride
) + sub_offsets[np.newaxis, :]
# Flatten and apply skip_count / truncation
flat = all_ptrs.ravel()
@@ -122,6 +125,14 @@ def compute_sub_block_ptrs(
def pin_mmap_region(region: SharedOffloadRegion) -> None:
"""Register the entire mmap as CUDA pinned memory via cudaHostRegister."""
if not current_platform.is_cuda_alike():
logger.info(
"Skipping mmap host registration on %s; cudaHostRegister is only "
"available on CUDA/ROCm.",
current_platform.device_name,
)
return
rank = region.rank
base_ptr = region._base.data_ptr()
@@ -146,10 +157,12 @@ def _new_descriptor_buffers(
num_copy_ops: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
pin = is_pin_memory_available()
# CUDA cache_kernels.cu requires int64; XPU DMA engine requires uint64.
ptr_dtype = torch.uint64 if current_platform.is_xpu() else torch.int64
return (
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin),
torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin),
torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin),
)
@@ -190,7 +203,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
for gpu_tensor, cpu_tensor in zip(gpu_tensors, cpu_tensors):
assert gpu_tensor.dtype == torch.int8
assert gpu_tensor.ndim == 2
assert gpu_tensor.is_cuda
assert gpu_tensor.is_cuda or gpu_tensor.is_xpu
assert cpu_tensor.dtype == torch.int8
assert cpu_tensor.ndim == 2
assert cpu_tensor.device.type == "cpu"
@@ -355,7 +368,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
assert dst_offset == num_dst_blocks
assert op_idx == num_copy_ops
stream = self._stream_pool.pop() if self._stream_pool else torch.cuda.Stream()
stream = (
self._stream_pool.pop() if self._stream_pool else current_platform.Stream()
)
start_event = (
self._event_pool.pop()
if self._event_pool
@@ -369,7 +384,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
if self.gpu_to_cpu:
# wait for model computation to finish before offloading
stream.wait_stream(torch.cuda.current_stream())
stream.wait_stream(current_platform.current_stream())
if self._transfers:
last_transfer: Transfer = self._transfers[-1]
last_event = last_transfer.end_event
@@ -382,7 +397,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
# writing; we must keep STREAM ordering so source reads are gated
# by the transfer stream's wait_stream(compute) barrier.
is_src_access_order_any = not self.gpu_to_cpu
with torch.cuda.stream(stream):
with current_platform.stream(stream):
start_event.record(stream)
if num_copy_ops > 0:
self._swap_blocks_batch(
@@ -7,6 +7,7 @@ import time
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -171,12 +172,15 @@ class SharedOffloadRegion:
def cleanup(self) -> None:
if self.is_pinned and self._base is not None:
base_ptr = self._base.data_ptr()
result = torch.cuda.cudart().cudaHostUnregister(base_ptr)
if result.value != 0:
logger.warning(
"cudaHostUnregister failed for rank=%d (code=%d)", self.rank, result
)
if current_platform.is_cuda_alike():
base_ptr = self._base.data_ptr()
result = torch.cuda.cudart().cudaHostUnregister(base_ptr)
if result.value != 0:
logger.warning(
"cudaHostUnregister failed for rank=%d (code=%d)",
self.rank,
result,
)
self.is_pinned = False
# Release views before _base: each view holds a _base reference and a
# direct StorageImpl reference. Freeing views first lets both refcounts
+3 -2
View File
@@ -107,9 +107,10 @@ class CPUOffloadingSpec(OffloadingSpec):
self, kv_caches: CanonicalKVCaches
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
if not self._handlers:
if not current_platform.is_cuda_alike():
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
raise Exception(
"CPU Offloading is currently only supported on CUDA-alike GPUs"
"CPU Offloading is currently only supported on CUDA-alike "
"and XPU GPUs"
)
self._handlers = self.create_handlers(kv_caches)