[XPU] Support cpu kv offloading and tiering offloading on XPU platform (#36423)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
@@ -53,3 +53,26 @@ steps:
|
||||
pytest -v -s v1/test_oracle.py &&
|
||||
pytest -v -s v1/test_request.py &&
|
||||
pytest -v -s v1/test_outputs.py'
|
||||
|
||||
- label: XPU CPU Offload
|
||||
timeout_in_minutes: 60
|
||||
device: intel_gpu
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- vllm/v1/kv_offload/
|
||||
- vllm/v1/kv_connector/
|
||||
- tests/v1/kv_offload/
|
||||
- tests/v1/kv_connector/unit/test_offloading_connector.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
cd tests &&
|
||||
pytest -v -s v1/kv_offload &&
|
||||
pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
|
||||
|
||||
@@ -30,6 +30,7 @@ BLOCK_SIZE = 16
|
||||
NUM_KV_HEADS = 4
|
||||
HEAD_SIZE = 64
|
||||
DTYPE = torch.float16
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
# Attention backends to test
|
||||
ATTN_BACKENDS: list[str] = []
|
||||
@@ -42,6 +43,8 @@ if current_platform.is_cuda():
|
||||
]
|
||||
elif current_platform.is_rocm():
|
||||
ATTN_BACKENDS = ["TRITON_ATTN"]
|
||||
elif current_platform.is_xpu():
|
||||
ATTN_BACKENDS = ["TRITON_ATTN", "FLASH_ATTN"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -270,7 +273,7 @@ def test_register_kv_caches(backend):
|
||||
kv_caches = _allocate_and_reshape_kv_caches(
|
||||
kv_cache_config,
|
||||
attn_groups,
|
||||
device=torch.device("cuda:0"),
|
||||
device=torch.device(f"{DEVICE_TYPE}:0"),
|
||||
)
|
||||
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
@@ -413,7 +416,7 @@ def test_register_kv_caches_uniform_type(backend):
|
||||
kv_caches = _allocate_and_reshape_kv_caches(
|
||||
kv_cache_config,
|
||||
attn_groups,
|
||||
device=torch.device("cuda:0"),
|
||||
device=torch.device(f"{DEVICE_TYPE}:0"),
|
||||
)
|
||||
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
|
||||
@@ -14,11 +14,14 @@ from vllm.config import KVEventsConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_events import BlockStored, KVEventBatch
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
CPU_BLOCK_SIZES: int = 64 if current_platform.is_xpu() else 48
|
||||
_ATTN_BACKENDS: list[str] = []
|
||||
if current_platform.is_cuda():
|
||||
_ATTN_BACKENDS = ["FLASH_ATTN", "FLASHINFER", "TRITON_ATTN"]
|
||||
elif current_platform.is_rocm():
|
||||
_ATTN_BACKENDS = ["TRITON_ATTN"]
|
||||
elif current_platform.is_xpu():
|
||||
_ATTN_BACKENDS = ["FLASH_ATTN", "TRITON_ATTN"]
|
||||
|
||||
# (model, attn_backend | None, block_size | None, uses_hma)
|
||||
#
|
||||
@@ -30,14 +33,14 @@ elif current_platform.is_rocm():
|
||||
# After page-size unification the mamba and attention groups have
|
||||
# different block sizes.
|
||||
MODEL_PARAMS: list[tuple[str, str | None, int | None, bool]] = [
|
||||
("meta-llama/Llama-3.2-1B-Instruct", backend, 48, False)
|
||||
("meta-llama/Llama-3.2-1B-Instruct", backend, CPU_BLOCK_SIZES, False)
|
||||
for backend in _ATTN_BACKENDS
|
||||
]
|
||||
# HMA / Mamba models are only tested on CUDA (not ROCm).
|
||||
if current_platform.is_cuda():
|
||||
MODEL_PARAMS += [
|
||||
("google/gemma-3-1b-it", None, 48, True),
|
||||
("state-spaces/mamba-130m-hf", None, 48, True),
|
||||
("google/gemma-3-1b-it", None, CPU_BLOCK_SIZES, True),
|
||||
("state-spaces/mamba-130m-hf", None, CPU_BLOCK_SIZES, True),
|
||||
# Falcon-H1: parallel hybrid (every layer has both attention and SSM).
|
||||
# The mamba and attention groups end up with different GPU block sizes
|
||||
# after page-size unification, so we leave cpu_block_size=None
|
||||
@@ -304,7 +307,7 @@ def test_tiering_offloading() -> None:
|
||||
"""Tests OffloadingConnector with TieringOffloadingSpec."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 500 << 20,
|
||||
"block_size": 48,
|
||||
"block_size": CPU_BLOCK_SIZES,
|
||||
"spec_name": "TieringOffloadingSpec",
|
||||
"secondary_tiers": [{"type": "example"}],
|
||||
}
|
||||
@@ -350,7 +353,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
+ fs secondary tier."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 1 << 30,
|
||||
"block_size": 48,
|
||||
"block_size": CPU_BLOCK_SIZES,
|
||||
"spec_name": "TieringOffloadingSpec",
|
||||
"secondary_tiers": [{"type": "fs", "root_dir": str(tmp_path)}],
|
||||
}
|
||||
|
||||
+9
-4
@@ -2812,7 +2812,9 @@ def swap_blocks_batch(
|
||||
Batch version of swap_blocks: submit all copies in a single driver call.
|
||||
|
||||
Each entry specifies a raw pointer copy: src_ptrs[i] -> dst_ptrs[i]
|
||||
of sizes[i] bytes. All three tensors must be int64 CPU tensors.
|
||||
of sizes[i] bytes. All three tensors must be CPU tensors with the
|
||||
platform-appropriate pointer dtype: int64 on CUDA/ROCm (required by
|
||||
cache_kernels.cu) and uint64 on XPU (required by the XPU DMA engine).
|
||||
On CUDA 12.8+ this uses cuMemcpyBatchAsync for minimal submission
|
||||
overhead; on older CUDA it falls back to a loop of cudaMemcpyAsync.
|
||||
|
||||
@@ -2822,9 +2824,12 @@ def swap_blocks_batch(
|
||||
writing to the source. Defaults to False (STREAM ordering), which
|
||||
is always safe.
|
||||
"""
|
||||
torch.ops._C_cache_ops.swap_blocks_batch(
|
||||
src_ptrs, dst_ptrs, sizes, is_src_access_order_any
|
||||
)
|
||||
if current_platform.is_xpu():
|
||||
torch.ops._C_cache_ops.swap_blocks_batch(src_ptrs, dst_ptrs, sizes)
|
||||
else:
|
||||
torch.ops._C_cache_ops.swap_blocks_batch(
|
||||
src_ptrs, dst_ptrs, sizes, is_src_access_order_any
|
||||
)
|
||||
|
||||
|
||||
def convert_fp8(
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing_extensions import override
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
@@ -43,8 +44,10 @@ def _select_swap_blocks_fn(
|
||||
if gpu_to_cpu:
|
||||
return ops.swap_blocks_batch
|
||||
# Fall back to the C++ DMA path on platforms where Triton isn't usable
|
||||
# (e.g. ROCm builds without Triton).
|
||||
if not HAS_TRITON:
|
||||
# (e.g. ROCm builds without Triton) or where GPU kernels cannot directly
|
||||
# dereference CPU pointers (XPU lacks CUDA's unified virtual address space,
|
||||
# so the Triton kernel's tl.load(cpu_ptr) is invalid on XPU).
|
||||
if not HAS_TRITON or current_platform.is_xpu():
|
||||
return ops.swap_blocks_batch
|
||||
page_sizes = [r.page_size_bytes for g in kv_cache_groups_data_refs for r in g]
|
||||
# Triton wins only on small, 8-byte-aligned payloads.
|
||||
@@ -92,7 +95,7 @@ def compute_sub_block_ptrs(
|
||||
Args:
|
||||
block_ids: array of block IDs at the tensor's native granularity.
|
||||
block_size_factor: number of sub-blocks per block.
|
||||
output: pre-allocated int64 array to write pointers into.
|
||||
output: pre-allocated pointer array to write pointers into.
|
||||
tensor: the source or destination tensor.
|
||||
skip_count: sub-blocks to skip in the first block.
|
||||
"""
|
||||
@@ -104,16 +107,16 @@ def compute_sub_block_ptrs(
|
||||
|
||||
if block_size_factor == 1:
|
||||
# Fast path: 1:1 mapping, no sub-block expansion needed.
|
||||
output[:] = base_ptr + block_ids[:num_sub_blocks] * row_stride
|
||||
output[:] = base_ptr + block_ids.astype(np.uint64)[:num_sub_blocks] * row_stride
|
||||
return
|
||||
|
||||
# Vectorized expansion for block_size_factor > 1.
|
||||
assert tensor.shape[1] % block_size_factor == 0
|
||||
sub_block_size = tensor.shape[1] // block_size_factor
|
||||
sub_offsets = np.arange(block_size_factor, dtype=np.int64) * sub_block_size
|
||||
sub_offsets = np.arange(block_size_factor, dtype=np.uint64) * sub_block_size
|
||||
# (num_blocks, 1) + (1, block_size_factor) -> (num_blocks, block_size_factor)
|
||||
all_ptrs = (
|
||||
base_ptr + block_ids.astype(np.int64)[:, np.newaxis] * row_stride
|
||||
base_ptr + block_ids.astype(np.uint64)[:, np.newaxis] * row_stride
|
||||
) + sub_offsets[np.newaxis, :]
|
||||
# Flatten and apply skip_count / truncation
|
||||
flat = all_ptrs.ravel()
|
||||
@@ -122,6 +125,14 @@ def compute_sub_block_ptrs(
|
||||
|
||||
def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
"""Register the entire mmap as CUDA pinned memory via cudaHostRegister."""
|
||||
if not current_platform.is_cuda_alike():
|
||||
logger.info(
|
||||
"Skipping mmap host registration on %s; cudaHostRegister is only "
|
||||
"available on CUDA/ROCm.",
|
||||
current_platform.device_name,
|
||||
)
|
||||
return
|
||||
|
||||
rank = region.rank
|
||||
|
||||
base_ptr = region._base.data_ptr()
|
||||
@@ -146,10 +157,12 @@ def _new_descriptor_buffers(
|
||||
num_copy_ops: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
pin = is_pin_memory_available()
|
||||
# CUDA cache_kernels.cu requires int64; XPU DMA engine requires uint64.
|
||||
ptr_dtype = torch.uint64 if current_platform.is_xpu() else torch.int64
|
||||
return (
|
||||
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
|
||||
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
|
||||
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
|
||||
torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin),
|
||||
torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin),
|
||||
torch.empty(num_copy_ops, dtype=ptr_dtype, pin_memory=pin),
|
||||
)
|
||||
|
||||
|
||||
@@ -190,7 +203,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
for gpu_tensor, cpu_tensor in zip(gpu_tensors, cpu_tensors):
|
||||
assert gpu_tensor.dtype == torch.int8
|
||||
assert gpu_tensor.ndim == 2
|
||||
assert gpu_tensor.is_cuda
|
||||
assert gpu_tensor.is_cuda or gpu_tensor.is_xpu
|
||||
assert cpu_tensor.dtype == torch.int8
|
||||
assert cpu_tensor.ndim == 2
|
||||
assert cpu_tensor.device.type == "cpu"
|
||||
@@ -355,7 +368,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
assert dst_offset == num_dst_blocks
|
||||
assert op_idx == num_copy_ops
|
||||
|
||||
stream = self._stream_pool.pop() if self._stream_pool else torch.cuda.Stream()
|
||||
stream = (
|
||||
self._stream_pool.pop() if self._stream_pool else current_platform.Stream()
|
||||
)
|
||||
start_event = (
|
||||
self._event_pool.pop()
|
||||
if self._event_pool
|
||||
@@ -369,7 +384,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
|
||||
if self.gpu_to_cpu:
|
||||
# wait for model computation to finish before offloading
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
stream.wait_stream(current_platform.current_stream())
|
||||
if self._transfers:
|
||||
last_transfer: Transfer = self._transfers[-1]
|
||||
last_event = last_transfer.end_event
|
||||
@@ -382,7 +397,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
# writing; we must keep STREAM ordering so source reads are gated
|
||||
# by the transfer stream's wait_stream(compute) barrier.
|
||||
is_src_access_order_any = not self.gpu_to_cpu
|
||||
with torch.cuda.stream(stream):
|
||||
with current_platform.stream(stream):
|
||||
start_event.record(stream)
|
||||
if num_copy_ops > 0:
|
||||
self._swap_blocks_batch(
|
||||
|
||||
@@ -7,6 +7,7 @@ import time
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -171,12 +172,15 @@ class SharedOffloadRegion:
|
||||
|
||||
def cleanup(self) -> None:
|
||||
if self.is_pinned and self._base is not None:
|
||||
base_ptr = self._base.data_ptr()
|
||||
result = torch.cuda.cudart().cudaHostUnregister(base_ptr)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"cudaHostUnregister failed for rank=%d (code=%d)", self.rank, result
|
||||
)
|
||||
if current_platform.is_cuda_alike():
|
||||
base_ptr = self._base.data_ptr()
|
||||
result = torch.cuda.cudart().cudaHostUnregister(base_ptr)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"cudaHostUnregister failed for rank=%d (code=%d)",
|
||||
self.rank,
|
||||
result,
|
||||
)
|
||||
self.is_pinned = False
|
||||
# Release views before _base: each view holds a _base reference and a
|
||||
# direct StorageImpl reference. Freeing views first lets both refcounts
|
||||
|
||||
@@ -107,9 +107,10 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
if not self._handlers:
|
||||
if not current_platform.is_cuda_alike():
|
||||
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
|
||||
raise Exception(
|
||||
"CPU Offloading is currently only supported on CUDA-alike GPUs"
|
||||
"CPU Offloading is currently only supported on CUDA-alike "
|
||||
"and XPU GPUs"
|
||||
)
|
||||
self._handlers = self.create_handlers(kv_caches)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user