[XPU] Support cpu kv offloading and tiering offloading on XPU platform (#36423)

Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
Chaojun Zhang
2026-06-07 09:59:28 +08:00
committed by GitHub
parent 2a983c79ac
commit 810966453a
7 changed files with 86 additions and 32 deletions
@@ -30,6 +30,7 @@ BLOCK_SIZE = 16
NUM_KV_HEADS = 4
HEAD_SIZE = 64
DTYPE = torch.float16
DEVICE_TYPE = current_platform.device_type
# Attention backends to test
ATTN_BACKENDS: list[str] = []
@@ -42,6 +43,8 @@ if current_platform.is_cuda():
]
elif current_platform.is_rocm():
ATTN_BACKENDS = ["TRITON_ATTN"]
elif current_platform.is_xpu():
ATTN_BACKENDS = ["TRITON_ATTN", "FLASH_ATTN"]
# ---------------------------------------------------------------------------
# Helpers
@@ -270,7 +273,7 @@ def test_register_kv_caches(backend):
kv_caches = _allocate_and_reshape_kv_caches(
kv_cache_config,
attn_groups,
device=torch.device("cuda:0"),
device=torch.device(f"{DEVICE_TYPE}:0"),
)
worker, spec = _make_worker(kv_cache_config)
@@ -413,7 +416,7 @@ def test_register_kv_caches_uniform_type(backend):
kv_caches = _allocate_and_reshape_kv_caches(
kv_cache_config,
attn_groups,
device=torch.device("cuda:0"),
device=torch.device(f"{DEVICE_TYPE}:0"),
)
worker, spec = _make_worker(kv_cache_config)
@@ -14,11 +14,14 @@ from vllm.config import KVEventsConfig, KVTransferConfig
from vllm.distributed.kv_events import BlockStored, KVEventBatch
from vllm.platforms import current_platform
CPU_BLOCK_SIZES: int = 64 if current_platform.is_xpu() else 48
_ATTN_BACKENDS: list[str] = []
if current_platform.is_cuda():
_ATTN_BACKENDS = ["FLASH_ATTN", "FLASHINFER", "TRITON_ATTN"]
elif current_platform.is_rocm():
_ATTN_BACKENDS = ["TRITON_ATTN"]
elif current_platform.is_xpu():
_ATTN_BACKENDS = ["FLASH_ATTN", "TRITON_ATTN"]
# (model, attn_backend | None, block_size | None, uses_hma)
#
@@ -30,14 +33,14 @@ elif current_platform.is_rocm():
# After page-size unification the mamba and attention groups have
# different block sizes.
MODEL_PARAMS: list[tuple[str, str | None, int | None, bool]] = [
("meta-llama/Llama-3.2-1B-Instruct", backend, 48, False)
("meta-llama/Llama-3.2-1B-Instruct", backend, CPU_BLOCK_SIZES, False)
for backend in _ATTN_BACKENDS
]
# HMA / Mamba models are only tested on CUDA (not ROCm).
if current_platform.is_cuda():
MODEL_PARAMS += [
("google/gemma-3-1b-it", None, 48, True),
("state-spaces/mamba-130m-hf", None, 48, True),
("google/gemma-3-1b-it", None, CPU_BLOCK_SIZES, True),
("state-spaces/mamba-130m-hf", None, CPU_BLOCK_SIZES, True),
# Falcon-H1: parallel hybrid (every layer has both attention and SSM).
# The mamba and attention groups end up with different GPU block sizes
# after page-size unification, so we leave cpu_block_size=None
@@ -304,7 +307,7 @@ def test_tiering_offloading() -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec."""
extra_config: dict = {
"cpu_bytes_to_use": 500 << 20,
"block_size": 48,
"block_size": CPU_BLOCK_SIZES,
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [{"type": "example"}],
}
@@ -350,7 +353,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
+ fs secondary tier."""
extra_config: dict = {
"cpu_bytes_to_use": 1 << 30,
"block_size": 48,
"block_size": CPU_BLOCK_SIZES,
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [{"type": "fs", "root_dir": str(tmp_path)}],
}