[XPU] Support cpu kv offloading and tiering offloading on XPU platform (#36423)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
@@ -30,6 +30,7 @@ BLOCK_SIZE = 16
|
||||
NUM_KV_HEADS = 4
|
||||
HEAD_SIZE = 64
|
||||
DTYPE = torch.float16
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
# Attention backends to test
|
||||
ATTN_BACKENDS: list[str] = []
|
||||
@@ -42,6 +43,8 @@ if current_platform.is_cuda():
|
||||
]
|
||||
elif current_platform.is_rocm():
|
||||
ATTN_BACKENDS = ["TRITON_ATTN"]
|
||||
elif current_platform.is_xpu():
|
||||
ATTN_BACKENDS = ["TRITON_ATTN", "FLASH_ATTN"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
@@ -270,7 +273,7 @@ def test_register_kv_caches(backend):
|
||||
kv_caches = _allocate_and_reshape_kv_caches(
|
||||
kv_cache_config,
|
||||
attn_groups,
|
||||
device=torch.device("cuda:0"),
|
||||
device=torch.device(f"{DEVICE_TYPE}:0"),
|
||||
)
|
||||
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
@@ -413,7 +416,7 @@ def test_register_kv_caches_uniform_type(backend):
|
||||
kv_caches = _allocate_and_reshape_kv_caches(
|
||||
kv_cache_config,
|
||||
attn_groups,
|
||||
device=torch.device("cuda:0"),
|
||||
device=torch.device(f"{DEVICE_TYPE}:0"),
|
||||
)
|
||||
|
||||
worker, spec = _make_worker(kv_cache_config)
|
||||
|
||||
@@ -14,11 +14,14 @@ from vllm.config import KVEventsConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_events import BlockStored, KVEventBatch
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
CPU_BLOCK_SIZES: int = 64 if current_platform.is_xpu() else 48
|
||||
_ATTN_BACKENDS: list[str] = []
|
||||
if current_platform.is_cuda():
|
||||
_ATTN_BACKENDS = ["FLASH_ATTN", "FLASHINFER", "TRITON_ATTN"]
|
||||
elif current_platform.is_rocm():
|
||||
_ATTN_BACKENDS = ["TRITON_ATTN"]
|
||||
elif current_platform.is_xpu():
|
||||
_ATTN_BACKENDS = ["FLASH_ATTN", "TRITON_ATTN"]
|
||||
|
||||
# (model, attn_backend | None, block_size | None, uses_hma)
|
||||
#
|
||||
@@ -30,14 +33,14 @@ elif current_platform.is_rocm():
|
||||
# After page-size unification the mamba and attention groups have
|
||||
# different block sizes.
|
||||
MODEL_PARAMS: list[tuple[str, str | None, int | None, bool]] = [
|
||||
("meta-llama/Llama-3.2-1B-Instruct", backend, 48, False)
|
||||
("meta-llama/Llama-3.2-1B-Instruct", backend, CPU_BLOCK_SIZES, False)
|
||||
for backend in _ATTN_BACKENDS
|
||||
]
|
||||
# HMA / Mamba models are only tested on CUDA (not ROCm).
|
||||
if current_platform.is_cuda():
|
||||
MODEL_PARAMS += [
|
||||
("google/gemma-3-1b-it", None, 48, True),
|
||||
("state-spaces/mamba-130m-hf", None, 48, True),
|
||||
("google/gemma-3-1b-it", None, CPU_BLOCK_SIZES, True),
|
||||
("state-spaces/mamba-130m-hf", None, CPU_BLOCK_SIZES, True),
|
||||
# Falcon-H1: parallel hybrid (every layer has both attention and SSM).
|
||||
# The mamba and attention groups end up with different GPU block sizes
|
||||
# after page-size unification, so we leave cpu_block_size=None
|
||||
@@ -304,7 +307,7 @@ def test_tiering_offloading() -> None:
|
||||
"""Tests OffloadingConnector with TieringOffloadingSpec."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 500 << 20,
|
||||
"block_size": 48,
|
||||
"block_size": CPU_BLOCK_SIZES,
|
||||
"spec_name": "TieringOffloadingSpec",
|
||||
"secondary_tiers": [{"type": "example"}],
|
||||
}
|
||||
@@ -350,7 +353,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
+ fs secondary tier."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 1 << 30,
|
||||
"block_size": 48,
|
||||
"block_size": CPU_BLOCK_SIZES,
|
||||
"spec_name": "TieringOffloadingSpec",
|
||||
"secondary_tiers": [{"type": "fs", "root_dir": str(tmp_path)}],
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user