[XPU][Feature] transparent sleep mode support for XPU platform (#37149)

Signed-off-by: Yan Ma <yan.ma@intel.com>
This commit is contained in:
Yan Ma
2026-06-07 10:45:31 +08:00
committed by GitHub
parent 810966453a
commit 3bb46975bd
10 changed files with 410 additions and 47 deletions
@@ -0,0 +1,22 @@
group: Basic Correctness
depends_on:
- image-build-xpu
steps:
- label: XPU Sleep Mode
timeout_in_minutes: 30
device: intel_gpu
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- tests/basic_correctness/test_cumem.py
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
+2 -2
View File
@@ -908,10 +908,10 @@ steps:
- vllm/
- tests/basic_correctness/test_basic_correctness
- tests/basic_correctness/test_cpu_offload
- tests/basic_correctness/test_cumem.py
- tests/basic_correctness/test_mem.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/test_cumem.py
- pytest -v -s basic_correctness/test_mem.py
- pytest -v -s basic_correctness/test_basic_correctness.py
- pytest -v -s basic_correctness/test_cpu_offload.py
+2 -2
View File
@@ -10,9 +10,9 @@ steps:
- vllm/
- tests/basic_correctness/test_basic_correctness
- tests/basic_correctness/test_cpu_offload
- tests/basic_correctness/test_cumem.py
- tests/basic_correctness/test_mem.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/test_cumem.py
- pytest -v -s basic_correctness/test_mem.py
- pytest -v -s basic_correctness/test_basic_correctness.py
- pytest -v -s basic_correctness/test_cpu_offload.py
@@ -7,7 +7,7 @@ import pytest
import torch
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
from vllm.device_allocator.cumem import CuMemAllocator
from vllm.device_allocator import get_mem_allocator_instance
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
@@ -16,14 +16,14 @@ from ..utils import create_new_process_for_each_test, requires_fp8
DEVICE_TYPE = current_platform.device_type
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_python_error():
"""
Test if Python error occurs when there's low-level
error happening from the C++ side.
"""
allocator = CuMemAllocator.get_instance()
total_bytes = torch.cuda.mem_get_info()[1]
allocator = get_mem_allocator_instance()
total_bytes = current_platform.mem_get_info()[1]
alloc_bytes = int(total_bytes * 0.7)
tensors = []
with allocator.use_memory_pool():
@@ -42,7 +42,7 @@ def test_python_error():
allocator.wake_up()
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_basic_cumem():
# some tensors from default memory pool
shape = (1024, 1024)
@@ -50,7 +50,7 @@ def test_basic_cumem():
x.zero_()
# some tensors from custom memory pool
allocator = CuMemAllocator.get_instance()
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
# custom memory pool
y = torch.empty(shape, device=DEVICE_TYPE)
@@ -64,9 +64,9 @@ def test_basic_cumem():
output = x + y + z
assert torch.allclose(output, torch.ones_like(output) * 3)
free_bytes = torch.cuda.mem_get_info()[0]
free_bytes = current_platform.mem_get_info()[0]
allocator.sleep()
free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
free_bytes_after_sleep = current_platform.mem_get_info()[0]
assert free_bytes_after_sleep > free_bytes
allocator.wake_up()
@@ -75,9 +75,10 @@ def test_basic_cumem():
assert torch.allclose(output, torch.ones_like(output) * 3)
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
def test_cumem_with_cudagraph():
allocator = CuMemAllocator.get_instance()
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
weight = torch.eye(1024, device=DEVICE_TYPE)
with allocator.use_memory_pool(tag="discard"):
@@ -98,9 +99,9 @@ def test_cumem_with_cudagraph():
with torch.cuda.graph(model_graph):
y = model(x)
free_bytes = torch.cuda.mem_get_info()[0]
free_bytes = current_platform.mem_get_info()[0]
allocator.sleep()
free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
free_bytes_after_sleep = current_platform.mem_get_info()[0]
assert free_bytes_after_sleep > free_bytes
allocator.wake_up()
@@ -120,7 +121,7 @@ def test_cumem_with_cudagraph():
assert torch.allclose(y, x + 1)
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.parametrize(
"model",
[
@@ -131,7 +132,7 @@ def test_cumem_with_cudagraph():
],
)
def test_end_to_end(model: str):
free, total = torch.cuda.mem_get_info()
free, total = current_platform.mem_get_info()
used_bytes_baseline = total - free # in case other process is running
llm = LLM(model, enable_sleep_mode=True)
prompt = "How are you?"
@@ -143,7 +144,7 @@ def test_end_to_end(model: str):
# test sleep level 1 here.
llm.sleep(level=1)
free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
# now the memory usage is mostly cudagraph memory pool,
# and it should be less than the model weights (1B model, 2GiB weights)
@@ -163,7 +164,7 @@ def test_end_to_end(model: str):
llm.sleep(level=1)
llm.wake_up(tags=["weights"])
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
# should just reallocate memory for weights (1B model, ~2GiB weights)
@@ -180,7 +181,7 @@ def test_end_to_end(model: str):
@create_new_process_for_each_test()
def test_deep_sleep():
model = "hmellor/tiny-random-LlamaForCausalLM"
free, total = torch.cuda.mem_get_info()
free, total = current_platform.mem_get_info()
used_bytes_baseline = total - free # in case other process is running
llm = LLM(model, enable_sleep_mode=True)
prompt = "How are you?"
@@ -190,13 +191,13 @@ def test_deep_sleep():
# Put the engine to deep sleep
llm.sleep(level=2)
free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
assert used_bytes < 3 * GiB_bytes
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
assert used_bytes < 4 * GiB_bytes
@@ -212,7 +213,7 @@ def test_deep_sleep():
def test_deep_sleep_async():
async def test():
model = "hmellor/tiny-random-LlamaForCausalLM"
free, total = torch.cuda.mem_get_info()
free, total = current_platform.mem_get_info()
used_bytes_baseline = total - free # in case other process is running
engine_args = AsyncEngineArgs(
model=model,
@@ -231,7 +232,7 @@ def test_deep_sleep_async():
await llm.wake_up(tags=["weights"])
await llm.collective_rpc("reload_weights")
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
assert used_bytes < 4 * GiB_bytes
+1 -1
View File
@@ -528,7 +528,7 @@ class ModelConfig:
if self.enable_sleep_mode:
if not current_platform.is_sleep_mode_available():
raise ValueError("Sleep mode is not supported on current platform.")
if not self.enable_cumem_allocator:
if current_platform.is_cuda_alike() and not self.enable_cumem_allocator:
logger.info_once(
"Enabling cumem allocator because sleep mode requires it."
)
+48
View File
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from contextlib import AbstractContextManager
from typing import Protocol
import torch
from vllm.platforms import current_platform
# py_device, py_size_or_aligned_size, py_ptr, py_handle
HandleType = tuple[int, int, int, int]
@dataclasses.dataclass
class AllocationData:
handle: HandleType
tag: str
cpu_backup_tensor: torch.Tensor | None = None
class MemAllocator(Protocol):
def use_memory_pool(self, tag: str | None = None) -> AbstractContextManager: ...
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None: ...
def wake_up(self, tags: list[str] | None = None) -> None: ...
def get_current_usage(self) -> int: ...
def get_mem_allocator_instance() -> MemAllocator:
if current_platform.is_cuda_alike():
from vllm.device_allocator.cumem import CuMemAllocator
return CuMemAllocator.get_instance()
if current_platform.is_xpu():
from vllm.device_allocator.xpumem import XpuMemAllocator
return XpuMemAllocator.get_instance()
raise RuntimeError(
"Sleep mode allocator is not available on platform "
f"{type(current_platform).__name__} "
f"(device_type={current_platform.device_type})."
)
+1 -11
View File
@@ -8,7 +8,6 @@
# both of them failed because of cuda context mismatch.
# not sure why, they are created from a different context.
# the only successful approach is to call cuda driver API in C.
import dataclasses
import gc
import os
from collections.abc import Callable, Iterator
@@ -17,6 +16,7 @@ from typing import Any
import torch
from vllm.device_allocator import AllocationData, HandleType
from vllm.logger import init_logger
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.system_utils import find_loaded_library
@@ -44,16 +44,6 @@ except ModuleNotFoundError:
python_unmap_and_release = None
lib_name = None
# py_device, py_alignedSize, py_d_mem, py_p_memHandle
HandleType = tuple[int, int, int, int]
@dataclasses.dataclass
class AllocationData:
handle: HandleType
tag: str
cpu_backup_tensor: torch.Tensor | None = None
def create_and_map(allocation_handle: HandleType) -> None:
python_create_and_map(*allocation_handle)
+295
View File
@@ -0,0 +1,295 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import atexit
import gc
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import torch
from vllm.device_allocator import AllocationData, HandleType
from vllm.logger import init_logger
from vllm.utils.platform_utils import is_pin_memory_available
logger = init_logger(__name__)
MEMCPY_HOST_TO_DEVICE = 0
MEMCPY_DEVICE_TO_HOST = 1
MEMCPY_DEVICE_TO_DEVICE = 2
xpumem_available = False
xpumem_allocator: Any = None
try:
from vllm_xpu_kernels import xpumem_allocator as _xpumem_allocator
xpumem_allocator = _xpumem_allocator
xpumem_available = True
except ImportError:
xpumem_allocator = None
def _xpu_memory_module() -> Any:
mem_mod = getattr(torch.xpu, "memory", None)
if mem_mod is None:
raise RuntimeError("torch.xpu.memory is not available")
return mem_mod
def _supports_xpu_mem_pool(mem_mod: Any) -> bool:
return hasattr(mem_mod, "MemPool") and hasattr(mem_mod, "use_mem_pool")
def _xpu_memcpy_sync(
dst_ptr: int,
src_ptr: int,
n_bytes: int,
kind: int,
device: int,
) -> None:
def _to_i64_ptr(ptr: int) -> int:
# torch custom-op `int` arguments are signed int64.
# data_ptr() may return a uint64 value above 2^63-1, so normalize it.
return ptr if ptr < (1 << 63) else ptr - (1 << 64)
torch.ops._C.xpu_memcpy_sync(
_to_i64_ptr(dst_ptr),
_to_i64_ptr(src_ptr),
n_bytes,
kind,
device,
)
def get_pluggable_allocator(
python_malloc_fn: Callable[[HandleType], None],
python_free_func: Callable[[int], HandleType],
) -> Any:
if not xpumem_available or xpumem_allocator is None:
raise RuntimeError("xpumem allocator extension is not available")
xpumem_allocator.init_module(python_malloc_fn, python_free_func)
mem_mod = _xpu_memory_module()
alloc_cls = getattr(mem_mod, "XPUPluggableAllocator", None)
if alloc_cls is None:
raise RuntimeError("torch.xpu.memory.XPUPluggableAllocator is not available")
lib_name = xpumem_allocator.__file__
return alloc_cls(lib_name, "my_malloc", "my_free")
def create_and_allocate(allocation_handle: HandleType) -> None:
if not xpumem_available or xpumem_allocator is None:
raise RuntimeError("xpumem allocator extension is not available")
xpumem_allocator.python_create_and_allocate(*allocation_handle)
def unmap_and_release(allocation_handle: HandleType) -> None:
if not xpumem_available or xpumem_allocator is None:
raise RuntimeError("xpumem allocator extension is not available")
xpumem_allocator.python_unmap_and_release(*allocation_handle)
@contextmanager
def use_memory_pool_with_allocator(
python_malloc_fn: Callable[[HandleType], None],
python_free_func: Callable[[int], HandleType],
) -> Iterator[tuple[Any, Any]]:
mem_mod = _xpu_memory_module()
if not _supports_xpu_mem_pool(mem_mod):
raise RuntimeError(
"torch.xpu.memory MemPool APIs are not available "
"(need MemPool and use_mem_pool)."
)
new_alloc = get_pluggable_allocator(python_malloc_fn, python_free_func)
mem_pool = mem_mod.MemPool(new_alloc._allocator)
with mem_mod.use_mem_pool(mem_pool):
yield mem_pool, new_alloc
class XpuMemAllocator:
"""A singleton pluggable allocator helper for XPU.
Note:
Sleep will offload selected payloads to CPU or discard and unmap XPU
physical memory. Wake-up remaps physical memory back to the same
reserved virtual address and restores payload.
"""
instance: "XpuMemAllocator | None" = None
default_tag: str = "default"
@staticmethod
def get_instance() -> "XpuMemAllocator":
assert xpumem_available, "xpumem allocator is not available"
if XpuMemAllocator.instance is None:
XpuMemAllocator.instance = XpuMemAllocator()
# Ensure MemPool/allocator wrappers are released before interpreter
# finalization tears down XPU runtime internals.
atexit.register(XpuMemAllocator._shutdown_singleton)
return XpuMemAllocator.instance
@staticmethod
def _shutdown_singleton() -> None:
instance = XpuMemAllocator.instance
if instance is None:
return
try:
instance.release_pools()
except Exception:
logger.exception("XpuMemAllocator singleton shutdown failed")
def __init__(self):
self.pointer_to_data: dict[int, AllocationData] = {}
self.current_tag: str = XpuMemAllocator.default_tag
self.allocator_and_pools: dict[str, Any] = {}
self.python_malloc_callback = self._python_malloc_callback
self.python_free_callback = self._python_free_callback
def _python_malloc_callback(self, allocation_handle: HandleType) -> None:
ptr = allocation_handle[2]
self.pointer_to_data[ptr] = AllocationData(allocation_handle, self.current_tag)
logger.debug(
"Allocated %s bytes for %s at %s",
allocation_handle[1],
self.current_tag,
ptr,
)
def _python_free_callback(self, ptr: int) -> HandleType:
data = self.pointer_to_data.pop(ptr)
data.cpu_backup_tensor = None
logger.debug("Freed %s bytes for %s at %s", data.handle[1], data.tag, ptr)
return data.handle
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None:
if offload_tags is None:
offload_tags = (XpuMemAllocator.default_tag,)
elif isinstance(offload_tags, str):
offload_tags = (offload_tags,)
assert isinstance(offload_tags, tuple)
total_bytes = 0
backup_bytes = 0
for ptr, data in self.pointer_to_data.items():
size_in_bytes = data.handle[1]
total_bytes += size_in_bytes
if data.tag not in offload_tags:
unmap_and_release(data.handle)
continue
backup_bytes += size_in_bytes
device, _, _, _ = data.handle
cpu_backup_tensor = torch.empty(
size_in_bytes,
dtype=torch.uint8,
device="cpu",
pin_memory=is_pin_memory_available(),
)
cpu_ptr = cpu_backup_tensor.data_ptr()
_xpu_memcpy_sync(
cpu_ptr,
ptr,
size_in_bytes,
MEMCPY_DEVICE_TO_HOST,
device,
)
data.cpu_backup_tensor = cpu_backup_tensor
unmap_and_release(data.handle)
logger.info(
"XpuMemAllocator: sleep freed %.2f GiB memory in total, of which "
"%.2f GiB is backed up in CPU and the rest %.2f GiB is discarded "
"directly.",
total_bytes / 1024**3,
backup_bytes / 1024**3,
(total_bytes - backup_bytes) / 1024**3,
)
gc.collect()
xpu_empty_cache = getattr(torch.xpu, "empty_cache", None)
if callable(xpu_empty_cache):
xpu_empty_cache()
def wake_up(self, tags: list[str] | None = None) -> None:
for ptr, data in self.pointer_to_data.items():
if tags is not None and data.tag not in tags:
continue
create_and_allocate(data.handle)
cpu_backup_tensor = data.cpu_backup_tensor
if cpu_backup_tensor is None:
continue
device, size_in_bytes, _, _ = data.handle
_xpu_memcpy_sync(
ptr,
cpu_backup_tensor.data_ptr(),
size_in_bytes,
MEMCPY_HOST_TO_DEVICE,
device,
)
data.cpu_backup_tensor = None
def release_pools(self) -> None:
"""Drop Python references to MemPool/pluggable allocators eagerly.
This prevents pool destruction from being deferred to interpreter
finalization, which can happen after parts of XPU runtime are already
torn down.
"""
if not self.allocator_and_pools:
return
# Note: keep allocators alive while MemPool objects are destroyed.
# MemPool teardown may invoke allocator virtual methods (e.g. raw_delete)
# when releasing cached blocks. If allocator wrappers are dropped first,
# C++ can hit "pure virtual method called" during shutdown.
pool_entries = list(self.allocator_and_pools.values())
self.allocator_and_pools.clear()
mem_pools = [entry[0] for entry in pool_entries]
allocators = [entry[1] for entry in pool_entries]
pool_entries.clear()
xpu_sync = getattr(torch.xpu, "synchronize", None)
if callable(xpu_sync):
try:
xpu_sync()
except Exception:
logger.debug("torch.xpu.synchronize() failed during release_pools")
# Phase 1: drop MemPool refs while allocators are still strongly held.
mem_pools.clear()
gc.collect()
# Phase 2: now it is safe to release allocator wrappers.
allocators.clear()
@contextmanager
def use_memory_pool(self, tag: str | None = None):
if tag is None:
tag = XpuMemAllocator.default_tag
old_tag = self.current_tag
self.current_tag = tag
try:
with use_memory_pool_with_allocator(
self.python_malloc_callback,
self.python_free_callback,
) as data:
self.allocator_and_pools[tag] = data
yield
finally:
self.current_tag = old_tag
def get_current_usage(self) -> int:
total = 0
for data in self.pointer_to_data.values():
total += data.handle[1]
return total
+1 -1
View File
@@ -197,7 +197,7 @@ class Platform:
# for ROCm, but currently we don't have a way to detect the
# exact GPU model statelessly here. So we return True for
# all ROCm platforms for now.
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM, PlatformEnum.XPU)
def is_cumem_allocator_available(self) -> bool:
try:
+16 -9
View File
@@ -18,6 +18,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import CUDAGraphMode, VllmConfig, set_current_vllm_config
from vllm.config.compilation import CompilationMode
from vllm.device_allocator import get_mem_allocator_instance
from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
@@ -162,8 +163,6 @@ class Worker(WorkerBase):
self._pp_send_work: list[Handle] = []
def sleep(self, level: int = 1) -> None:
from vllm.device_allocator.cumem import CuMemAllocator
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
# Save the buffers before level 2 sleep
@@ -173,7 +172,7 @@ class Worker(WorkerBase):
name: buffer.cpu().clone() for name, buffer in model.named_buffers()
}
allocator = CuMemAllocator.get_instance()
allocator = get_mem_allocator_instance()
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
@@ -186,9 +185,7 @@ class Worker(WorkerBase):
)
def wake_up(self, tags: list[str] | None = None) -> None:
from vllm.device_allocator.cumem import CuMemAllocator
allocator = CuMemAllocator.get_instance()
allocator = get_mem_allocator_instance()
allocator.wake_up(tags)
# Restore the buffers after level 2 sleep
@@ -203,12 +200,22 @@ class Worker(WorkerBase):
self.model_runner.post_kv_cache_wake_up()
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
if not self.vllm_config.model_config.enable_cumem_allocator:
if (
current_platform.is_cuda_alike()
and not self.vllm_config.model_config.enable_cumem_allocator
):
return nullcontext()
from vllm.device_allocator.cumem import CuMemAllocator
if (
current_platform.is_xpu()
and not self.vllm_config.model_config.enable_sleep_mode
):
return nullcontext()
allocator = CuMemAllocator.get_instance()
if current_platform.is_cpu():
return nullcontext()
allocator = get_mem_allocator_instance()
if tag == "weights":
assert allocator.get_current_usage() == 0, (
"CuMem allocator can only be used for one instance per process."