[XPU][Feature] transparent sleep mode support for XPU platform (#37149)

Signed-off-by: Yan Ma <yan.ma@intel.com>
This commit is contained in:
Yan Ma
2026-06-07 10:45:31 +08:00
committed by GitHub
parent 810966453a
commit 3bb46975bd
10 changed files with 410 additions and 47 deletions
@@ -7,7 +7,7 @@ import pytest
import torch
from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams
from vllm.device_allocator.cumem import CuMemAllocator
from vllm.device_allocator import get_mem_allocator_instance
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
@@ -16,14 +16,14 @@ from ..utils import create_new_process_for_each_test, requires_fp8
DEVICE_TYPE = current_platform.device_type
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_python_error():
"""
Test if Python error occurs when there's low-level
error happening from the C++ side.
"""
allocator = CuMemAllocator.get_instance()
total_bytes = torch.cuda.mem_get_info()[1]
allocator = get_mem_allocator_instance()
total_bytes = current_platform.mem_get_info()[1]
alloc_bytes = int(total_bytes * 0.7)
tensors = []
with allocator.use_memory_pool():
@@ -42,7 +42,7 @@ def test_python_error():
allocator.wake_up()
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_basic_cumem():
# some tensors from default memory pool
shape = (1024, 1024)
@@ -50,7 +50,7 @@ def test_basic_cumem():
x.zero_()
# some tensors from custom memory pool
allocator = CuMemAllocator.get_instance()
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
# custom memory pool
y = torch.empty(shape, device=DEVICE_TYPE)
@@ -64,9 +64,9 @@ def test_basic_cumem():
output = x + y + z
assert torch.allclose(output, torch.ones_like(output) * 3)
free_bytes = torch.cuda.mem_get_info()[0]
free_bytes = current_platform.mem_get_info()[0]
allocator.sleep()
free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
free_bytes_after_sleep = current_platform.mem_get_info()[0]
assert free_bytes_after_sleep > free_bytes
allocator.wake_up()
@@ -75,9 +75,10 @@ def test_basic_cumem():
assert torch.allclose(output, torch.ones_like(output) * 3)
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="CUDA graph not supported on XPU")
def test_cumem_with_cudagraph():
allocator = CuMemAllocator.get_instance()
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool():
weight = torch.eye(1024, device=DEVICE_TYPE)
with allocator.use_memory_pool(tag="discard"):
@@ -98,9 +99,9 @@ def test_cumem_with_cudagraph():
with torch.cuda.graph(model_graph):
y = model(x)
free_bytes = torch.cuda.mem_get_info()[0]
free_bytes = current_platform.mem_get_info()[0]
allocator.sleep()
free_bytes_after_sleep = torch.cuda.mem_get_info()[0]
free_bytes_after_sleep = current_platform.mem_get_info()[0]
assert free_bytes_after_sleep > free_bytes
allocator.wake_up()
@@ -120,7 +121,7 @@ def test_cumem_with_cudagraph():
assert torch.allclose(y, x + 1)
@create_new_process_for_each_test("fork" if not current_platform.is_rocm() else "spawn")
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.parametrize(
"model",
[
@@ -131,7 +132,7 @@ def test_cumem_with_cudagraph():
],
)
def test_end_to_end(model: str):
free, total = torch.cuda.mem_get_info()
free, total = current_platform.mem_get_info()
used_bytes_baseline = total - free # in case other process is running
llm = LLM(model, enable_sleep_mode=True)
prompt = "How are you?"
@@ -143,7 +144,7 @@ def test_end_to_end(model: str):
# test sleep level 1 here.
llm.sleep(level=1)
free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
# now the memory usage is mostly cudagraph memory pool,
# and it should be less than the model weights (1B model, 2GiB weights)
@@ -163,7 +164,7 @@ def test_end_to_end(model: str):
llm.sleep(level=1)
llm.wake_up(tags=["weights"])
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
# should just reallocate memory for weights (1B model, ~2GiB weights)
@@ -180,7 +181,7 @@ def test_end_to_end(model: str):
@create_new_process_for_each_test()
def test_deep_sleep():
model = "hmellor/tiny-random-LlamaForCausalLM"
free, total = torch.cuda.mem_get_info()
free, total = current_platform.mem_get_info()
used_bytes_baseline = total - free # in case other process is running
llm = LLM(model, enable_sleep_mode=True)
prompt = "How are you?"
@@ -190,13 +191,13 @@ def test_deep_sleep():
# Put the engine to deep sleep
llm.sleep(level=2)
free_gpu_bytes_after_sleep, total = torch.cuda.mem_get_info()
free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
assert used_bytes < 3 * GiB_bytes
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
assert used_bytes < 4 * GiB_bytes
@@ -212,7 +213,7 @@ def test_deep_sleep():
def test_deep_sleep_async():
async def test():
model = "hmellor/tiny-random-LlamaForCausalLM"
free, total = torch.cuda.mem_get_info()
free, total = current_platform.mem_get_info()
used_bytes_baseline = total - free # in case other process is running
engine_args = AsyncEngineArgs(
model=model,
@@ -231,7 +232,7 @@ def test_deep_sleep_async():
await llm.wake_up(tags=["weights"])
await llm.collective_rpc("reload_weights")
free_gpu_bytes_wake_up_w, total = torch.cuda.mem_get_info()
free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
assert used_bytes < 4 * GiB_bytes