[Feature] Release the CUDA graph pool on sleep (#59160)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-04 01:13:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9e0b6fcf77
commit 5e56e9af2a
16 changed files with 299 additions and 27 deletions
@@ -423,3 +423,58 @@ def test_cumem_with_cudagraph():
# output content is as expected
assert torch.allclose(y, x + 1)
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
def test_cudagraph_pool_sleep(level):
"""Routing, and the graph pool backed up at both sleep levels and restored
in place by any wake."""
from contextlib import nullcontext
from types import SimpleNamespace
import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc
from vllm.compilation.cudagraph_pool import (
capture_outside_cumem_pool,
capture_pool,
)
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
allocator = get_mem_allocator_instance()
on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False))
handle = current_platform.graph_pool_handle()
for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())):
with ctx, capture_pool(handle, cfg) as used:
assert used == handle and allocator.current_tag != "cudagraph"
with allocator.use_memory_pool("weights"):
weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE)
x = torch.ones_like(weight)
def capture() -> list:
graph = torch.cuda.CUDAGraph()
stream = torch.cuda.Stream()
with (
capture_pool(handle, on) as pool,
torch.cuda.graph(graph, pool=pool, stream=stream),
):
assert nccl_alloc._graph_pool_id == pool != handle
const = torch.empty_like(x) # Never written by replay: needs a backup.
y = x * weight + const
return [graph, y, const.fill_(3.0)]
def graph_pool() -> dict[int, int]:
data = allocator.pointer_to_data.items()
return {
p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep
}
held, backend = capture(), CuMemBackend()
mapped = graph_pool()
backend.suspend(level=level)
backend.resume(tags=["kv_cache"])
assert graph_pool() == mapped
backend.resume(tags=["weights"])
weight.fill_(2.0) # Level 2 discards weights; emulate the reload.
held[0].replay()
assert torch.equal(held[1], torch.full_like(x, 5.0))
@@ -3,6 +3,7 @@
import asyncio
import math
import os
import pytest
import torch
@@ -14,7 +15,12 @@ from vllm.model_executor.model_loader import get_model_loader
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
from ....utils import (
create_new_process_for_each_test,
multi_gpu_marks,
multi_gpu_test,
requires_fp8,
)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@@ -195,6 +201,73 @@ def test_deep_sleep_lora_tp2(monkeypatch):
assert output[0].outputs[0].text == output2[0].outputs[0].text
def _cudagraph_bytes(worker) -> int:
data = cumem.CuMemAllocator.get_instance().pointer_to_data.values()
return sum(d.handle[1] for d in data if d.tag == "cudagraph")
def _custom_ar_active(worker) -> bool:
from vllm.distributed.parallel_state import get_tp_group
ca_comm = get_tp_group().device_communicator.ca_comm
return ca_comm is not None and not ca_comm.disabled
@pytest.mark.parametrize(
("mode", "breakable", "tp", "offload"),
[
("FULL", False, 1, True),
("PIECEWISE", False, 1, True),
("PIECEWISE", True, 1, True),
pytest.param(
"FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2)
),
("FULL_AND_PIECEWISE", False, 1, False),
],
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
)
@create_new_process_for_each_test()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
unregistered custom AR; off changes nothing."""
for name, value in [
("VLLM_USE_V2_MODEL_RUNNER", "1"),
("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"),
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
("VLLM_ALLREDUCE_USE_FLASHINFER", "0"),
("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"),
]:
monkeypatch.setenv(name, value)
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
llm = LLM(
"hmellor/tiny-random-LlamaForCausalLM",
enable_sleep_mode=True,
sleep_mode_offload_cudagraph=offload,
tensor_parallel_size=tp,
compilation_config={
"cudagraph_mode": mode,
"pass_config": {"fuse_allreduce_rms": False},
},
)
graph_bytes = llm.collective_rpc(_cudagraph_bytes)
assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None)
assert all(graph_bytes) if offload else not any(graph_bytes)
if not offload:
return
if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)):
pytest.skip("Custom allreduce is unavailable on these GPUs")
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, params)[0].outputs[0].text
for level in (1, 2):
llm.sleep(level=level)
llm.wake_up(tags=["weights"])
if level == 2:
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
assert llm.generate(prompt, params)[0].outputs[0].text == expected
@create_new_process_for_each_test()
def test_deep_sleep_async(gpu_memory_cleared):
async def test():