[Feature] Release the CUDA graph pool on sleep (#59160)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-04 01:13:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9e0b6fcf77
commit 5e56e9af2a
16 changed files with 299 additions and 27 deletions
+35
View File
@@ -101,6 +101,41 @@ llm.release_kv_cache_memory()
llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling.
```
#### Offloading CUDA graph memory
By default, CUDA graph memory stays on the GPU while asleep. With
`sleep_mode_offload_cudagraph=True` (off by default), CUDA graphs are captured
into a cuMem pool that sleep backs up to CPU memory at both levels and any wake
restores in place, so graphs are replayed, not recaptured. It needs the default
`cumem` backend, CUDA and CUDA graphs; otherwise it has no effect.
```python
llm = LLM("Qwen/Qwen3-8B", enable_sleep_mode=True, sleep_mode_offload_cudagraph=True)
```
or `vllm serve <model> --enable-sleep-mode --sleep-mode-offload-cudagraph`.
The cost is pinned host memory for the pool's backup, also at level 2, and one
extra copy per captured custom allreduce (about 3% decode latency at batch
size 1). `NCCL_GRAPH_REGISTER` defaults to `0`, since NCCL graph registration
would pin the pool; an explicit value is kept with a warning. Graph executables
outside PyTorch pools stay resident.
#### Releasing NCCL communicator memory
By default, NCCL communicators keep their GPU buffers while asleep. With
`enable_nccl_comm_suspend=True` (off by default, experimental), sleep releases
them with `ncclCommSuspend` and wake restores them with `ncclCommResume`. The
communicators keep their topology, so they are not re-created. It needs NCCL
2.29.7 or newer; with an older library, a warning is logged and the memory stays
on the GPU.
```python
llm = LLM("Qwen/Qwen3-8B", enable_sleep_mode=True, enable_nccl_comm_suspend=True)
```
or `vllm serve <model> --enable-sleep-mode --enable-nccl-comm-suspend`.
### Online Serving
To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server.
@@ -423,3 +423,58 @@ def test_cumem_with_cudagraph():
# output content is as expected
assert torch.allclose(y, x + 1)
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
def test_cudagraph_pool_sleep(level):
"""Routing, and the graph pool backed up at both sleep levels and restored
in place by any wake."""
from contextlib import nullcontext
from types import SimpleNamespace
import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc
from vllm.compilation.cudagraph_pool import (
capture_outside_cumem_pool,
capture_pool,
)
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
allocator = get_mem_allocator_instance()
on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False))
handle = current_platform.graph_pool_handle()
for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())):
with ctx, capture_pool(handle, cfg) as used:
assert used == handle and allocator.current_tag != "cudagraph"
with allocator.use_memory_pool("weights"):
weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE)
x = torch.ones_like(weight)
def capture() -> list:
graph = torch.cuda.CUDAGraph()
stream = torch.cuda.Stream()
with (
capture_pool(handle, on) as pool,
torch.cuda.graph(graph, pool=pool, stream=stream),
):
assert nccl_alloc._graph_pool_id == pool != handle
const = torch.empty_like(x) # Never written by replay: needs a backup.
y = x * weight + const
return [graph, y, const.fill_(3.0)]
def graph_pool() -> dict[int, int]:
data = allocator.pointer_to_data.items()
return {
p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep
}
held, backend = capture(), CuMemBackend()
mapped = graph_pool()
backend.suspend(level=level)
backend.resume(tags=["kv_cache"])
assert graph_pool() == mapped
backend.resume(tags=["weights"])
weight.fill_(2.0) # Level 2 discards weights; emulate the reload.
held[0].replay()
assert torch.equal(held[1], torch.full_like(x, 5.0))
@@ -3,6 +3,7 @@
import asyncio
import math
import os
import pytest
import torch
@@ -14,7 +15,12 @@ from vllm.model_executor.model_loader import get_model_loader
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
from ....utils import (
create_new_process_for_each_test,
multi_gpu_marks,
multi_gpu_test,
requires_fp8,
)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@@ -195,6 +201,73 @@ def test_deep_sleep_lora_tp2(monkeypatch):
assert output[0].outputs[0].text == output2[0].outputs[0].text
def _cudagraph_bytes(worker) -> int:
data = cumem.CuMemAllocator.get_instance().pointer_to_data.values()
return sum(d.handle[1] for d in data if d.tag == "cudagraph")
def _custom_ar_active(worker) -> bool:
from vllm.distributed.parallel_state import get_tp_group
ca_comm = get_tp_group().device_communicator.ca_comm
return ca_comm is not None and not ca_comm.disabled
@pytest.mark.parametrize(
("mode", "breakable", "tp", "offload"),
[
("FULL", False, 1, True),
("PIECEWISE", False, 1, True),
("PIECEWISE", True, 1, True),
pytest.param(
"FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2)
),
("FULL_AND_PIECEWISE", False, 1, False),
],
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
)
@create_new_process_for_each_test()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
unregistered custom AR; off changes nothing."""
for name, value in [
("VLLM_USE_V2_MODEL_RUNNER", "1"),
("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"),
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
("VLLM_ALLREDUCE_USE_FLASHINFER", "0"),
("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"),
]:
monkeypatch.setenv(name, value)
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
llm = LLM(
"hmellor/tiny-random-LlamaForCausalLM",
enable_sleep_mode=True,
sleep_mode_offload_cudagraph=offload,
tensor_parallel_size=tp,
compilation_config={
"cudagraph_mode": mode,
"pass_config": {"fuse_allreduce_rms": False},
},
)
graph_bytes = llm.collective_rpc(_cudagraph_bytes)
assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None)
assert all(graph_bytes) if offload else not any(graph_bytes)
if not offload:
return
if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)):
pytest.skip("Custom allreduce is unavailable on these GPUs")
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, params)[0].outputs[0].text
for level in (1, 2):
llm.sleep(level=level)
llm.wake_up(tags=["weights"])
if level == 2:
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
assert llm.generate(prompt, params)[0].outputs[0].text == expected
@create_new_process_for_each_test()
def test_deep_sleep_async(gpu_memory_cleared):
async def test():
+3 -1
View File
@@ -177,7 +177,9 @@ def _worker(rank: int, tp_size: int, port: int) -> None:
# Supply the resolved Engram settings without constructing a full model.
vllm_config.engram_config = EngramConfig()
vllm_config.model_config = SimpleNamespace(
architecture="DeepseekV41ForCausalLM", is_moe=True
architecture="DeepseekV41ForCausalLM",
is_moe=True,
sleep_mode_offload_cudagraph=False,
)
try:
init_distributed_environment()
@@ -50,6 +50,7 @@ def _create_vllm_config() -> MagicMock:
vllm_config.parallel_config = ParallelConfig()
vllm_config.speculative_config = None
vllm_config.num_speculative_tokens = 0
vllm_config.use_cumem_cudagraph_pool = False
return vllm_config
+5 -8
View File
@@ -34,9 +34,9 @@ from typing import Any, ClassVar, TypeVar
import torch
import vllm.envs as envs
from vllm.compilation.cudagraph_pool import capture_pool
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.forward_context import (
BatchDescriptor,
get_forward_context,
@@ -370,11 +370,6 @@ class BreakableCUDAGraphWrapper:
entry.input_addresses = self._collect_tensor_addresses(args, kwargs)
if self.graph_pool is not None:
set_graph_pool_id(self.graph_pool)
else:
set_graph_pool_id(current_platform.graph_pool_handle())
# Match torch.cuda.graph()'s pre-capture cleanup, which we bypass.
# Skip it when gc is disabled: bulk capture runs under
# freeze_gc_for_cudagraph_capture, which already did this cleanup,
@@ -386,8 +381,10 @@ class BreakableCUDAGraphWrapper:
# pre-capture prefetches are complete and don't leak into the graph.
get_offloader().sync_prev_onload()
capture = BreakableCUDAGraphCapture(pool=self.graph_pool)
with capture:
with (
capture_pool(self.graph_pool, self.vllm_config) as pool,
BreakableCUDAGraphCapture(pool=pool) as capture,
):
output = self.runnable(*args, **kwargs)
# Join the offloader's copy stream while we still hold the last
# segment open, so the join is captured into the graph (otherwise
+5 -6
View File
@@ -13,9 +13,9 @@ from unittest.mock import patch
import torch
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cudagraph_pool import capture_pool
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.forward_context import (
BatchDescriptor,
get_forward_context,
@@ -302,10 +302,9 @@ class CUDAGraphWrapper:
)
)
if self.graph_pool is not None:
set_graph_pool_id(self.graph_pool)
else:
set_graph_pool_id(current_platform.graph_pool_handle())
graph_pool = stack.enter_context(
capture_pool(self.graph_pool, self.vllm_config)
)
# Sync offloader's copy stream before capture.
# Ensure any pre-capture prefetches from offloader are complete.
@@ -314,7 +313,7 @@ class CUDAGraphWrapper:
# mind-exploding: carefully manage the reference and memory.
with torch.cuda.graph(
cudagraph,
pool=self.graph_pool,
pool=graph_pool,
stream=current_stream(),
):
# `output` is managed by pytorch's cudagraph pool
+45
View File
@@ -0,0 +1,45 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Graph pool routing for CUDA graph capture under sleep mode."""
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from typing import TYPE_CHECKING, cast
from vllm.config import VllmConfig
from vllm.device_allocator import get_mem_allocator_instance
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.device_allocator.cumem import CuMemAllocator
_outside_cumem: ContextVar[bool] = ContextVar("capture_outside_cumem", default=False)
@contextmanager
def capture_outside_cumem_pool() -> Iterator[None]:
"""CUDA graphs captured here stay out of cuMem, so that memory profiling can
free its throwaway graphs normally."""
token = _outside_cumem.set(True)
try:
yield
finally:
_outside_cumem.reset(token)
@contextmanager
def capture_pool(
pool: tuple[int, int] | None, vllm_config: VllmConfig
) -> Iterator[tuple[int, int] | None]:
"""Yield the pool to capture into, the cuMem graph pool when sleep offloads
graph memory, and point NCCL's graph allocator at it."""
if vllm_config.use_cumem_cudagraph_pool and not _outside_cumem.get():
allocator = cast("CuMemAllocator", get_mem_allocator_instance())
with allocator.cudagraph_pool() as cumem_pool:
set_graph_pool_id(cumem_pool)
yield cumem_pool
else:
set_graph_pool_id(pool or current_platform.graph_pool_handle())
yield pool
+3
View File
@@ -378,6 +378,9 @@ class ModelConfig:
(default) uses the built-in ``CuMemAllocator`` and is behavior-compatible
with prior releases. Additional backends (CUDA checkpoint, CRIU, durable
snapshot) may be registered in-tree or by plugins (RFC #34303)."""
sleep_mode_offload_cudagraph: bool = False
"""Back up CUDA graph memory to CPU during sleep, restored in place on wake.
Takes effect with enable_sleep_mode, the cumem backend and CUDA graphs."""
enable_nccl_comm_suspend: bool = False
"""Enable releasing NCCL communicator memory during sleep mode
(``ncclCommSuspend``/``ncclCommResume``). Experimental; when disabled
+23
View File
@@ -697,6 +697,21 @@ class VllmConfig:
"""
return 1 + self.num_speculative_tokens
@property
def use_cumem_cudagraph_pool(self) -> bool:
"""Whether CUDA graphs go to the cuMem pool that sleep offloads."""
from vllm.platforms import current_platform
model_config = self.model_config
return (
model_config is not None
and model_config.sleep_mode_offload_cudagraph
and model_config.enable_sleep_mode
and model_config.sleep_mode_backend == "cumem"
and self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
and current_platform.is_cuda()
)
@property
def use_v2_model_runner(self) -> bool:
if self.attention_config.hisparse_config is not None:
@@ -2321,6 +2336,14 @@ class VllmConfig:
custom_ops.append("+quant_fp8")
self._verify_kv_transfer_compat()
if self.use_cumem_cudagraph_pool:
# NCCL graph registration pins the offloaded pool; workers inherit this.
value = os.environ.setdefault("NCCL_GRAPH_REGISTER", "0")
if value != "0":
logger.warning(
"NCCL_GRAPH_REGISTER=%s pins the CUDA graph pool during sleep.",
value,
)
# Log the custom passes that are enabled
self.compilation_config.pass_config.log_enabled_passes()
+17
View File
@@ -430,6 +430,23 @@ class CuMemAllocator:
if allocation["allocated_size"] == 0 and data and not data.is_asleep:
unmap_and_release(self._python_free_callback(allocation["address"]))
@contextmanager
def cudagraph_pool(self) -> Iterator[tuple[int, int]]:
"""Tag CUDA graph capture allocations and yield the graph pool id."""
if "cudagraph" not in self.allocator_and_pools:
allocator = get_pluggable_allocator(
self.python_malloc_callback, self.python_free_callback
)
mem_pool = torch.cuda.memory.MemPool(allocator._allocator)
self.allocator_and_pools["cudagraph"] = [(mem_pool, allocator)]
old_tag = self.current_tag
self.current_tag = "cudagraph"
try:
# capture_begin routes allocations to this pool; no use_mem_pool.
yield self.allocator_and_pools["cudagraph"][0][0].id
finally:
self.current_tag = old_tag
def get_current_usage(self) -> int:
"""Get the total number of bytes allocated in the memory pool."""
sum_bytes: int = 0
+5 -2
View File
@@ -129,8 +129,11 @@ class CuMemBackend(SleepModeBackend):
self._state = "SUSPENDED"
allocator = get_mem_allocator_instance()
# Runtime state is kept at every level; weights only at level 1.
offload_tags = ("weights", "runtime") if level == 1 else ("runtime",)
# Runtime state and graph pools are kept at every level; weights only
# at level 1.
offload_tags: tuple[str, ...] = ("runtime", "cudagraph")
if level == 1:
offload_tags = ("weights", *offload_tags)
allocator.sleep(offload_tags=offload_tags)
def resume(self, tags: list[str] | None = None) -> None:
@@ -7,6 +7,7 @@ from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import get_current_vllm_config_or_none
from vllm.distributed.device_communicators.all_reduce_utils import (
NCCL_SYMM_MEM_ALL_REDUCE_CONFIG,
should_nccl_symm_mem_ag_rs,
@@ -114,6 +115,10 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.aiter_ar_comm: AiterCustomAllreduce | None = None
self.use_aiter_ag_rs: bool = False
# cuMem graph buffers cannot be IPC-registered; capture copies them instead.
config = get_current_vllm_config_or_none()
register = config is None or not config.use_cumem_cudagraph_pool
if use_torch_symm_mem and current_platform.is_cuda():
self.symm_mem_comm = SymmMemCommunicator(
group=self.cpu_group,
@@ -151,6 +156,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
symm_mem_enabled=(
self.symm_mem_comm is not None and not self.symm_mem_comm.disabled
),
register_graph_buffers=register,
)
# AITER custom all-gather/reduce-scatter DP-attention dispatch/combine
@@ -147,18 +147,22 @@ class CustomAllreduce:
_DEFAULT_MNNVL_MULTIMEM_REDUCE_SCATTER_MAX_SIZE
),
symm_mem_enabled=False,
*,
register_graph_buffers: bool = True,
) -> None:
"""Args:
group: the process group to work on. If None, it will use the
default process group.
device: the device to bind the CustomAllreduce to. If None,
it will be bound to f"cuda:{local_rank}".
register_graph_buffers: whether graph capture IPC-registers its buffers.
It is the caller's responsibility to make sure each communicator
is bind to a unique device, and all communicators in this group
are in the same node.
"""
self._IS_CAPTURING = False
self._capture_registered = register_graph_buffers
self._ptr = 0
self.disabled = True
self.mnnvl_buffer = None
@@ -549,7 +553,7 @@ class CustomAllreduce:
return None
if self._IS_CAPTURING:
if torch.cuda.is_current_stream_capturing():
return self.all_reduce(input, registered=True)
return self.all_reduce(input, registered=self._capture_registered)
else:
# If warm up, mimic the allocation pattern since custom
# allreduce is out-of-place.
+6
View File
@@ -770,6 +770,7 @@ class EngineArgs:
generation_config: str = ModelConfig.generation_config
enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
sleep_mode_offload_cudagraph: bool = ModelConfig.sleep_mode_offload_cudagraph
sleep_preserve_parameter_names: list[str] = get_field(
ModelConfig, "sleep_preserve_parameter_names"
)
@@ -1007,6 +1008,10 @@ class EngineArgs:
model_group.add_argument(
"--enable-sleep-mode", **model_kwargs["enable_sleep_mode"]
)
model_group.add_argument(
"--sleep-mode-offload-cudagraph",
**model_kwargs["sleep_mode_offload_cudagraph"],
)
model_group.add_argument(
"--sleep-preserve-parameter-names",
**model_kwargs["sleep_preserve_parameter_names"],
@@ -1984,6 +1989,7 @@ class EngineArgs:
generation_config=self.generation_config,
override_generation_config=self.override_generation_config,
enable_sleep_mode=self.enable_sleep_mode,
sleep_mode_offload_cudagraph=self.sleep_mode_offload_cudagraph,
sleep_preserve_parameter_names=self.sleep_preserve_parameter_names,
enable_cumem_allocator=self.enable_cumem_allocator,
enable_nccl_comm_suspend=self.enable_nccl_comm_suspend,
+11 -8
View File
@@ -19,10 +19,13 @@ from vllm.compilation.breakable_cudagraph import (
)
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphStat, CUDAGraphWrapper
from vllm.compilation.cudagraph_pool import (
capture_outside_cumem_pool,
capture_pool,
)
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.distributed.parallel_state import (
get_pp_group,
graph_capture,
@@ -481,15 +484,14 @@ class CudaGraphManager:
# Sync offloader's copy stream before capture.
# Ensure any pre-capture prefetches from offloader are complete.
get_offloader().sync_prev_onload()
if self.pool is not None:
set_graph_pool_id(self.pool)
else:
set_graph_pool_id(current_platform.graph_pool_handle())
if self._capture_mem_samples is not None:
torch.accelerator.synchronize()
free_before = torch.accelerator.get_memory_info()[0]
with torch.cuda.graph(
graph, self.pool, stream=self._capture_stream(desc)
with (
capture_pool(self.pool, self.vllm_config) as pool,
torch.cuda.graph(
graph, pool, stream=self._capture_stream(desc)
),
):
forward_fn(CUDAGraphMode.NONE)
# Join offloader's copy stream after forward to avoid
@@ -941,7 +943,8 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
mem_samples: list[int] = []
manager._capture_mem_samples = mem_samples
measured = int(runner.capture_model(profile_only=True))
with capture_outside_cumem_pool():
measured = int(runner.capture_model(profile_only=True))
# The measured delta covers PIECEWISE, encoder and speculator graphs
# plus the sampled FULL graphs; swap the sampled FULL cost for the