From 5e56e9af2aaede223a5f723cba88fa35999ac813 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Sun, 4 Oct 2026 09:13:10 +0800 Subject: [PATCH] [Feature] Release the CUDA graph pool on sleep (#59160) Signed-off-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- docs/features/sleep_mode.md | 35 +++++++++ .../memory/cumem/test_cumem.py | 55 ++++++++++++++ .../memory/sleep_mode/test_sleep_mode.py | 75 ++++++++++++++++++- tests/distributed/test_engram_dp_shard.py | 4 +- tests/v1/cudagraph/test_cudagraph_manager.py | 1 + vllm/compilation/breakable_cudagraph.py | 13 ++-- vllm/compilation/cuda_graph.py | 11 ++- vllm/compilation/cudagraph_pool.py | 45 +++++++++++ vllm/config/model.py | 3 + vllm/config/vllm.py | 23 ++++++ vllm/device_allocator/cumem.py | 17 +++++ vllm/device_allocator/sleep_mode_backend.py | 7 +- .../device_communicators/cuda_communicator.py | 6 ++ .../device_communicators/custom_all_reduce.py | 6 +- vllm/engine/arg_utils.py | 6 ++ vllm/v1/worker/gpu/cudagraph_utils.py | 19 +++-- 16 files changed, 299 insertions(+), 27 deletions(-) create mode 100644 vllm/compilation/cudagraph_pool.py diff --git a/docs/features/sleep_mode.md b/docs/features/sleep_mode.md index 3e419707f2..a8aa15af7b 100644 --- a/docs/features/sleep_mode.md +++ b/docs/features/sleep_mode.md @@ -101,6 +101,41 @@ llm.release_kv_cache_memory() llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling. ``` +#### Offloading CUDA graph memory + +By default, CUDA graph memory stays on the GPU while asleep. With +`sleep_mode_offload_cudagraph=True` (off by default), CUDA graphs are captured +into a cuMem pool that sleep backs up to CPU memory at both levels and any wake +restores in place, so graphs are replayed, not recaptured. It needs the default +`cumem` backend, CUDA and CUDA graphs; otherwise it has no effect. + +```python +llm = LLM("Qwen/Qwen3-8B", enable_sleep_mode=True, sleep_mode_offload_cudagraph=True) +``` + +or `vllm serve --enable-sleep-mode --sleep-mode-offload-cudagraph`. + +The cost is pinned host memory for the pool's backup, also at level 2, and one +extra copy per captured custom allreduce (about 3% decode latency at batch +size 1). `NCCL_GRAPH_REGISTER` defaults to `0`, since NCCL graph registration +would pin the pool; an explicit value is kept with a warning. Graph executables +outside PyTorch pools stay resident. + +#### Releasing NCCL communicator memory + +By default, NCCL communicators keep their GPU buffers while asleep. With +`enable_nccl_comm_suspend=True` (off by default, experimental), sleep releases +them with `ncclCommSuspend` and wake restores them with `ncclCommResume`. The +communicators keep their topology, so they are not re-created. It needs NCCL +2.29.7 or newer; with an older library, a warning is logged and the memory stays +on the GPU. + +```python +llm = LLM("Qwen/Qwen3-8B", enable_sleep_mode=True, enable_nccl_comm_suspend=True) +``` + +or `vllm serve --enable-sleep-mode --enable-nccl-comm-suspend`. + ### Online Serving To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server. diff --git a/tests/basic_correctness/memory/cumem/test_cumem.py b/tests/basic_correctness/memory/cumem/test_cumem.py index fa06facf29..df8c604882 100644 --- a/tests/basic_correctness/memory/cumem/test_cumem.py +++ b/tests/basic_correctness/memory/cumem/test_cumem.py @@ -423,3 +423,58 @@ def test_cumem_with_cudagraph(): # output content is as expected assert torch.allclose(y, x + 1) + + +@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool") +def test_cudagraph_pool_sleep(level): + """Routing, and the graph pool backed up at both sleep levels and restored + in place by any wake.""" + from contextlib import nullcontext + from types import SimpleNamespace + + import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc + from vllm.compilation.cudagraph_pool import ( + capture_outside_cumem_pool, + capture_pool, + ) + from vllm.device_allocator.sleep_mode_backend import CuMemBackend + + allocator = get_mem_allocator_instance() + on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False)) + handle = current_platform.graph_pool_handle() + for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())): + with ctx, capture_pool(handle, cfg) as used: + assert used == handle and allocator.current_tag != "cudagraph" + with allocator.use_memory_pool("weights"): + weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE) + x = torch.ones_like(weight) + + def capture() -> list: + graph = torch.cuda.CUDAGraph() + stream = torch.cuda.Stream() + with ( + capture_pool(handle, on) as pool, + torch.cuda.graph(graph, pool=pool, stream=stream), + ): + assert nccl_alloc._graph_pool_id == pool != handle + const = torch.empty_like(x) # Never written by replay: needs a backup. + y = x * weight + const + return [graph, y, const.fill_(3.0)] + + def graph_pool() -> dict[int, int]: + data = allocator.pointer_to_data.items() + return { + p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep + } + + held, backend = capture(), CuMemBackend() + mapped = graph_pool() + backend.suspend(level=level) + backend.resume(tags=["kv_cache"]) + assert graph_pool() == mapped + backend.resume(tags=["weights"]) + weight.fill_(2.0) # Level 2 discards weights; emulate the reload. + held[0].replay() + assert torch.equal(held[1], torch.full_like(x, 5.0)) diff --git a/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py b/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py index 3ed4703c8b..73af6e3910 100644 --- a/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py +++ b/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py @@ -3,6 +3,7 @@ import asyncio import math +import os import pytest import torch @@ -14,7 +15,12 @@ from vllm.model_executor.model_loader import get_model_loader from vllm.platforms import current_platform from vllm.utils.mem_constants import GiB_bytes -from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8 +from ....utils import ( + create_new_process_for_each_test, + multi_gpu_marks, + multi_gpu_test, + requires_fp8, +) @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") @@ -195,6 +201,73 @@ def test_deep_sleep_lora_tp2(monkeypatch): assert output[0].outputs[0].text == output2[0].outputs[0].text +def _cudagraph_bytes(worker) -> int: + data = cumem.CuMemAllocator.get_instance().pointer_to_data.values() + return sum(d.handle[1] for d in data if d.tag == "cudagraph") + + +def _custom_ar_active(worker) -> bool: + from vllm.distributed.parallel_state import get_tp_group + + ca_comm = get_tp_group().device_communicator.ca_comm + return ca_comm is not None and not ca_comm.disabled + + +@pytest.mark.parametrize( + ("mode", "breakable", "tp", "offload"), + [ + ("FULL", False, 1, True), + ("PIECEWISE", False, 1, True), + ("PIECEWISE", True, 1, True), + pytest.param( + "FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2) + ), + ("FULL_AND_PIECEWISE", False, 1, False), + ], + ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"], +) +@create_new_process_for_each_test() +@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool") +def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload): + """Each capture site uses the pool, exact across sleeps; TP=2 needs + unregistered custom AR; off changes nothing.""" + for name, value in [ + ("VLLM_USE_V2_MODEL_RUNNER", "1"), + ("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"), + ("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"), + ("VLLM_ALLREDUCE_USE_FLASHINFER", "0"), + ("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"), + ]: + monkeypatch.setenv(name, value) + monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False) + llm = LLM( + "hmellor/tiny-random-LlamaForCausalLM", + enable_sleep_mode=True, + sleep_mode_offload_cudagraph=offload, + tensor_parallel_size=tp, + compilation_config={ + "cudagraph_mode": mode, + "pass_config": {"fuse_allreduce_rms": False}, + }, + ) + graph_bytes = llm.collective_rpc(_cudagraph_bytes) + assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None) + assert all(graph_bytes) if offload else not any(graph_bytes) + if not offload: + return + if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)): + pytest.skip("Custom allreduce is unavailable on these GPUs") + prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10) + expected = llm.generate(prompt, params)[0].outputs[0].text + for level in (1, 2): + llm.sleep(level=level) + llm.wake_up(tags=["weights"]) + if level == 2: + llm.collective_rpc("reload_weights") + llm.wake_up(tags=["kv_cache"]) + assert llm.generate(prompt, params)[0].outputs[0].text == expected + + @create_new_process_for_each_test() def test_deep_sleep_async(gpu_memory_cleared): async def test(): diff --git a/tests/distributed/test_engram_dp_shard.py b/tests/distributed/test_engram_dp_shard.py index ac73188534..637d179a5d 100644 --- a/tests/distributed/test_engram_dp_shard.py +++ b/tests/distributed/test_engram_dp_shard.py @@ -177,7 +177,9 @@ def _worker(rank: int, tp_size: int, port: int) -> None: # Supply the resolved Engram settings without constructing a full model. vllm_config.engram_config = EngramConfig() vllm_config.model_config = SimpleNamespace( - architecture="DeepseekV41ForCausalLM", is_moe=True + architecture="DeepseekV41ForCausalLM", + is_moe=True, + sleep_mode_offload_cudagraph=False, ) try: init_distributed_environment() diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index 8f58462d6b..154116b80f 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -50,6 +50,7 @@ def _create_vllm_config() -> MagicMock: vllm_config.parallel_config = ParallelConfig() vllm_config.speculative_config = None vllm_config.num_speculative_tokens = 0 + vllm_config.use_cumem_cudagraph_pool = False return vllm_config diff --git a/vllm/compilation/breakable_cudagraph.py b/vllm/compilation/breakable_cudagraph.py index 781868804e..b70fe70d8c 100644 --- a/vllm/compilation/breakable_cudagraph.py +++ b/vllm/compilation/breakable_cudagraph.py @@ -34,9 +34,9 @@ from typing import Any, ClassVar, TypeVar import torch import vllm.envs as envs +from vllm.compilation.cudagraph_pool import capture_pool from vllm.compilation.monitor import validate_cudagraph_capturing_enabled from vllm.config import CUDAGraphMode, VllmConfig -from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id from vllm.forward_context import ( BatchDescriptor, get_forward_context, @@ -370,11 +370,6 @@ class BreakableCUDAGraphWrapper: entry.input_addresses = self._collect_tensor_addresses(args, kwargs) - if self.graph_pool is not None: - set_graph_pool_id(self.graph_pool) - else: - set_graph_pool_id(current_platform.graph_pool_handle()) - # Match torch.cuda.graph()'s pre-capture cleanup, which we bypass. # Skip it when gc is disabled: bulk capture runs under # freeze_gc_for_cudagraph_capture, which already did this cleanup, @@ -386,8 +381,10 @@ class BreakableCUDAGraphWrapper: # pre-capture prefetches are complete and don't leak into the graph. get_offloader().sync_prev_onload() - capture = BreakableCUDAGraphCapture(pool=self.graph_pool) - with capture: + with ( + capture_pool(self.graph_pool, self.vllm_config) as pool, + BreakableCUDAGraphCapture(pool=pool) as capture, + ): output = self.runnable(*args, **kwargs) # Join the offloader's copy stream while we still hold the last # segment open, so the join is captured into the graph (otherwise diff --git a/vllm/compilation/cuda_graph.py b/vllm/compilation/cuda_graph.py index dc8a8a9c84..2dea34e0ff 100644 --- a/vllm/compilation/cuda_graph.py +++ b/vllm/compilation/cuda_graph.py @@ -13,9 +13,9 @@ from unittest.mock import patch import torch from vllm.compilation.counter import compilation_counter +from vllm.compilation.cudagraph_pool import capture_pool from vllm.compilation.monitor import validate_cudagraph_capturing_enabled from vllm.config import CUDAGraphMode, VllmConfig -from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id from vllm.forward_context import ( BatchDescriptor, get_forward_context, @@ -302,10 +302,9 @@ class CUDAGraphWrapper: ) ) - if self.graph_pool is not None: - set_graph_pool_id(self.graph_pool) - else: - set_graph_pool_id(current_platform.graph_pool_handle()) + graph_pool = stack.enter_context( + capture_pool(self.graph_pool, self.vllm_config) + ) # Sync offloader's copy stream before capture. # Ensure any pre-capture prefetches from offloader are complete. @@ -314,7 +313,7 @@ class CUDAGraphWrapper: # mind-exploding: carefully manage the reference and memory. with torch.cuda.graph( cudagraph, - pool=self.graph_pool, + pool=graph_pool, stream=current_stream(), ): # `output` is managed by pytorch's cudagraph pool diff --git a/vllm/compilation/cudagraph_pool.py b/vllm/compilation/cudagraph_pool.py new file mode 100644 index 0000000000..c4cfeabd81 --- /dev/null +++ b/vllm/compilation/cudagraph_pool.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Graph pool routing for CUDA graph capture under sleep mode.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import TYPE_CHECKING, cast + +from vllm.config import VllmConfig +from vllm.device_allocator import get_mem_allocator_instance +from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id +from vllm.platforms import current_platform + +if TYPE_CHECKING: + from vllm.device_allocator.cumem import CuMemAllocator + +_outside_cumem: ContextVar[bool] = ContextVar("capture_outside_cumem", default=False) + + +@contextmanager +def capture_outside_cumem_pool() -> Iterator[None]: + """CUDA graphs captured here stay out of cuMem, so that memory profiling can + free its throwaway graphs normally.""" + token = _outside_cumem.set(True) + try: + yield + finally: + _outside_cumem.reset(token) + + +@contextmanager +def capture_pool( + pool: tuple[int, int] | None, vllm_config: VllmConfig +) -> Iterator[tuple[int, int] | None]: + """Yield the pool to capture into, the cuMem graph pool when sleep offloads + graph memory, and point NCCL's graph allocator at it.""" + if vllm_config.use_cumem_cudagraph_pool and not _outside_cumem.get(): + allocator = cast("CuMemAllocator", get_mem_allocator_instance()) + with allocator.cudagraph_pool() as cumem_pool: + set_graph_pool_id(cumem_pool) + yield cumem_pool + else: + set_graph_pool_id(pool or current_platform.graph_pool_handle()) + yield pool diff --git a/vllm/config/model.py b/vllm/config/model.py index 3e481d8c91..538bdcc834 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -378,6 +378,9 @@ class ModelConfig: (default) uses the built-in ``CuMemAllocator`` and is behavior-compatible with prior releases. Additional backends (CUDA checkpoint, CRIU, durable snapshot) may be registered in-tree or by plugins (RFC #34303).""" + sleep_mode_offload_cudagraph: bool = False + """Back up CUDA graph memory to CPU during sleep, restored in place on wake. + Takes effect with enable_sleep_mode, the cumem backend and CUDA graphs.""" enable_nccl_comm_suspend: bool = False """Enable releasing NCCL communicator memory during sleep mode (``ncclCommSuspend``/``ncclCommResume``). Experimental; when disabled diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 281666a633..32b49f4272 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -697,6 +697,21 @@ class VllmConfig: """ return 1 + self.num_speculative_tokens + @property + def use_cumem_cudagraph_pool(self) -> bool: + """Whether CUDA graphs go to the cuMem pool that sleep offloads.""" + from vllm.platforms import current_platform + + model_config = self.model_config + return ( + model_config is not None + and model_config.sleep_mode_offload_cudagraph + and model_config.enable_sleep_mode + and model_config.sleep_mode_backend == "cumem" + and self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE + and current_platform.is_cuda() + ) + @property def use_v2_model_runner(self) -> bool: if self.attention_config.hisparse_config is not None: @@ -2321,6 +2336,14 @@ class VllmConfig: custom_ops.append("+quant_fp8") self._verify_kv_transfer_compat() + if self.use_cumem_cudagraph_pool: + # NCCL graph registration pins the offloaded pool; workers inherit this. + value = os.environ.setdefault("NCCL_GRAPH_REGISTER", "0") + if value != "0": + logger.warning( + "NCCL_GRAPH_REGISTER=%s pins the CUDA graph pool during sleep.", + value, + ) # Log the custom passes that are enabled self.compilation_config.pass_config.log_enabled_passes() diff --git a/vllm/device_allocator/cumem.py b/vllm/device_allocator/cumem.py index 6bf0ea71f8..b85be08d28 100644 --- a/vllm/device_allocator/cumem.py +++ b/vllm/device_allocator/cumem.py @@ -430,6 +430,23 @@ class CuMemAllocator: if allocation["allocated_size"] == 0 and data and not data.is_asleep: unmap_and_release(self._python_free_callback(allocation["address"])) + @contextmanager + def cudagraph_pool(self) -> Iterator[tuple[int, int]]: + """Tag CUDA graph capture allocations and yield the graph pool id.""" + if "cudagraph" not in self.allocator_and_pools: + allocator = get_pluggable_allocator( + self.python_malloc_callback, self.python_free_callback + ) + mem_pool = torch.cuda.memory.MemPool(allocator._allocator) + self.allocator_and_pools["cudagraph"] = [(mem_pool, allocator)] + old_tag = self.current_tag + self.current_tag = "cudagraph" + try: + # capture_begin routes allocations to this pool; no use_mem_pool. + yield self.allocator_and_pools["cudagraph"][0][0].id + finally: + self.current_tag = old_tag + def get_current_usage(self) -> int: """Get the total number of bytes allocated in the memory pool.""" sum_bytes: int = 0 diff --git a/vllm/device_allocator/sleep_mode_backend.py b/vllm/device_allocator/sleep_mode_backend.py index 8ec9986ffc..2132b880de 100644 --- a/vllm/device_allocator/sleep_mode_backend.py +++ b/vllm/device_allocator/sleep_mode_backend.py @@ -129,8 +129,11 @@ class CuMemBackend(SleepModeBackend): self._state = "SUSPENDED" allocator = get_mem_allocator_instance() - # Runtime state is kept at every level; weights only at level 1. - offload_tags = ("weights", "runtime") if level == 1 else ("runtime",) + # Runtime state and graph pools are kept at every level; weights only + # at level 1. + offload_tags: tuple[str, ...] = ("runtime", "cudagraph") + if level == 1: + offload_tags = ("weights", *offload_tags) allocator.sleep(offload_tags=offload_tags) def resume(self, tags: list[str] | None = None) -> None: diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index 0a203956f0..99ebcd717d 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -7,6 +7,7 @@ from torch.distributed import ProcessGroup import vllm.envs as envs from vllm._aiter_ops import rocm_aiter_ops +from vllm.config import get_current_vllm_config_or_none from vllm.distributed.device_communicators.all_reduce_utils import ( NCCL_SYMM_MEM_ALL_REDUCE_CONFIG, should_nccl_symm_mem_ag_rs, @@ -114,6 +115,10 @@ class CudaCommunicator(DeviceCommunicatorBase): self.aiter_ar_comm: AiterCustomAllreduce | None = None self.use_aiter_ag_rs: bool = False + # cuMem graph buffers cannot be IPC-registered; capture copies them instead. + config = get_current_vllm_config_or_none() + register = config is None or not config.use_cumem_cudagraph_pool + if use_torch_symm_mem and current_platform.is_cuda(): self.symm_mem_comm = SymmMemCommunicator( group=self.cpu_group, @@ -151,6 +156,7 @@ class CudaCommunicator(DeviceCommunicatorBase): symm_mem_enabled=( self.symm_mem_comm is not None and not self.symm_mem_comm.disabled ), + register_graph_buffers=register, ) # AITER custom all-gather/reduce-scatter DP-attention dispatch/combine diff --git a/vllm/distributed/device_communicators/custom_all_reduce.py b/vllm/distributed/device_communicators/custom_all_reduce.py index 9e8477e494..d249b5e2f0 100644 --- a/vllm/distributed/device_communicators/custom_all_reduce.py +++ b/vllm/distributed/device_communicators/custom_all_reduce.py @@ -147,18 +147,22 @@ class CustomAllreduce: _DEFAULT_MNNVL_MULTIMEM_REDUCE_SCATTER_MAX_SIZE ), symm_mem_enabled=False, + *, + register_graph_buffers: bool = True, ) -> None: """Args: group: the process group to work on. If None, it will use the default process group. device: the device to bind the CustomAllreduce to. If None, it will be bound to f"cuda:{local_rank}". + register_graph_buffers: whether graph capture IPC-registers its buffers. It is the caller's responsibility to make sure each communicator is bind to a unique device, and all communicators in this group are in the same node. """ self._IS_CAPTURING = False + self._capture_registered = register_graph_buffers self._ptr = 0 self.disabled = True self.mnnvl_buffer = None @@ -549,7 +553,7 @@ class CustomAllreduce: return None if self._IS_CAPTURING: if torch.cuda.is_current_stream_capturing(): - return self.all_reduce(input, registered=True) + return self.all_reduce(input, registered=self._capture_registered) else: # If warm up, mimic the allocation pattern since custom # allreduce is out-of-place. diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index f4cbcf9012..4f1f417289 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -770,6 +770,7 @@ class EngineArgs: generation_config: str = ModelConfig.generation_config enable_sleep_mode: bool = ModelConfig.enable_sleep_mode + sleep_mode_offload_cudagraph: bool = ModelConfig.sleep_mode_offload_cudagraph sleep_preserve_parameter_names: list[str] = get_field( ModelConfig, "sleep_preserve_parameter_names" ) @@ -1007,6 +1008,10 @@ class EngineArgs: model_group.add_argument( "--enable-sleep-mode", **model_kwargs["enable_sleep_mode"] ) + model_group.add_argument( + "--sleep-mode-offload-cudagraph", + **model_kwargs["sleep_mode_offload_cudagraph"], + ) model_group.add_argument( "--sleep-preserve-parameter-names", **model_kwargs["sleep_preserve_parameter_names"], @@ -1984,6 +1989,7 @@ class EngineArgs: generation_config=self.generation_config, override_generation_config=self.override_generation_config, enable_sleep_mode=self.enable_sleep_mode, + sleep_mode_offload_cudagraph=self.sleep_mode_offload_cudagraph, sleep_preserve_parameter_names=self.sleep_preserve_parameter_names, enable_cumem_allocator=self.enable_cumem_allocator, enable_nccl_comm_suspend=self.enable_nccl_comm_suspend, diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 19f3157cea..05fcf5f98f 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -19,10 +19,13 @@ from vllm.compilation.breakable_cudagraph import ( ) from vllm.compilation.counter import compilation_counter from vllm.compilation.cuda_graph import CUDAGraphStat, CUDAGraphWrapper +from vllm.compilation.cudagraph_pool import ( + capture_outside_cumem_pool, + capture_pool, +) from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper from vllm.config import VllmConfig, set_current_vllm_config from vllm.config.compilation import CUDAGraphMode -from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id from vllm.distributed.parallel_state import ( get_pp_group, graph_capture, @@ -481,15 +484,14 @@ class CudaGraphManager: # Sync offloader's copy stream before capture. # Ensure any pre-capture prefetches from offloader are complete. get_offloader().sync_prev_onload() - if self.pool is not None: - set_graph_pool_id(self.pool) - else: - set_graph_pool_id(current_platform.graph_pool_handle()) if self._capture_mem_samples is not None: torch.accelerator.synchronize() free_before = torch.accelerator.get_memory_info()[0] - with torch.cuda.graph( - graph, self.pool, stream=self._capture_stream(desc) + with ( + capture_pool(self.pool, self.vllm_config) as pool, + torch.cuda.graph( + graph, pool, stream=self._capture_stream(desc) + ), ): forward_fn(CUDAGraphMode.NONE) # Join offloader's copy stream after forward to avoid @@ -941,7 +943,8 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int: mem_samples: list[int] = [] manager._capture_mem_samples = mem_samples - measured = int(runner.capture_model(profile_only=True)) + with capture_outside_cumem_pool(): + measured = int(runner.capture_model(profile_only=True)) # The measured delta covers PIECEWISE, encoder and speculator graphs # plus the sampled FULL graphs; swap the sampled FULL cost for the