[Feature] Release the CUDA graph pool on sleep (#59160)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-04 01:13:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9e0b6fcf77
commit 5e56e9af2a
16 changed files with 299 additions and 27 deletions
+5 -8
View File
@@ -34,9 +34,9 @@ from typing import Any, ClassVar, TypeVar
import torch
import vllm.envs as envs
from vllm.compilation.cudagraph_pool import capture_pool
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.forward_context import (
BatchDescriptor,
get_forward_context,
@@ -370,11 +370,6 @@ class BreakableCUDAGraphWrapper:
entry.input_addresses = self._collect_tensor_addresses(args, kwargs)
if self.graph_pool is not None:
set_graph_pool_id(self.graph_pool)
else:
set_graph_pool_id(current_platform.graph_pool_handle())
# Match torch.cuda.graph()'s pre-capture cleanup, which we bypass.
# Skip it when gc is disabled: bulk capture runs under
# freeze_gc_for_cudagraph_capture, which already did this cleanup,
@@ -386,8 +381,10 @@ class BreakableCUDAGraphWrapper:
# pre-capture prefetches are complete and don't leak into the graph.
get_offloader().sync_prev_onload()
capture = BreakableCUDAGraphCapture(pool=self.graph_pool)
with capture:
with (
capture_pool(self.graph_pool, self.vllm_config) as pool,
BreakableCUDAGraphCapture(pool=pool) as capture,
):
output = self.runnable(*args, **kwargs)
# Join the offloader's copy stream while we still hold the last
# segment open, so the join is captured into the graph (otherwise
+5 -6
View File
@@ -13,9 +13,9 @@ from unittest.mock import patch
import torch
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cudagraph_pool import capture_pool
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.forward_context import (
BatchDescriptor,
get_forward_context,
@@ -302,10 +302,9 @@ class CUDAGraphWrapper:
)
)
if self.graph_pool is not None:
set_graph_pool_id(self.graph_pool)
else:
set_graph_pool_id(current_platform.graph_pool_handle())
graph_pool = stack.enter_context(
capture_pool(self.graph_pool, self.vllm_config)
)
# Sync offloader's copy stream before capture.
# Ensure any pre-capture prefetches from offloader are complete.
@@ -314,7 +313,7 @@ class CUDAGraphWrapper:
# mind-exploding: carefully manage the reference and memory.
with torch.cuda.graph(
cudagraph,
pool=self.graph_pool,
pool=graph_pool,
stream=current_stream(),
):
# `output` is managed by pytorch's cudagraph pool
+45
View File
@@ -0,0 +1,45 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Graph pool routing for CUDA graph capture under sleep mode."""
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from typing import TYPE_CHECKING, cast
from vllm.config import VllmConfig
from vllm.device_allocator import get_mem_allocator_instance
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.device_allocator.cumem import CuMemAllocator
_outside_cumem: ContextVar[bool] = ContextVar("capture_outside_cumem", default=False)
@contextmanager
def capture_outside_cumem_pool() -> Iterator[None]:
"""CUDA graphs captured here stay out of cuMem, so that memory profiling can
free its throwaway graphs normally."""
token = _outside_cumem.set(True)
try:
yield
finally:
_outside_cumem.reset(token)
@contextmanager
def capture_pool(
pool: tuple[int, int] | None, vllm_config: VllmConfig
) -> Iterator[tuple[int, int] | None]:
"""Yield the pool to capture into, the cuMem graph pool when sleep offloads
graph memory, and point NCCL's graph allocator at it."""
if vllm_config.use_cumem_cudagraph_pool and not _outside_cumem.get():
allocator = cast("CuMemAllocator", get_mem_allocator_instance())
with allocator.cudagraph_pool() as cumem_pool:
set_graph_pool_id(cumem_pool)
yield cumem_pool
else:
set_graph_pool_id(pool or current_platform.graph_pool_handle())
yield pool