[Feature] Release the CUDA graph pool on sleep (#59160)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9e0b6fcf77
commit
5e56e9af2a
@@ -34,9 +34,9 @@ from typing import Any, ClassVar, TypeVar
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.cudagraph_pool import capture_pool
|
||||
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
get_forward_context,
|
||||
@@ -370,11 +370,6 @@ class BreakableCUDAGraphWrapper:
|
||||
|
||||
entry.input_addresses = self._collect_tensor_addresses(args, kwargs)
|
||||
|
||||
if self.graph_pool is not None:
|
||||
set_graph_pool_id(self.graph_pool)
|
||||
else:
|
||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||
|
||||
# Match torch.cuda.graph()'s pre-capture cleanup, which we bypass.
|
||||
# Skip it when gc is disabled: bulk capture runs under
|
||||
# freeze_gc_for_cudagraph_capture, which already did this cleanup,
|
||||
@@ -386,8 +381,10 @@ class BreakableCUDAGraphWrapper:
|
||||
# pre-capture prefetches are complete and don't leak into the graph.
|
||||
get_offloader().sync_prev_onload()
|
||||
|
||||
capture = BreakableCUDAGraphCapture(pool=self.graph_pool)
|
||||
with capture:
|
||||
with (
|
||||
capture_pool(self.graph_pool, self.vllm_config) as pool,
|
||||
BreakableCUDAGraphCapture(pool=pool) as capture,
|
||||
):
|
||||
output = self.runnable(*args, **kwargs)
|
||||
# Join the offloader's copy stream while we still hold the last
|
||||
# segment open, so the join is captured into the graph (otherwise
|
||||
|
||||
@@ -13,9 +13,9 @@ from unittest.mock import patch
|
||||
import torch
|
||||
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cudagraph_pool import capture_pool
|
||||
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
get_forward_context,
|
||||
@@ -302,10 +302,9 @@ class CUDAGraphWrapper:
|
||||
)
|
||||
)
|
||||
|
||||
if self.graph_pool is not None:
|
||||
set_graph_pool_id(self.graph_pool)
|
||||
else:
|
||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||
graph_pool = stack.enter_context(
|
||||
capture_pool(self.graph_pool, self.vllm_config)
|
||||
)
|
||||
|
||||
# Sync offloader's copy stream before capture.
|
||||
# Ensure any pre-capture prefetches from offloader are complete.
|
||||
@@ -314,7 +313,7 @@ class CUDAGraphWrapper:
|
||||
# mind-exploding: carefully manage the reference and memory.
|
||||
with torch.cuda.graph(
|
||||
cudagraph,
|
||||
pool=self.graph_pool,
|
||||
pool=graph_pool,
|
||||
stream=current_stream(),
|
||||
):
|
||||
# `output` is managed by pytorch's cudagraph pool
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Graph pool routing for CUDA graph capture under sleep mode."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
_outside_cumem: ContextVar[bool] = ContextVar("capture_outside_cumem", default=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def capture_outside_cumem_pool() -> Iterator[None]:
|
||||
"""CUDA graphs captured here stay out of cuMem, so that memory profiling can
|
||||
free its throwaway graphs normally."""
|
||||
token = _outside_cumem.set(True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_outside_cumem.reset(token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def capture_pool(
|
||||
pool: tuple[int, int] | None, vllm_config: VllmConfig
|
||||
) -> Iterator[tuple[int, int] | None]:
|
||||
"""Yield the pool to capture into, the cuMem graph pool when sleep offloads
|
||||
graph memory, and point NCCL's graph allocator at it."""
|
||||
if vllm_config.use_cumem_cudagraph_pool and not _outside_cumem.get():
|
||||
allocator = cast("CuMemAllocator", get_mem_allocator_instance())
|
||||
with allocator.cudagraph_pool() as cumem_pool:
|
||||
set_graph_pool_id(cumem_pool)
|
||||
yield cumem_pool
|
||||
else:
|
||||
set_graph_pool_id(pool or current_platform.graph_pool_handle())
|
||||
yield pool
|
||||
Reference in New Issue
Block a user