[Feature] Release the CUDA graph pool on sleep (#59160)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-04 01:13:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9e0b6fcf77
commit 5e56e9af2a
16 changed files with 299 additions and 27 deletions
+5 -6
View File
@@ -13,9 +13,9 @@ from unittest.mock import patch
import torch
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cudagraph_pool import capture_pool
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.forward_context import (
BatchDescriptor,
get_forward_context,
@@ -302,10 +302,9 @@ class CUDAGraphWrapper:
)
)
if self.graph_pool is not None:
set_graph_pool_id(self.graph_pool)
else:
set_graph_pool_id(current_platform.graph_pool_handle())
graph_pool = stack.enter_context(
capture_pool(self.graph_pool, self.vllm_config)
)
# Sync offloader's copy stream before capture.
# Ensure any pre-capture prefetches from offloader are complete.
@@ -314,7 +313,7 @@ class CUDAGraphWrapper:
# mind-exploding: carefully manage the reference and memory.
with torch.cuda.graph(
cudagraph,
pool=self.graph_pool,
pool=graph_pool,
stream=current_stream(),
):
# `output` is managed by pytorch's cudagraph pool