[Feature] Release the CUDA graph pool on sleep (#59160)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9e0b6fcf77
commit
5e56e9af2a
@@ -13,9 +13,9 @@ from unittest.mock import patch
|
||||
import torch
|
||||
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cudagraph_pool import capture_pool
|
||||
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
get_forward_context,
|
||||
@@ -302,10 +302,9 @@ class CUDAGraphWrapper:
|
||||
)
|
||||
)
|
||||
|
||||
if self.graph_pool is not None:
|
||||
set_graph_pool_id(self.graph_pool)
|
||||
else:
|
||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||
graph_pool = stack.enter_context(
|
||||
capture_pool(self.graph_pool, self.vllm_config)
|
||||
)
|
||||
|
||||
# Sync offloader's copy stream before capture.
|
||||
# Ensure any pre-capture prefetches from offloader are complete.
|
||||
@@ -314,7 +313,7 @@ class CUDAGraphWrapper:
|
||||
# mind-exploding: carefully manage the reference and memory.
|
||||
with torch.cuda.graph(
|
||||
cudagraph,
|
||||
pool=self.graph_pool,
|
||||
pool=graph_pool,
|
||||
stream=current_stream(),
|
||||
):
|
||||
# `output` is managed by pytorch's cudagraph pool
|
||||
|
||||
Reference in New Issue
Block a user