[Feature] Release the CUDA graph pool on sleep (#59160)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-04 01:13:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9e0b6fcf77
commit 5e56e9af2a
16 changed files with 299 additions and 27 deletions
@@ -7,6 +7,7 @@ from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import get_current_vllm_config_or_none
from vllm.distributed.device_communicators.all_reduce_utils import (
NCCL_SYMM_MEM_ALL_REDUCE_CONFIG,
should_nccl_symm_mem_ag_rs,
@@ -114,6 +115,10 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.aiter_ar_comm: AiterCustomAllreduce | None = None
self.use_aiter_ag_rs: bool = False
# cuMem graph buffers cannot be IPC-registered; capture copies them instead.
config = get_current_vllm_config_or_none()
register = config is None or not config.use_cumem_cudagraph_pool
if use_torch_symm_mem and current_platform.is_cuda():
self.symm_mem_comm = SymmMemCommunicator(
group=self.cpu_group,
@@ -151,6 +156,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
symm_mem_enabled=(
self.symm_mem_comm is not None and not self.symm_mem_comm.disabled
),
register_graph_buffers=register,
)
# AITER custom all-gather/reduce-scatter DP-attention dispatch/combine
@@ -147,18 +147,22 @@ class CustomAllreduce:
_DEFAULT_MNNVL_MULTIMEM_REDUCE_SCATTER_MAX_SIZE
),
symm_mem_enabled=False,
*,
register_graph_buffers: bool = True,
) -> None:
"""Args:
group: the process group to work on. If None, it will use the
default process group.
device: the device to bind the CustomAllreduce to. If None,
it will be bound to f"cuda:{local_rank}".
register_graph_buffers: whether graph capture IPC-registers its buffers.
It is the caller's responsibility to make sure each communicator
is bind to a unique device, and all communicators in this group
are in the same node.
"""
self._IS_CAPTURING = False
self._capture_registered = register_graph_buffers
self._ptr = 0
self.disabled = True
self.mnnvl_buffer = None
@@ -549,7 +553,7 @@ class CustomAllreduce:
return None
if self._IS_CAPTURING:
if torch.cuda.is_current_stream_capturing():
return self.all_reduce(input, registered=True)
return self.all_reduce(input, registered=self._capture_registered)
else:
# If warm up, mimic the allocation pattern since custom
# allreduce is out-of-place.