[Feature] Release the CUDA graph pool on sleep (#59160)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9e0b6fcf77
commit
5e56e9af2a
@@ -7,6 +7,7 @@ from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.distributed.device_communicators.all_reduce_utils import (
|
||||
NCCL_SYMM_MEM_ALL_REDUCE_CONFIG,
|
||||
should_nccl_symm_mem_ag_rs,
|
||||
@@ -114,6 +115,10 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.aiter_ar_comm: AiterCustomAllreduce | None = None
|
||||
self.use_aiter_ag_rs: bool = False
|
||||
|
||||
# cuMem graph buffers cannot be IPC-registered; capture copies them instead.
|
||||
config = get_current_vllm_config_or_none()
|
||||
register = config is None or not config.use_cumem_cudagraph_pool
|
||||
|
||||
if use_torch_symm_mem and current_platform.is_cuda():
|
||||
self.symm_mem_comm = SymmMemCommunicator(
|
||||
group=self.cpu_group,
|
||||
@@ -151,6 +156,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
symm_mem_enabled=(
|
||||
self.symm_mem_comm is not None and not self.symm_mem_comm.disabled
|
||||
),
|
||||
register_graph_buffers=register,
|
||||
)
|
||||
|
||||
# AITER custom all-gather/reduce-scatter DP-attention dispatch/combine
|
||||
|
||||
@@ -147,18 +147,22 @@ class CustomAllreduce:
|
||||
_DEFAULT_MNNVL_MULTIMEM_REDUCE_SCATTER_MAX_SIZE
|
||||
),
|
||||
symm_mem_enabled=False,
|
||||
*,
|
||||
register_graph_buffers: bool = True,
|
||||
) -> None:
|
||||
"""Args:
|
||||
group: the process group to work on. If None, it will use the
|
||||
default process group.
|
||||
device: the device to bind the CustomAllreduce to. If None,
|
||||
it will be bound to f"cuda:{local_rank}".
|
||||
register_graph_buffers: whether graph capture IPC-registers its buffers.
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device, and all communicators in this group
|
||||
are in the same node.
|
||||
|
||||
"""
|
||||
self._IS_CAPTURING = False
|
||||
self._capture_registered = register_graph_buffers
|
||||
self._ptr = 0
|
||||
self.disabled = True
|
||||
self.mnnvl_buffer = None
|
||||
@@ -549,7 +553,7 @@ class CustomAllreduce:
|
||||
return None
|
||||
if self._IS_CAPTURING:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return self.all_reduce(input, registered=True)
|
||||
return self.all_reduce(input, registered=self._capture_registered)
|
||||
else:
|
||||
# If warm up, mimic the allocation pattern since custom
|
||||
# allreduce is out-of-place.
|
||||
|
||||
Reference in New Issue
Block a user