[Feature] Release the CUDA graph pool on sleep (#59160)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9e0b6fcf77
commit
5e56e9af2a
@@ -101,6 +101,41 @@ llm.release_kv_cache_memory()
|
||||
llm.wake_up(tags=["kv_cache"]) # Reallocate KV cache and resume scheduling.
|
||||
```
|
||||
|
||||
#### Offloading CUDA graph memory
|
||||
|
||||
By default, CUDA graph memory stays on the GPU while asleep. With
|
||||
`sleep_mode_offload_cudagraph=True` (off by default), CUDA graphs are captured
|
||||
into a cuMem pool that sleep backs up to CPU memory at both levels and any wake
|
||||
restores in place, so graphs are replayed, not recaptured. It needs the default
|
||||
`cumem` backend, CUDA and CUDA graphs; otherwise it has no effect.
|
||||
|
||||
```python
|
||||
llm = LLM("Qwen/Qwen3-8B", enable_sleep_mode=True, sleep_mode_offload_cudagraph=True)
|
||||
```
|
||||
|
||||
or `vllm serve <model> --enable-sleep-mode --sleep-mode-offload-cudagraph`.
|
||||
|
||||
The cost is pinned host memory for the pool's backup, also at level 2, and one
|
||||
extra copy per captured custom allreduce (about 3% decode latency at batch
|
||||
size 1). `NCCL_GRAPH_REGISTER` defaults to `0`, since NCCL graph registration
|
||||
would pin the pool; an explicit value is kept with a warning. Graph executables
|
||||
outside PyTorch pools stay resident.
|
||||
|
||||
#### Releasing NCCL communicator memory
|
||||
|
||||
By default, NCCL communicators keep their GPU buffers while asleep. With
|
||||
`enable_nccl_comm_suspend=True` (off by default, experimental), sleep releases
|
||||
them with `ncclCommSuspend` and wake restores them with `ncclCommResume`. The
|
||||
communicators keep their topology, so they are not re-created. It needs NCCL
|
||||
2.29.7 or newer; with an older library, a warning is logged and the memory stays
|
||||
on the GPU.
|
||||
|
||||
```python
|
||||
llm = LLM("Qwen/Qwen3-8B", enable_sleep_mode=True, enable_nccl_comm_suspend=True)
|
||||
```
|
||||
|
||||
or `vllm serve <model> --enable-sleep-mode --enable-nccl-comm-suspend`.
|
||||
|
||||
### Online Serving
|
||||
|
||||
To enable sleep mode in a vLLM server you need to initialize it with the flag `VLLM_SERVER_DEV_MODE=1` and pass `--enable-sleep-mode` to the vLLM server.
|
||||
|
||||
@@ -423,3 +423,58 @@ def test_cumem_with_cudagraph():
|
||||
|
||||
# output content is as expected
|
||||
assert torch.allclose(y, x + 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
|
||||
def test_cudagraph_pool_sleep(level):
|
||||
"""Routing, and the graph pool backed up at both sleep levels and restored
|
||||
in place by any wake."""
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc
|
||||
from vllm.compilation.cudagraph_pool import (
|
||||
capture_outside_cumem_pool,
|
||||
capture_pool,
|
||||
)
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False))
|
||||
handle = current_platform.graph_pool_handle()
|
||||
for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())):
|
||||
with ctx, capture_pool(handle, cfg) as used:
|
||||
assert used == handle and allocator.current_tag != "cudagraph"
|
||||
with allocator.use_memory_pool("weights"):
|
||||
weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE)
|
||||
x = torch.ones_like(weight)
|
||||
|
||||
def capture() -> list:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
stream = torch.cuda.Stream()
|
||||
with (
|
||||
capture_pool(handle, on) as pool,
|
||||
torch.cuda.graph(graph, pool=pool, stream=stream),
|
||||
):
|
||||
assert nccl_alloc._graph_pool_id == pool != handle
|
||||
const = torch.empty_like(x) # Never written by replay: needs a backup.
|
||||
y = x * weight + const
|
||||
return [graph, y, const.fill_(3.0)]
|
||||
|
||||
def graph_pool() -> dict[int, int]:
|
||||
data = allocator.pointer_to_data.items()
|
||||
return {
|
||||
p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep
|
||||
}
|
||||
|
||||
held, backend = capture(), CuMemBackend()
|
||||
mapped = graph_pool()
|
||||
backend.suspend(level=level)
|
||||
backend.resume(tags=["kv_cache"])
|
||||
assert graph_pool() == mapped
|
||||
backend.resume(tags=["weights"])
|
||||
weight.fill_(2.0) # Level 2 discards weights; emulate the reload.
|
||||
held[0].replay()
|
||||
assert torch.equal(held[1], torch.full_like(x, 5.0))
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -14,7 +15,12 @@ from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
|
||||
from ....utils import (
|
||||
create_new_process_for_each_test,
|
||||
multi_gpu_marks,
|
||||
multi_gpu_test,
|
||||
requires_fp8,
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@@ -195,6 +201,73 @@ def test_deep_sleep_lora_tp2(monkeypatch):
|
||||
assert output[0].outputs[0].text == output2[0].outputs[0].text
|
||||
|
||||
|
||||
def _cudagraph_bytes(worker) -> int:
|
||||
data = cumem.CuMemAllocator.get_instance().pointer_to_data.values()
|
||||
return sum(d.handle[1] for d in data if d.tag == "cudagraph")
|
||||
|
||||
|
||||
def _custom_ar_active(worker) -> bool:
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
ca_comm = get_tp_group().device_communicator.ca_comm
|
||||
return ca_comm is not None and not ca_comm.disabled
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "breakable", "tp", "offload"),
|
||||
[
|
||||
("FULL", False, 1, True),
|
||||
("PIECEWISE", False, 1, True),
|
||||
("PIECEWISE", True, 1, True),
|
||||
pytest.param(
|
||||
"FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2)
|
||||
),
|
||||
("FULL_AND_PIECEWISE", False, 1, False),
|
||||
],
|
||||
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
|
||||
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
|
||||
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
|
||||
unregistered custom AR; off changes nothing."""
|
||||
for name, value in [
|
||||
("VLLM_USE_V2_MODEL_RUNNER", "1"),
|
||||
("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"),
|
||||
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
|
||||
("VLLM_ALLREDUCE_USE_FLASHINFER", "0"),
|
||||
("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"),
|
||||
]:
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
|
||||
llm = LLM(
|
||||
"hmellor/tiny-random-LlamaForCausalLM",
|
||||
enable_sleep_mode=True,
|
||||
sleep_mode_offload_cudagraph=offload,
|
||||
tensor_parallel_size=tp,
|
||||
compilation_config={
|
||||
"cudagraph_mode": mode,
|
||||
"pass_config": {"fuse_allreduce_rms": False},
|
||||
},
|
||||
)
|
||||
graph_bytes = llm.collective_rpc(_cudagraph_bytes)
|
||||
assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None)
|
||||
assert all(graph_bytes) if offload else not any(graph_bytes)
|
||||
if not offload:
|
||||
return
|
||||
if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)):
|
||||
pytest.skip("Custom allreduce is unavailable on these GPUs")
|
||||
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, params)[0].outputs[0].text
|
||||
for level in (1, 2):
|
||||
llm.sleep(level=level)
|
||||
llm.wake_up(tags=["weights"])
|
||||
if level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_async(gpu_memory_cleared):
|
||||
async def test():
|
||||
|
||||
@@ -177,7 +177,9 @@ def _worker(rank: int, tp_size: int, port: int) -> None:
|
||||
# Supply the resolved Engram settings without constructing a full model.
|
||||
vllm_config.engram_config = EngramConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
architecture="DeepseekV41ForCausalLM", is_moe=True
|
||||
architecture="DeepseekV41ForCausalLM",
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
try:
|
||||
init_distributed_environment()
|
||||
|
||||
@@ -50,6 +50,7 @@ def _create_vllm_config() -> MagicMock:
|
||||
vllm_config.parallel_config = ParallelConfig()
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.num_speculative_tokens = 0
|
||||
vllm_config.use_cumem_cudagraph_pool = False
|
||||
return vllm_config
|
||||
|
||||
|
||||
|
||||
@@ -34,9 +34,9 @@ from typing import Any, ClassVar, TypeVar
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.cudagraph_pool import capture_pool
|
||||
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
get_forward_context,
|
||||
@@ -370,11 +370,6 @@ class BreakableCUDAGraphWrapper:
|
||||
|
||||
entry.input_addresses = self._collect_tensor_addresses(args, kwargs)
|
||||
|
||||
if self.graph_pool is not None:
|
||||
set_graph_pool_id(self.graph_pool)
|
||||
else:
|
||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||
|
||||
# Match torch.cuda.graph()'s pre-capture cleanup, which we bypass.
|
||||
# Skip it when gc is disabled: bulk capture runs under
|
||||
# freeze_gc_for_cudagraph_capture, which already did this cleanup,
|
||||
@@ -386,8 +381,10 @@ class BreakableCUDAGraphWrapper:
|
||||
# pre-capture prefetches are complete and don't leak into the graph.
|
||||
get_offloader().sync_prev_onload()
|
||||
|
||||
capture = BreakableCUDAGraphCapture(pool=self.graph_pool)
|
||||
with capture:
|
||||
with (
|
||||
capture_pool(self.graph_pool, self.vllm_config) as pool,
|
||||
BreakableCUDAGraphCapture(pool=pool) as capture,
|
||||
):
|
||||
output = self.runnable(*args, **kwargs)
|
||||
# Join the offloader's copy stream while we still hold the last
|
||||
# segment open, so the join is captured into the graph (otherwise
|
||||
|
||||
@@ -13,9 +13,9 @@ from unittest.mock import patch
|
||||
import torch
|
||||
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cudagraph_pool import capture_pool
|
||||
from vllm.compilation.monitor import validate_cudagraph_capturing_enabled
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.forward_context import (
|
||||
BatchDescriptor,
|
||||
get_forward_context,
|
||||
@@ -302,10 +302,9 @@ class CUDAGraphWrapper:
|
||||
)
|
||||
)
|
||||
|
||||
if self.graph_pool is not None:
|
||||
set_graph_pool_id(self.graph_pool)
|
||||
else:
|
||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||
graph_pool = stack.enter_context(
|
||||
capture_pool(self.graph_pool, self.vllm_config)
|
||||
)
|
||||
|
||||
# Sync offloader's copy stream before capture.
|
||||
# Ensure any pre-capture prefetches from offloader are complete.
|
||||
@@ -314,7 +313,7 @@ class CUDAGraphWrapper:
|
||||
# mind-exploding: carefully manage the reference and memory.
|
||||
with torch.cuda.graph(
|
||||
cudagraph,
|
||||
pool=self.graph_pool,
|
||||
pool=graph_pool,
|
||||
stream=current_stream(),
|
||||
):
|
||||
# `output` is managed by pytorch's cudagraph pool
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Graph pool routing for CUDA graph capture under sleep mode."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.device_allocator import get_mem_allocator_instance
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
_outside_cumem: ContextVar[bool] = ContextVar("capture_outside_cumem", default=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def capture_outside_cumem_pool() -> Iterator[None]:
|
||||
"""CUDA graphs captured here stay out of cuMem, so that memory profiling can
|
||||
free its throwaway graphs normally."""
|
||||
token = _outside_cumem.set(True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_outside_cumem.reset(token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def capture_pool(
|
||||
pool: tuple[int, int] | None, vllm_config: VllmConfig
|
||||
) -> Iterator[tuple[int, int] | None]:
|
||||
"""Yield the pool to capture into, the cuMem graph pool when sleep offloads
|
||||
graph memory, and point NCCL's graph allocator at it."""
|
||||
if vllm_config.use_cumem_cudagraph_pool and not _outside_cumem.get():
|
||||
allocator = cast("CuMemAllocator", get_mem_allocator_instance())
|
||||
with allocator.cudagraph_pool() as cumem_pool:
|
||||
set_graph_pool_id(cumem_pool)
|
||||
yield cumem_pool
|
||||
else:
|
||||
set_graph_pool_id(pool or current_platform.graph_pool_handle())
|
||||
yield pool
|
||||
@@ -378,6 +378,9 @@ class ModelConfig:
|
||||
(default) uses the built-in ``CuMemAllocator`` and is behavior-compatible
|
||||
with prior releases. Additional backends (CUDA checkpoint, CRIU, durable
|
||||
snapshot) may be registered in-tree or by plugins (RFC #34303)."""
|
||||
sleep_mode_offload_cudagraph: bool = False
|
||||
"""Back up CUDA graph memory to CPU during sleep, restored in place on wake.
|
||||
Takes effect with enable_sleep_mode, the cumem backend and CUDA graphs."""
|
||||
enable_nccl_comm_suspend: bool = False
|
||||
"""Enable releasing NCCL communicator memory during sleep mode
|
||||
(``ncclCommSuspend``/``ncclCommResume``). Experimental; when disabled
|
||||
|
||||
@@ -697,6 +697,21 @@ class VllmConfig:
|
||||
"""
|
||||
return 1 + self.num_speculative_tokens
|
||||
|
||||
@property
|
||||
def use_cumem_cudagraph_pool(self) -> bool:
|
||||
"""Whether CUDA graphs go to the cuMem pool that sleep offloads."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
model_config = self.model_config
|
||||
return (
|
||||
model_config is not None
|
||||
and model_config.sleep_mode_offload_cudagraph
|
||||
and model_config.enable_sleep_mode
|
||||
and model_config.sleep_mode_backend == "cumem"
|
||||
and self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
|
||||
and current_platform.is_cuda()
|
||||
)
|
||||
|
||||
@property
|
||||
def use_v2_model_runner(self) -> bool:
|
||||
if self.attention_config.hisparse_config is not None:
|
||||
@@ -2321,6 +2336,14 @@ class VllmConfig:
|
||||
custom_ops.append("+quant_fp8")
|
||||
|
||||
self._verify_kv_transfer_compat()
|
||||
if self.use_cumem_cudagraph_pool:
|
||||
# NCCL graph registration pins the offloaded pool; workers inherit this.
|
||||
value = os.environ.setdefault("NCCL_GRAPH_REGISTER", "0")
|
||||
if value != "0":
|
||||
logger.warning(
|
||||
"NCCL_GRAPH_REGISTER=%s pins the CUDA graph pool during sleep.",
|
||||
value,
|
||||
)
|
||||
# Log the custom passes that are enabled
|
||||
self.compilation_config.pass_config.log_enabled_passes()
|
||||
|
||||
|
||||
@@ -430,6 +430,23 @@ class CuMemAllocator:
|
||||
if allocation["allocated_size"] == 0 and data and not data.is_asleep:
|
||||
unmap_and_release(self._python_free_callback(allocation["address"]))
|
||||
|
||||
@contextmanager
|
||||
def cudagraph_pool(self) -> Iterator[tuple[int, int]]:
|
||||
"""Tag CUDA graph capture allocations and yield the graph pool id."""
|
||||
if "cudagraph" not in self.allocator_and_pools:
|
||||
allocator = get_pluggable_allocator(
|
||||
self.python_malloc_callback, self.python_free_callback
|
||||
)
|
||||
mem_pool = torch.cuda.memory.MemPool(allocator._allocator)
|
||||
self.allocator_and_pools["cudagraph"] = [(mem_pool, allocator)]
|
||||
old_tag = self.current_tag
|
||||
self.current_tag = "cudagraph"
|
||||
try:
|
||||
# capture_begin routes allocations to this pool; no use_mem_pool.
|
||||
yield self.allocator_and_pools["cudagraph"][0][0].id
|
||||
finally:
|
||||
self.current_tag = old_tag
|
||||
|
||||
def get_current_usage(self) -> int:
|
||||
"""Get the total number of bytes allocated in the memory pool."""
|
||||
sum_bytes: int = 0
|
||||
|
||||
@@ -129,8 +129,11 @@ class CuMemBackend(SleepModeBackend):
|
||||
|
||||
self._state = "SUSPENDED"
|
||||
allocator = get_mem_allocator_instance()
|
||||
# Runtime state is kept at every level; weights only at level 1.
|
||||
offload_tags = ("weights", "runtime") if level == 1 else ("runtime",)
|
||||
# Runtime state and graph pools are kept at every level; weights only
|
||||
# at level 1.
|
||||
offload_tags: tuple[str, ...] = ("runtime", "cudagraph")
|
||||
if level == 1:
|
||||
offload_tags = ("weights", *offload_tags)
|
||||
allocator.sleep(offload_tags=offload_tags)
|
||||
|
||||
def resume(self, tags: list[str] | None = None) -> None:
|
||||
|
||||
@@ -7,6 +7,7 @@ from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.distributed.device_communicators.all_reduce_utils import (
|
||||
NCCL_SYMM_MEM_ALL_REDUCE_CONFIG,
|
||||
should_nccl_symm_mem_ag_rs,
|
||||
@@ -114,6 +115,10 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.aiter_ar_comm: AiterCustomAllreduce | None = None
|
||||
self.use_aiter_ag_rs: bool = False
|
||||
|
||||
# cuMem graph buffers cannot be IPC-registered; capture copies them instead.
|
||||
config = get_current_vllm_config_or_none()
|
||||
register = config is None or not config.use_cumem_cudagraph_pool
|
||||
|
||||
if use_torch_symm_mem and current_platform.is_cuda():
|
||||
self.symm_mem_comm = SymmMemCommunicator(
|
||||
group=self.cpu_group,
|
||||
@@ -151,6 +156,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
symm_mem_enabled=(
|
||||
self.symm_mem_comm is not None and not self.symm_mem_comm.disabled
|
||||
),
|
||||
register_graph_buffers=register,
|
||||
)
|
||||
|
||||
# AITER custom all-gather/reduce-scatter DP-attention dispatch/combine
|
||||
|
||||
@@ -147,18 +147,22 @@ class CustomAllreduce:
|
||||
_DEFAULT_MNNVL_MULTIMEM_REDUCE_SCATTER_MAX_SIZE
|
||||
),
|
||||
symm_mem_enabled=False,
|
||||
*,
|
||||
register_graph_buffers: bool = True,
|
||||
) -> None:
|
||||
"""Args:
|
||||
group: the process group to work on. If None, it will use the
|
||||
default process group.
|
||||
device: the device to bind the CustomAllreduce to. If None,
|
||||
it will be bound to f"cuda:{local_rank}".
|
||||
register_graph_buffers: whether graph capture IPC-registers its buffers.
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device, and all communicators in this group
|
||||
are in the same node.
|
||||
|
||||
"""
|
||||
self._IS_CAPTURING = False
|
||||
self._capture_registered = register_graph_buffers
|
||||
self._ptr = 0
|
||||
self.disabled = True
|
||||
self.mnnvl_buffer = None
|
||||
@@ -549,7 +553,7 @@ class CustomAllreduce:
|
||||
return None
|
||||
if self._IS_CAPTURING:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return self.all_reduce(input, registered=True)
|
||||
return self.all_reduce(input, registered=self._capture_registered)
|
||||
else:
|
||||
# If warm up, mimic the allocation pattern since custom
|
||||
# allreduce is out-of-place.
|
||||
|
||||
@@ -770,6 +770,7 @@ class EngineArgs:
|
||||
|
||||
generation_config: str = ModelConfig.generation_config
|
||||
enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
|
||||
sleep_mode_offload_cudagraph: bool = ModelConfig.sleep_mode_offload_cudagraph
|
||||
sleep_preserve_parameter_names: list[str] = get_field(
|
||||
ModelConfig, "sleep_preserve_parameter_names"
|
||||
)
|
||||
@@ -1007,6 +1008,10 @@ class EngineArgs:
|
||||
model_group.add_argument(
|
||||
"--enable-sleep-mode", **model_kwargs["enable_sleep_mode"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--sleep-mode-offload-cudagraph",
|
||||
**model_kwargs["sleep_mode_offload_cudagraph"],
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--sleep-preserve-parameter-names",
|
||||
**model_kwargs["sleep_preserve_parameter_names"],
|
||||
@@ -1984,6 +1989,7 @@ class EngineArgs:
|
||||
generation_config=self.generation_config,
|
||||
override_generation_config=self.override_generation_config,
|
||||
enable_sleep_mode=self.enable_sleep_mode,
|
||||
sleep_mode_offload_cudagraph=self.sleep_mode_offload_cudagraph,
|
||||
sleep_preserve_parameter_names=self.sleep_preserve_parameter_names,
|
||||
enable_cumem_allocator=self.enable_cumem_allocator,
|
||||
enable_nccl_comm_suspend=self.enable_nccl_comm_suspend,
|
||||
|
||||
@@ -19,10 +19,13 @@ from vllm.compilation.breakable_cudagraph import (
|
||||
)
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat, CUDAGraphWrapper
|
||||
from vllm.compilation.cudagraph_pool import (
|
||||
capture_outside_cumem_pool,
|
||||
capture_pool,
|
||||
)
|
||||
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pp_group,
|
||||
graph_capture,
|
||||
@@ -481,15 +484,14 @@ class CudaGraphManager:
|
||||
# Sync offloader's copy stream before capture.
|
||||
# Ensure any pre-capture prefetches from offloader are complete.
|
||||
get_offloader().sync_prev_onload()
|
||||
if self.pool is not None:
|
||||
set_graph_pool_id(self.pool)
|
||||
else:
|
||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||
if self._capture_mem_samples is not None:
|
||||
torch.accelerator.synchronize()
|
||||
free_before = torch.accelerator.get_memory_info()[0]
|
||||
with torch.cuda.graph(
|
||||
graph, self.pool, stream=self._capture_stream(desc)
|
||||
with (
|
||||
capture_pool(self.pool, self.vllm_config) as pool,
|
||||
torch.cuda.graph(
|
||||
graph, pool, stream=self._capture_stream(desc)
|
||||
),
|
||||
):
|
||||
forward_fn(CUDAGraphMode.NONE)
|
||||
# Join offloader's copy stream after forward to avoid
|
||||
@@ -941,7 +943,8 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
|
||||
mem_samples: list[int] = []
|
||||
manager._capture_mem_samples = mem_samples
|
||||
|
||||
measured = int(runner.capture_model(profile_only=True))
|
||||
with capture_outside_cumem_pool():
|
||||
measured = int(runner.capture_model(profile_only=True))
|
||||
|
||||
# The measured delta covers PIECEWISE, encoder and speculator graphs
|
||||
# plus the sampled FULL graphs; swap the sampled FULL cost for the
|
||||
|
||||
Reference in New Issue
Block a user