[Feature] Release the CUDA graph pool on sleep (#59160)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9e0b6fcf77
commit
5e56e9af2a
@@ -423,3 +423,58 @@ def test_cumem_with_cudagraph():
|
||||
|
||||
# output content is as expected
|
||||
assert torch.allclose(y, x + 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
|
||||
def test_cudagraph_pool_sleep(level):
|
||||
"""Routing, and the graph pool backed up at both sleep levels and restored
|
||||
in place by any wake."""
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
|
||||
import vllm.distributed.device_communicators.pynccl_allocator as nccl_alloc
|
||||
from vllm.compilation.cudagraph_pool import (
|
||||
capture_outside_cumem_pool,
|
||||
capture_pool,
|
||||
)
|
||||
from vllm.device_allocator.sleep_mode_backend import CuMemBackend
|
||||
|
||||
allocator = get_mem_allocator_instance()
|
||||
on, off = (SimpleNamespace(use_cumem_cudagraph_pool=v) for v in (True, False))
|
||||
handle = current_platform.graph_pool_handle()
|
||||
for cfg, ctx in ((off, nullcontext()), (on, capture_outside_cumem_pool())):
|
||||
with ctx, capture_pool(handle, cfg) as used:
|
||||
assert used == handle and allocator.current_tag != "cudagraph"
|
||||
with allocator.use_memory_pool("weights"):
|
||||
weight = torch.full((1 << 20,), 2.0, device=DEVICE_TYPE)
|
||||
x = torch.ones_like(weight)
|
||||
|
||||
def capture() -> list:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
stream = torch.cuda.Stream()
|
||||
with (
|
||||
capture_pool(handle, on) as pool,
|
||||
torch.cuda.graph(graph, pool=pool, stream=stream),
|
||||
):
|
||||
assert nccl_alloc._graph_pool_id == pool != handle
|
||||
const = torch.empty_like(x) # Never written by replay: needs a backup.
|
||||
y = x * weight + const
|
||||
return [graph, y, const.fill_(3.0)]
|
||||
|
||||
def graph_pool() -> dict[int, int]:
|
||||
data = allocator.pointer_to_data.items()
|
||||
return {
|
||||
p: d.handle[1] for p, d in data if d.tag == "cudagraph" and not d.is_asleep
|
||||
}
|
||||
|
||||
held, backend = capture(), CuMemBackend()
|
||||
mapped = graph_pool()
|
||||
backend.suspend(level=level)
|
||||
backend.resume(tags=["kv_cache"])
|
||||
assert graph_pool() == mapped
|
||||
backend.resume(tags=["weights"])
|
||||
weight.fill_(2.0) # Level 2 discards weights; emulate the reload.
|
||||
held[0].replay()
|
||||
assert torch.equal(held[1], torch.full_like(x, 5.0))
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -14,7 +15,12 @@ from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
from ....utils import create_new_process_for_each_test, multi_gpu_test, requires_fp8
|
||||
from ....utils import (
|
||||
create_new_process_for_each_test,
|
||||
multi_gpu_marks,
|
||||
multi_gpu_test,
|
||||
requires_fp8,
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@@ -195,6 +201,73 @@ def test_deep_sleep_lora_tp2(monkeypatch):
|
||||
assert output[0].outputs[0].text == output2[0].outputs[0].text
|
||||
|
||||
|
||||
def _cudagraph_bytes(worker) -> int:
|
||||
data = cumem.CuMemAllocator.get_instance().pointer_to_data.values()
|
||||
return sum(d.handle[1] for d in data if d.tag == "cudagraph")
|
||||
|
||||
|
||||
def _custom_ar_active(worker) -> bool:
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
ca_comm = get_tp_group().device_communicator.ca_comm
|
||||
return ca_comm is not None and not ca_comm.disabled
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "breakable", "tp", "offload"),
|
||||
[
|
||||
("FULL", False, 1, True),
|
||||
("PIECEWISE", False, 1, True),
|
||||
("PIECEWISE", True, 1, True),
|
||||
pytest.param(
|
||||
"FULL_AND_PIECEWISE", False, 2, True, marks=multi_gpu_marks(num_gpus=2)
|
||||
),
|
||||
("FULL_AND_PIECEWISE", False, 1, False),
|
||||
],
|
||||
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
|
||||
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
|
||||
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
|
||||
unregistered custom AR; off changes nothing."""
|
||||
for name, value in [
|
||||
("VLLM_USE_V2_MODEL_RUNNER", "1"),
|
||||
("VLLM_USE_BREAKABLE_CUDAGRAPH", "1" if breakable else "0"),
|
||||
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
|
||||
("VLLM_ALLREDUCE_USE_FLASHINFER", "0"),
|
||||
("VLLM_ALLREDUCE_USE_SYMM_MEM", "0"),
|
||||
]:
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
|
||||
llm = LLM(
|
||||
"hmellor/tiny-random-LlamaForCausalLM",
|
||||
enable_sleep_mode=True,
|
||||
sleep_mode_offload_cudagraph=offload,
|
||||
tensor_parallel_size=tp,
|
||||
compilation_config={
|
||||
"cudagraph_mode": mode,
|
||||
"pass_config": {"fuse_allreduce_rms": False},
|
||||
},
|
||||
)
|
||||
graph_bytes = llm.collective_rpc(_cudagraph_bytes)
|
||||
assert os.environ.get("NCCL_GRAPH_REGISTER") == ("0" if offload else None)
|
||||
assert all(graph_bytes) if offload else not any(graph_bytes)
|
||||
if not offload:
|
||||
return
|
||||
if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)):
|
||||
pytest.skip("Custom allreduce is unavailable on these GPUs")
|
||||
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, params)[0].outputs[0].text
|
||||
for level in (1, 2):
|
||||
llm.sleep(level=level)
|
||||
llm.wake_up(tags=["weights"])
|
||||
if level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_async(gpu_memory_cleared):
|
||||
async def test():
|
||||
|
||||
Reference in New Issue
Block a user