[ROCm] Enable the cuMem CUDA-graph pool offload on ROCm (#59523)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: indianspeedster <cspandey016@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: indianspeedster <cspandey016@gmail.com>
This commit is contained in:
aoshen02
2026-10-09 08:29:55 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 indianspeedster
parent bacbbe1878
commit a6f43e57d6
9 changed files with 84 additions and 38 deletions
@@ -359,7 +359,7 @@ def test_runtime_state_survives_sleep(level):
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
def test_level2_discards_ordinary_tensor_with_weights_tag():
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
"""Discarded weights-tag memory is remapped; ROCm zeroes it over stale pages."""
allocator = get_mem_allocator_instance()
with allocator.use_memory_pool("weights"):
@@ -374,8 +374,9 @@ def test_level2_discards_ordinary_tensor_with_weights_tag():
torch.accelerator.synchronize()
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
assert torch.all(fake_weight == 0xA5)
assert torch.all(ordinary_tensor == 0xA5)
expected = 0 if current_platform.is_rocm() else 0xA5
assert torch.all(fake_weight == expected)
assert torch.all(ordinary_tensor == expected)
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@@ -427,7 +428,9 @@ def test_cumem_with_cudagraph():
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="cuMem CUDA graph pool"
)
def test_cudagraph_pool_sleep(level):
"""Routing, and the graph pool backed up at both sleep levels and restored
in place by any wake."""
@@ -209,8 +209,20 @@ def _cudagraph_bytes(worker) -> int:
def _custom_ar_active(worker) -> bool:
from vllm.distributed.parallel_state import get_tp_group
ca_comm = get_tp_group().device_communicator.ca_comm
return ca_comm is not None and not ca_comm.disabled
comm = get_tp_group().device_communicator
return any(
c is not None and not c.disabled for c in (comm.ca_comm, comm.aiter_ar_comm)
)
def _capture_registers_buffers(worker) -> bool:
from vllm.distributed.parallel_state import get_tp_group
comm = get_tp_group().device_communicator
ca, aiter = comm.ca_comm, comm.aiter_ar_comm
return (ca is not None and ca._capture_registered) or (
aiter is not None and aiter.aiter_ca.enable_register_for_capturing
)
@pytest.mark.parametrize(
@@ -227,7 +239,9 @@ def _custom_ar_active(worker) -> bool:
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
)
@create_new_process_for_each_test()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="cuMem CUDA graph pool"
)
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
unregistered custom AR; off changes nothing."""
@@ -255,8 +269,10 @@ def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
assert all(graph_bytes) if offload else not any(graph_bytes)
if not offload:
return
if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)):
pytest.skip("Custom allreduce is unavailable on these GPUs")
if tp > 1:
if not all(llm.collective_rpc(_custom_ar_active)):
pytest.skip("Custom allreduce is unavailable on these GPUs")
assert not any(llm.collective_rpc(_capture_registers_buffers))
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, params)[0].outputs[0].text
for level in (1, 2):
@@ -268,6 +284,42 @@ def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
assert llm.generate(prompt, params)[0].outputs[0].text == expected
@multi_gpu_test(num_gpus=2)
@create_new_process_for_each_test()
@pytest.mark.skipif(not current_platform.is_rocm(), reason="AITER fused allreduce")
def test_sleep_cudagraph_pool_aiter_fused_ar_group_quant(monkeypatch):
"""The fused AITER AR + RMSNorm + per-group FP8 quant captures into the
cuMem pool without IPC-registering its input, across sleeps."""
for name, value in [
("VLLM_USE_V2_MODEL_RUNNER", "1"),
("VLLM_ROCM_USE_AITER", "1"),
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
]:
monkeypatch.setenv(name, value)
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
llm = LLM(
"Qwen/Qwen3-0.6B-FP8",
enable_sleep_mode=True,
sleep_mode_offload_cudagraph=True,
tensor_parallel_size=2,
max_model_len=1024,
compilation_config={"pass_config": {"fuse_allreduce_rms": True}},
)
if not all(llm.collective_rpc(_custom_ar_active)):
pytest.skip("AITER custom allreduce is unavailable on these GPUs")
assert not any(llm.collective_rpc(_capture_registers_buffers))
assert all(llm.collective_rpc(_cudagraph_bytes))
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
assert llm.generate(prompt, params)[0].outputs[0].text
for level in (1, 2):
llm.sleep(level=level)
llm.wake_up(tags=["weights"])
if level == 2:
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
assert llm.generate(prompt, params)[0].outputs[0].text
@create_new_process_for_each_test()
def test_deep_sleep_async(gpu_memory_cleared):
async def test():