[ROCm] Enable the cuMem CUDA-graph pool offload on ROCm (#59523)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: indianspeedster <cspandey016@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: indianspeedster <cspandey016@gmail.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
indianspeedster
parent
bacbbe1878
commit
a6f43e57d6
@@ -359,7 +359,7 @@ def test_runtime_state_survives_sleep(level):
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator")
|
||||
def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
"""Reproduce the level-2 variant for an ordinary tensor in weights."""
|
||||
"""Discarded weights-tag memory is remapped; ROCm zeroes it over stale pages."""
|
||||
allocator = get_mem_allocator_instance()
|
||||
|
||||
with allocator.use_memory_pool("weights"):
|
||||
@@ -374,8 +374,9 @@ def test_level2_discards_ordinary_tensor_with_weights_tag():
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert (fake_weight.data_ptr(), ordinary_tensor.data_ptr()) == pointers
|
||||
assert torch.all(fake_weight == 0xA5)
|
||||
assert torch.all(ordinary_tensor == 0xA5)
|
||||
expected = 0 if current_platform.is_rocm() else 0xA5
|
||||
assert torch.all(fake_weight == expected)
|
||||
assert torch.all(ordinary_tensor == expected)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@@ -427,7 +428,9 @@ def test_cumem_with_cudagraph():
|
||||
|
||||
@pytest.mark.parametrize("level", [1, 2], ids=["sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="cuMem CUDA graph pool"
|
||||
)
|
||||
def test_cudagraph_pool_sleep(level):
|
||||
"""Routing, and the graph pool backed up at both sleep levels and restored
|
||||
in place by any wake."""
|
||||
|
||||
@@ -209,8 +209,20 @@ def _cudagraph_bytes(worker) -> int:
|
||||
def _custom_ar_active(worker) -> bool:
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
ca_comm = get_tp_group().device_communicator.ca_comm
|
||||
return ca_comm is not None and not ca_comm.disabled
|
||||
comm = get_tp_group().device_communicator
|
||||
return any(
|
||||
c is not None and not c.disabled for c in (comm.ca_comm, comm.aiter_ar_comm)
|
||||
)
|
||||
|
||||
|
||||
def _capture_registers_buffers(worker) -> bool:
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
comm = get_tp_group().device_communicator
|
||||
ca, aiter = comm.ca_comm, comm.aiter_ar_comm
|
||||
return (ca is not None and ca._capture_registered) or (
|
||||
aiter is not None and aiter.aiter_ca.enable_register_for_capturing
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -227,7 +239,9 @@ def _custom_ar_active(worker) -> bool:
|
||||
ids=["full", "piecewise", "breakable", "custom-ar-tp2", "off-by-default"],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="cuMem CUDA graph pool")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="cuMem CUDA graph pool"
|
||||
)
|
||||
def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
|
||||
"""Each capture site uses the pool, exact across sleeps; TP=2 needs
|
||||
unregistered custom AR; off changes nothing."""
|
||||
@@ -255,8 +269,10 @@ def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
|
||||
assert all(graph_bytes) if offload else not any(graph_bytes)
|
||||
if not offload:
|
||||
return
|
||||
if tp > 1 and not all(llm.collective_rpc(_custom_ar_active)):
|
||||
pytest.skip("Custom allreduce is unavailable on these GPUs")
|
||||
if tp > 1:
|
||||
if not all(llm.collective_rpc(_custom_ar_active)):
|
||||
pytest.skip("Custom allreduce is unavailable on these GPUs")
|
||||
assert not any(llm.collective_rpc(_capture_registers_buffers))
|
||||
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, params)[0].outputs[0].text
|
||||
for level in (1, 2):
|
||||
@@ -268,6 +284,42 @@ def test_sleep_cudagraph_pool(monkeypatch, mode, breakable, tp, offload):
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text == expected
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="AITER fused allreduce")
|
||||
def test_sleep_cudagraph_pool_aiter_fused_ar_group_quant(monkeypatch):
|
||||
"""The fused AITER AR + RMSNorm + per-group FP8 quant captures into the
|
||||
cuMem pool without IPC-registering its input, across sleeps."""
|
||||
for name, value in [
|
||||
("VLLM_USE_V2_MODEL_RUNNER", "1"),
|
||||
("VLLM_ROCM_USE_AITER", "1"),
|
||||
("VLLM_ALLOW_INSECURE_SERIALIZATION", "1"),
|
||||
]:
|
||||
monkeypatch.setenv(name, value)
|
||||
monkeypatch.delenv("NCCL_GRAPH_REGISTER", raising=False)
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B-FP8",
|
||||
enable_sleep_mode=True,
|
||||
sleep_mode_offload_cudagraph=True,
|
||||
tensor_parallel_size=2,
|
||||
max_model_len=1024,
|
||||
compilation_config={"pass_config": {"fuse_allreduce_rms": True}},
|
||||
)
|
||||
if not all(llm.collective_rpc(_custom_ar_active)):
|
||||
pytest.skip("AITER custom allreduce is unavailable on these GPUs")
|
||||
assert not any(llm.collective_rpc(_capture_registers_buffers))
|
||||
assert all(llm.collective_rpc(_cudagraph_bytes))
|
||||
prompt, params = "How are you?", SamplingParams(temperature=0, max_tokens=10)
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text
|
||||
for level in (1, 2):
|
||||
llm.sleep(level=level)
|
||||
llm.wake_up(tags=["weights"])
|
||||
if level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
assert llm.generate(prompt, params)[0].outputs[0].text
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_deep_sleep_async(gpu_memory_cleared):
|
||||
async def test():
|
||||
|
||||
Reference in New Issue
Block a user