[ROCm][AITER] Directly Implement AITER Custom All-reduce in CudaCommunicator (#46065)

Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
BadrBasowid
2026-07-06 12:16:32 +00:00
committed by GitHub
parent 40cc2e8327
commit 740f379fae
12 changed files with 499 additions and 148 deletions
+1
View File
@@ -79,6 +79,7 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
):
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1" if use_deepgemm else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1" if use_aiter else "0")
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
@@ -30,6 +30,9 @@ from vllm.config import (
set_current_vllm_config,
)
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
AiterCustomAllreduce,
)
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
@@ -504,8 +507,12 @@ def all_reduce_fusion_pass_on_test_model(
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12345",
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": flashinfer_allreduce_backend,
"VLLM_ROCM_USE_AITER": str(int(use_aiter)),
"VLLM_ROCM_USE_AITER_CUSTOM_AR": str(int(use_aiter)),
}
)
if use_aiter:
rocm_aiter_ops.refresh_env_variables()
init_distributed_environment()
@@ -616,7 +623,7 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
if not rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group():
if not AiterCustomAllreduce.build_supports_per_group_quant():
pytest.skip(
"aiter build is missing 'fused_ar_rms_per_group_quant' (needs "
"ROCm/aiter PR #2823); the new patterns aren't registered."
@@ -671,6 +678,7 @@ def rocm_aiter_group_quant_fusion_pass_on_test_model(
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12345",
"VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_CUSTOM_AR": "1",
}
)
rocm_aiter_ops.refresh_env_variables()
@@ -0,0 +1,134 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import ray
import torch
import torch.distributed as dist
from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from vllm.envs import disable_envs_cache
from vllm.platforms import current_platform
from ..utils import (
assert_rocm_custom_allreduce_backend_state,
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_gpu_test,
multi_process_parallel,
)
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm-only AITER custom allreduce tests",
)
test_cases = [
((2, 7168), torch.float16),
((2, 7168), torch.bfloat16),
((128, 8192), torch.float16),
((128, 8192), torch.bfloat16),
]
def _configure_aiter_custom_ar_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False)
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1")
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE")
disable_envs_cache()
rocm_aiter_ops.refresh_env_variables()
def _assert_aiter_handles_input(inp: torch.Tensor) -> None:
aiter_ar_comm = get_tp_group().device_communicator.aiter_ar_comm
assert aiter_ar_comm is not None
assert aiter_ar_comm.should_custom_ar(inp), (
f"AITER CustomAllreduce does not support input shape {inp.shape}."
)
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
assert_rocm_custom_allreduce_backend_state(True, "NONE")
group = get_tp_group().device_group
# A small all_reduce for warmup.
# this is needed because device communicators might be created lazily
# (e.g. NCCL). This will ensure that the communicator is initialized
# before any communication happens, so that this group can be used for
# graph capture immediately.
data = torch.zeros(1)
data = data.to(device=device)
dist.all_reduce(data, group=group)
torch.accelerator.synchronize()
del data
for shape, dtype in test_cases:
with graph_capture(device=device) as graph_capture_context:
inp = torch.ones(shape, dtype=dtype, device=device)
_assert_aiter_handles_input(inp)
expected = inp * tp_size
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
out = tensor_model_parallel_all_reduce(inp)
graph.replay()
torch.testing.assert_close(out, expected)
@ray.remote(num_gpus=1, max_calls=1)
def eager_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
assert_rocm_custom_allreduce_backend_state(True, "NONE")
for shape, dtype in test_cases:
inp = torch.ones(shape, dtype=dtype, device=device)
_assert_aiter_handles_input(inp)
expected = inp * tp_size
out = tensor_model_parallel_all_reduce(inp)
torch.testing.assert_close(out, expected)
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1])
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
def test_rocm_aiter_custom_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pipeline_parallel_size,
test_target,
):
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
+40
View File
@@ -1459,6 +1459,46 @@ def multi_process_parallel(
ray.shutdown()
def assert_rocm_custom_allreduce_backend_state(
use_aiter_custom_ar: bool,
quick_reduce_quantization: str,
) -> None:
from vllm.distributed.parallel_state import get_tp_group
device_communicator = get_tp_group().device_communicator
aiter_ar_comm = device_communicator.aiter_ar_comm
if use_aiter_custom_ar:
assert aiter_ar_comm is not None, "AITER CustomAllreduce was not initialized."
assert not aiter_ar_comm.disabled, "AITER CustomAllreduce is disabled."
assert device_communicator.ca_comm is None, (
"vLLM CustomAllreduce should not be initialized when AITER CA is used."
)
else:
assert aiter_ar_comm is None, (
"AITER CustomAllreduce should not be initialized when disabled."
)
assert device_communicator.ca_comm is not None, (
"vLLM CustomAllreduce should be initialized when AITER CA is disabled."
)
qr_comm = device_communicator.qr_comm
assert qr_comm is not None, "QuickReduce communicator was not initialized."
if quick_reduce_quantization == "NONE":
assert qr_comm.disabled, "QuickReduce should be disabled."
else:
assert not qr_comm.disabled, "QuickReduce should be enabled."
def assert_rocm_custom_allreduce_backend_state_on_worker(
_worker,
use_aiter_custom_ar: bool,
quick_reduce_quantization: str,
) -> None:
assert_rocm_custom_allreduce_backend_state(
use_aiter_custom_ar, quick_reduce_quantization
)
@contextmanager
def error_on_warning(category: type[Warning] = Warning):
"""
@@ -0,0 +1,118 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode
from vllm.envs import disable_envs_cache
from vllm.platforms import current_platform
from ....conftest import VllmRunner
from ....utils import (
assert_rocm_custom_allreduce_backend_state_on_worker,
multi_gpu_test,
)
PROMPTS = ["Hello, my name is", "The capital of France is"]
def _run_generation(
vllm_runner: type[VllmRunner],
monkeypatch: pytest.MonkeyPatch,
compilation_config: CompilationConfig,
*,
model: str,
max_tokens: int,
use_aiter_custom_ar: bool,
quick_reduce_quantization: str,
) -> list[tuple[list[int], str]]:
with monkeypatch.context() as m:
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_CUSTOM_AR",
"1" if use_aiter_custom_ar else "0",
)
m.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", quick_reduce_quantization)
disable_envs_cache()
rocm_aiter_ops.refresh_env_variables()
with vllm_runner(
model,
dtype="half",
tensor_parallel_size=2,
compilation_config=compilation_config,
max_model_len=256,
max_num_seqs=len(PROMPTS),
gpu_memory_utilization=0.7,
) as llm:
llm.get_llm().collective_rpc(
assert_rocm_custom_allreduce_backend_state_on_worker,
args=(use_aiter_custom_ar, quick_reduce_quantization),
)
return llm.generate_greedy(PROMPTS, max_tokens)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only")
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"quick_reduce_quantization",
[
pytest.param("FP", id="quick-reduce-on"),
pytest.param("NONE", id="quick-reduce-off"),
],
)
@pytest.mark.parametrize(
"cudagraph_mode",
[
pytest.param(CUDAGraphMode.NONE, id="cudagraph-none"),
pytest.param(CUDAGraphMode.FULL, id="cudagraph-full"),
],
)
@pytest.mark.parametrize(
"model,max_tokens",
[
pytest.param("facebook/opt-125m", 8, id="opt-125m"),
],
)
def test_rocm_aiter_custom_ar_e2e(
vllm_runner: type[VllmRunner],
monkeypatch: pytest.MonkeyPatch,
cudagraph_mode: CUDAGraphMode,
quick_reduce_quantization: str,
model: str,
max_tokens: int,
):
compilation_mode = (
CompilationMode.NONE
if cudagraph_mode == CUDAGraphMode.NONE
else CompilationMode.VLLM_COMPILE
)
compilation_config = CompilationConfig(
mode=compilation_mode,
cudagraph_mode=cudagraph_mode,
)
baseline_generations = _run_generation(
vllm_runner,
monkeypatch,
compilation_config,
model=model,
max_tokens=max_tokens,
use_aiter_custom_ar=False,
quick_reduce_quantization=quick_reduce_quantization,
)
aiter_custom_ar_generations = _run_generation(
vllm_runner,
monkeypatch,
compilation_config,
model=model,
max_tokens=max_tokens,
use_aiter_custom_ar=True,
quick_reduce_quantization=quick_reduce_quantization,
)
assert aiter_custom_ar_generations == baseline_generations