[ROCm][AITER] Directly Implement AITER Custom All-reduce in CudaCommunicator (#46065)

Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
BadrBasowid
2026-07-06 12:16:32 +00:00
committed by GitHub
parent 40cc2e8327
commit 740f379fae
12 changed files with 499 additions and 148 deletions
+5 -1
View File
@@ -2782,15 +2782,19 @@ steps:
- vllm/envs.py - vllm/envs.py
- examples/offline_inference/data_parallel.py - examples/offline_inference/data_parallel.py
- tests/distributed/test_context_parallel.py - tests/distributed/test_context_parallel.py
- tests/distributed/test_rocm_aiter_custom_ar.py
- tests/distributed/test_rocm_quick_reduce.py - tests/distributed/test_rocm_quick_reduce.py
- tests/distributed/test_quick_all_reduce.py - tests/distributed/test_quick_all_reduce.py
- tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
- tests/v1/distributed/test_dbo.py - tests/v1/distributed/test_dbo.py
- tests/utils.py - tests/utils.py
commands: commands:
- pytest -v -s tests/distributed/test_context_parallel.py - pytest -v -s tests/distributed/test_context_parallel.py
- pytest -v -s tests/v1/distributed/test_dbo.py - pytest -v -s tests/distributed/test_rocm_aiter_custom_ar.py
- pytest -v -s tests/v1/e2e/general/test_rocm_aiter_custom_ar.py
- pytest -v -s tests/distributed/test_rocm_quick_reduce.py - pytest -v -s tests/distributed/test_rocm_quick_reduce.py
- pytest -v -s tests/distributed/test_quick_all_reduce.py - pytest -v -s tests/distributed/test_quick_all_reduce.py
- pytest -v -s tests/v1/distributed/test_dbo.py
#-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------# #-------------------------------------------------------- mi355 · entrypoints --------------------------------------------------------#
+1
View File
@@ -79,6 +79,7 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
): ):
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1" if use_deepgemm else "0") monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1" if use_deepgemm else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0") monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1" if use_aiter else "0")
from vllm._aiter_ops import rocm_aiter_ops from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables() rocm_aiter_ops.refresh_env_variables()
@@ -30,6 +30,9 @@ from vllm.config import (
set_current_vllm_config, set_current_vllm_config,
) )
from vllm.distributed import tensor_model_parallel_all_reduce from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
AiterCustomAllreduce,
)
from vllm.distributed.parallel_state import ( from vllm.distributed.parallel_state import (
init_distributed_environment, init_distributed_environment,
initialize_model_parallel, initialize_model_parallel,
@@ -504,8 +507,12 @@ def all_reduce_fusion_pass_on_test_model(
"MASTER_ADDR": "localhost", "MASTER_ADDR": "localhost",
"MASTER_PORT": "12345", "MASTER_PORT": "12345",
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": flashinfer_allreduce_backend, "VLLM_FLASHINFER_ALLREDUCE_BACKEND": flashinfer_allreduce_backend,
"VLLM_ROCM_USE_AITER": str(int(use_aiter)),
"VLLM_ROCM_USE_AITER_CUSTOM_AR": str(int(use_aiter)),
} }
) )
if use_aiter:
rocm_aiter_ops.refresh_env_variables()
init_distributed_environment() init_distributed_environment()
@@ -616,7 +623,7 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
m.setenv("VLLM_ROCM_USE_AITER", "1") m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables() rocm_aiter_ops.refresh_env_variables()
if not rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group(): if not AiterCustomAllreduce.build_supports_per_group_quant():
pytest.skip( pytest.skip(
"aiter build is missing 'fused_ar_rms_per_group_quant' (needs " "aiter build is missing 'fused_ar_rms_per_group_quant' (needs "
"ROCm/aiter PR #2823); the new patterns aren't registered." "ROCm/aiter PR #2823); the new patterns aren't registered."
@@ -671,6 +678,7 @@ def rocm_aiter_group_quant_fusion_pass_on_test_model(
"MASTER_ADDR": "localhost", "MASTER_ADDR": "localhost",
"MASTER_PORT": "12345", "MASTER_PORT": "12345",
"VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_CUSTOM_AR": "1",
} }
) )
rocm_aiter_ops.refresh_env_variables() rocm_aiter_ops.refresh_env_variables()
@@ -0,0 +1,134 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import ray
import torch
import torch.distributed as dist
from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from vllm.envs import disable_envs_cache
from vllm.platforms import current_platform
from ..utils import (
assert_rocm_custom_allreduce_backend_state,
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_gpu_test,
multi_process_parallel,
)
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm-only AITER custom allreduce tests",
)
test_cases = [
((2, 7168), torch.float16),
((2, 7168), torch.bfloat16),
((128, 8192), torch.float16),
((128, 8192), torch.bfloat16),
]
def _configure_aiter_custom_ar_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False)
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1")
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE")
disable_envs_cache()
rocm_aiter_ops.refresh_env_variables()
def _assert_aiter_handles_input(inp: torch.Tensor) -> None:
aiter_ar_comm = get_tp_group().device_communicator.aiter_ar_comm
assert aiter_ar_comm is not None
assert aiter_ar_comm.should_custom_ar(inp), (
f"AITER CustomAllreduce does not support input shape {inp.shape}."
)
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
assert_rocm_custom_allreduce_backend_state(True, "NONE")
group = get_tp_group().device_group
# A small all_reduce for warmup.
# this is needed because device communicators might be created lazily
# (e.g. NCCL). This will ensure that the communicator is initialized
# before any communication happens, so that this group can be used for
# graph capture immediately.
data = torch.zeros(1)
data = data.to(device=device)
dist.all_reduce(data, group=group)
torch.accelerator.synchronize()
del data
for shape, dtype in test_cases:
with graph_capture(device=device) as graph_capture_context:
inp = torch.ones(shape, dtype=dtype, device=device)
_assert_aiter_handles_input(inp)
expected = inp * tp_size
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
out = tensor_model_parallel_all_reduce(inp)
graph.replay()
torch.testing.assert_close(out, expected)
@ray.remote(num_gpus=1, max_calls=1)
def eager_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
assert_rocm_custom_allreduce_backend_state(True, "NONE")
for shape, dtype in test_cases:
inp = torch.ones(shape, dtype=dtype, device=device)
_assert_aiter_handles_input(inp)
expected = inp * tp_size
out = tensor_model_parallel_all_reduce(inp)
torch.testing.assert_close(out, expected)
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1])
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
def test_rocm_aiter_custom_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pipeline_parallel_size,
test_target,
):
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
+40
View File
@@ -1459,6 +1459,46 @@ def multi_process_parallel(
ray.shutdown() ray.shutdown()
def assert_rocm_custom_allreduce_backend_state(
use_aiter_custom_ar: bool,
quick_reduce_quantization: str,
) -> None:
from vllm.distributed.parallel_state import get_tp_group
device_communicator = get_tp_group().device_communicator
aiter_ar_comm = device_communicator.aiter_ar_comm
if use_aiter_custom_ar:
assert aiter_ar_comm is not None, "AITER CustomAllreduce was not initialized."
assert not aiter_ar_comm.disabled, "AITER CustomAllreduce is disabled."
assert device_communicator.ca_comm is None, (
"vLLM CustomAllreduce should not be initialized when AITER CA is used."
)
else:
assert aiter_ar_comm is None, (
"AITER CustomAllreduce should not be initialized when disabled."
)
assert device_communicator.ca_comm is not None, (
"vLLM CustomAllreduce should be initialized when AITER CA is disabled."
)
qr_comm = device_communicator.qr_comm
assert qr_comm is not None, "QuickReduce communicator was not initialized."
if quick_reduce_quantization == "NONE":
assert qr_comm.disabled, "QuickReduce should be disabled."
else:
assert not qr_comm.disabled, "QuickReduce should be enabled."
def assert_rocm_custom_allreduce_backend_state_on_worker(
_worker,
use_aiter_custom_ar: bool,
quick_reduce_quantization: str,
) -> None:
assert_rocm_custom_allreduce_backend_state(
use_aiter_custom_ar, quick_reduce_quantization
)
@contextmanager @contextmanager
def error_on_warning(category: type[Warning] = Warning): def error_on_warning(category: type[Warning] = Warning):
""" """
@@ -0,0 +1,118 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode
from vllm.envs import disable_envs_cache
from vllm.platforms import current_platform
from ....conftest import VllmRunner
from ....utils import (
assert_rocm_custom_allreduce_backend_state_on_worker,
multi_gpu_test,
)
PROMPTS = ["Hello, my name is", "The capital of France is"]
def _run_generation(
vllm_runner: type[VllmRunner],
monkeypatch: pytest.MonkeyPatch,
compilation_config: CompilationConfig,
*,
model: str,
max_tokens: int,
use_aiter_custom_ar: bool,
quick_reduce_quantization: str,
) -> list[tuple[list[int], str]]:
with monkeypatch.context() as m:
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_CUSTOM_AR",
"1" if use_aiter_custom_ar else "0",
)
m.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", quick_reduce_quantization)
disable_envs_cache()
rocm_aiter_ops.refresh_env_variables()
with vllm_runner(
model,
dtype="half",
tensor_parallel_size=2,
compilation_config=compilation_config,
max_model_len=256,
max_num_seqs=len(PROMPTS),
gpu_memory_utilization=0.7,
) as llm:
llm.get_llm().collective_rpc(
assert_rocm_custom_allreduce_backend_state_on_worker,
args=(use_aiter_custom_ar, quick_reduce_quantization),
)
return llm.generate_greedy(PROMPTS, max_tokens)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only")
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"quick_reduce_quantization",
[
pytest.param("FP", id="quick-reduce-on"),
pytest.param("NONE", id="quick-reduce-off"),
],
)
@pytest.mark.parametrize(
"cudagraph_mode",
[
pytest.param(CUDAGraphMode.NONE, id="cudagraph-none"),
pytest.param(CUDAGraphMode.FULL, id="cudagraph-full"),
],
)
@pytest.mark.parametrize(
"model,max_tokens",
[
pytest.param("facebook/opt-125m", 8, id="opt-125m"),
],
)
def test_rocm_aiter_custom_ar_e2e(
vllm_runner: type[VllmRunner],
monkeypatch: pytest.MonkeyPatch,
cudagraph_mode: CUDAGraphMode,
quick_reduce_quantization: str,
model: str,
max_tokens: int,
):
compilation_mode = (
CompilationMode.NONE
if cudagraph_mode == CUDAGraphMode.NONE
else CompilationMode.VLLM_COMPILE
)
compilation_config = CompilationConfig(
mode=compilation_mode,
cudagraph_mode=cudagraph_mode,
)
baseline_generations = _run_generation(
vllm_runner,
monkeypatch,
compilation_config,
model=model,
max_tokens=max_tokens,
use_aiter_custom_ar=False,
quick_reduce_quantization=quick_reduce_quantization,
)
aiter_custom_ar_generations = _run_generation(
vllm_runner,
monkeypatch,
compilation_config,
model=model,
max_tokens=max_tokens,
use_aiter_custom_ar=True,
quick_reduce_quantization=quick_reduce_quantization,
)
assert aiter_custom_ar_generations == baseline_generations
+34 -95
View File
@@ -2,12 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools import functools
from collections.abc import Callable from collections.abc import Callable
from contextlib import contextmanager
from typing import Protocol
import torch import torch
from torch._ops import OpOverload from torch._ops import OpOverload
from torch.distributed import ProcessGroup
import vllm.envs as envs import vllm.envs as envs
from vllm.platforms import current_platform from vllm.platforms import current_platform
@@ -52,42 +49,6 @@ def is_aiter_found() -> bool:
IS_AITER_FOUND = is_aiter_found() IS_AITER_FOUND = is_aiter_found()
class AiterCustomAllreduceProto(Protocol):
max_size: int
world_size: int
fully_connected: bool
@contextmanager
def capture(self): ...
def close(self) -> None: ...
def fused_ar_rms(
self,
inp: torch.Tensor,
res_inp: torch.Tensor,
*,
w: torch.Tensor,
eps: float,
registered: bool = False,
use_1stage: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]: ...
def fused_ar_rms_per_group_quant(
self,
inp: torch.Tensor,
res_inp: torch.Tensor,
*,
w: torch.Tensor,
eps: float,
group_size: int = 128,
registered: bool = False,
use_1stage: bool = False,
emit_bf16: bool = False,
) -> (
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
): ...
def should_custom_ar(self, inp: torch.Tensor) -> bool: ...
def is_aiter_found_and_supported() -> bool: def is_aiter_found_and_supported() -> bool:
"""Check if AITER library is available and platform supports it. """Check if AITER library is available and platform supports it.
@@ -830,6 +791,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
) -> tuple[torch.Tensor, torch.Tensor]: ) -> tuple[torch.Tensor, torch.Tensor]:
aiter_ar = rocm_aiter_ops.get_aiter_allreduce() aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized" assert aiter_ar is not None, "aiter allreduce must be initialized"
ca = aiter_ar.aiter_ca
total_bytes = input_.numel() * input_.element_size() total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1] hidden_dim = input_.shape[-1]
@@ -840,8 +802,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
else: else:
hidden_ok = False hidden_ok = False
token_ok = token_num <= 80 token_ok = token_num <= 80
world_size = aiter_ar.world_size world_size = ca.world_size
full_nvlink = aiter_ar.fully_connected full_nvlink = ca.fully_connected
if world_size == 2: if world_size == 2:
size_ok = True size_ok = True
@@ -854,12 +816,11 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
use_1stage = hidden_ok and token_ok and size_ok use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms( result = ca.custom_fused_ar_rms(
input_, input_,
residual, residual,
w=weight, weight,
eps=epsilon, epsilon,
registered=torch.cuda.is_current_stream_capturing(),
use_1stage=use_1stage, use_1stage=use_1stage,
) )
assert result is not None assert result is not None
@@ -890,6 +851,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
""" """
aiter_ar = rocm_aiter_ops.get_aiter_allreduce() aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized" assert aiter_ar is not None, "aiter allreduce must be initialized"
ca = aiter_ar.aiter_ca
total_bytes = input_.numel() * input_.element_size() total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1] hidden_dim = input_.shape[-1]
@@ -900,8 +862,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
else: else:
hidden_ok = False hidden_ok = False
token_ok = token_num <= 80 token_ok = token_num <= 80
world_size = aiter_ar.world_size world_size = ca.world_size
full_nvlink = aiter_ar.fully_connected full_nvlink = ca.fully_connected
if world_size == 2: if world_size == 2:
size_ok = True size_ok = True
@@ -914,7 +876,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
use_1stage = hidden_ok and token_ok and size_ok use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms_per_group_quant( result = ca.fused_ar_rms_per_group_quant(
input_, input_,
residual, residual,
w=weight, w=weight,
@@ -962,6 +924,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
""" """
aiter_ar = rocm_aiter_ops.get_aiter_allreduce() aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized" assert aiter_ar is not None, "aiter allreduce must be initialized"
ca = aiter_ar.aiter_ca
total_bytes = input_.numel() * input_.element_size() total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1] hidden_dim = input_.shape[-1]
@@ -972,8 +935,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
else: else:
hidden_ok = False hidden_ok = False
token_ok = token_num <= 80 token_ok = token_num <= 80
world_size = aiter_ar.world_size world_size = ca.world_size
full_nvlink = aiter_ar.fully_connected full_nvlink = ca.fully_connected
if world_size == 2: if world_size == 2:
size_ok = True size_ok = True
@@ -986,7 +949,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
use_1stage = hidden_ok and token_ok and size_ok use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms_per_group_quant( result = ca.fused_ar_rms_per_group_quant(
input_, input_,
residual, residual,
w=weight, w=weight,
@@ -1577,6 +1540,7 @@ class rocm_aiter_ops:
# Check if the env variable is set # Check if the env variable is set
_AITER_ENABLED = envs.VLLM_ROCM_USE_AITER _AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
_CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR
_LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR _LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
_FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE _FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE
_MLA_ENABLED = envs.VLLM_ROCM_USE_AITER_MLA _MLA_ENABLED = envs.VLLM_ROCM_USE_AITER_MLA
@@ -1598,9 +1562,6 @@ class rocm_aiter_ops:
# num_shared_experts / shared_expert_scoring_func args (7-arg form). # num_shared_experts / shared_expert_scoring_func args (7-arg form).
_TOPK_SOFTMAX_FUSED_SIGMOID: bool | None = None _TOPK_SOFTMAX_FUSED_SIGMOID: bool | None = None
_ALL_REDUCE_MAX_SIZE: int = 8192 * 1024 * 8 * 2
_CUSTOM_ALL_REDUCE: AiterCustomAllreduceProto | None = None
@classmethod @classmethod
def refresh_env_variables(cls): def refresh_env_variables(cls):
""" """
@@ -1611,6 +1572,7 @@ class rocm_aiter_ops:
you can call this function to reload the env variables. you can call this function to reload the env variables.
""" """
cls._AITER_ENABLED = envs.VLLM_ROCM_USE_AITER cls._AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
cls._CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR
cls._LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR cls._LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
cls._FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE cls._FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE
cls._MLA_ENABLED = envs.VLLM_ROCM_USE_AITER_MLA cls._MLA_ENABLED = envs.VLLM_ROCM_USE_AITER_MLA
@@ -1770,6 +1732,11 @@ class rocm_aiter_ops:
def is_mha_enabled(cls) -> bool: def is_mha_enabled(cls) -> bool:
return cls._AITER_ENABLED and cls._MHA_ENABLED return cls._AITER_ENABLED and cls._MHA_ENABLED
@classmethod
@if_aiter_supported
def is_custom_all_reduce_enabled(cls) -> bool:
return cls._AITER_ENABLED and cls._CUSTOM_ALL_REDUCE_ENABLED
@classmethod @classmethod
@if_aiter_supported @if_aiter_supported
def is_shuffle_kv_cache_enabled(cls) -> bool: def is_shuffle_kv_cache_enabled(cls) -> bool:
@@ -1824,33 +1791,20 @@ class rocm_aiter_ops:
return cls.is_linear_enabled() and on_gfx950() return cls.is_linear_enabled() and on_gfx950()
@classmethod @classmethod
def initialize_aiter_allreduce( def get_aiter_allreduce(cls):
cls, group: ProcessGroup, device: torch.device """Return the TP device communicator's AITER custom-allreduce if it has
) -> None: one, return None otherwise
try: """
from aiter.dist.device_communicators.custom_all_reduce import ( from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
CustomAllreduce as AiterCustomAllreduce, AiterCustomAllreduce,
) )
from vllm.distributed.parallel_state import get_tp_group
cls._CUSTOM_ALL_REDUCE = AiterCustomAllreduce(group, device) device_comm = get_tp_group().device_communicator
except Exception: aiter_ar_comm = getattr(device_comm, "aiter_ar_comm", None)
cls._CUSTOM_ALL_REDUCE = None return (
aiter_ar_comm if isinstance(aiter_ar_comm, AiterCustomAllreduce) else None
@classmethod )
def get_aiter_allreduce(cls) -> AiterCustomAllreduceProto | None:
return cls._CUSTOM_ALL_REDUCE
@classmethod
def destroy_aiter_allreduce(cls) -> None:
if cls._CUSTOM_ALL_REDUCE is not None:
cls._CUSTOM_ALL_REDUCE.close()
cls._CUSTOM_ALL_REDUCE = None
@classmethod
def get_aiter_allreduce_max_size(cls) -> int | None:
# effective max input size (based on upstream aiter version: v0.1.10.post3)
# https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/aiter/dist/device_communicators/custom_all_reduce.py#L272-L273
return int(cls._ALL_REDUCE_MAX_SIZE / 2)
@classmethod @classmethod
@if_aiter_supported @if_aiter_supported
@@ -2165,21 +2119,6 @@ class rocm_aiter_ops:
def get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() -> OpOverload: # noqa: E501 def get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() -> OpOverload: # noqa: E501
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm.default # noqa: E501 return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm.default # noqa: E501
# TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823).
@classmethod
def has_fused_allreduce_rmsnorm_quant_per_group(cls) -> bool:
"""True if the running AITER build exposes the per-group AR+RMS+quant
kernel (added in ROCm/aiter PR #2823).
The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off
this so vLLM degrades to the AR+RMS-only fusion when run against an
older aiter that lacks the per-group launcher.
"""
aiter_ar = cls.get_aiter_allreduce()
return aiter_ar is not None and hasattr(
aiter_ar, "fused_ar_rms_per_group_quant"
)
@staticmethod @staticmethod
def get_fused_mla_dual_rms_norm_op() -> OpOverload: def get_fused_mla_dual_rms_norm_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm.default return torch.ops.vllm.fused_mla_dual_rms_norm.default
@@ -19,7 +19,6 @@ from vllm.compilation.passes.fusion.rms_quant_fusion import (
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.utils import Range from vllm.config.utils import Range
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
from vllm.distributed.parallel_state import ( from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank, get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size, get_tensor_model_parallel_world_size,
@@ -1473,39 +1472,23 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
) )
return return
device_comm = get_tp_group().device_communicator ca_comm = rocm_aiter_ops.get_aiter_allreduce()
if device_comm is None:
logger.warning_once("Device communicator is required.")
return
ca_comm = getattr(device_comm, "ca_comm", None)
if ca_comm is None: if ca_comm is None:
logger.warning_once("Custom Allreduce is required.") logger.warning_once(
"AITER allreduce fusions are disabled "
"because AITER Custom All Reduce is not enabled. "
"Set VLLM_ROCM_USE_AITER_CUSTOM_AR=1 "
"to enable it."
)
return return
self.ca_comm = ca_comm self.ca_comm = ca_comm
assert isinstance(ca_comm, CustomAllreduce)
group = get_tp_group().cpu_group
rocm_aiter_ops.initialize_aiter_allreduce(group, self.device)
hidden_dim = config.model_config.get_hidden_size() hidden_dim = config.model_config.get_hidden_size()
element_size = torch.tensor([], dtype=self.model_dtype).element_size() element_size = torch.tensor([], dtype=self.model_dtype).element_size()
max_size = rocm_aiter_ops.get_aiter_allreduce_max_size() max_size = ca_comm.effective_max_size()
if max_size is None:
logger.warning("AITER allreduce fusion must be initialized")
return
# Aiter's fused_allreduce_rmsnorm kernel dispatches on hidden_dim.
# Before aiter v0.1.12 the launcher was template-specialized on HIDDEN_DIM
# and silently no-op'd for sizes outside {512, 1024, 2048, 4096}. From v0.1.12
# hidden_dim is a runtime argument. Detect the older API via the missing
# `_pool` attribute and skip fusion for unsupported sizes.
# Ref (old kernel): https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/csrc/include/custom_all_reduce.cuh#L2590
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
_AITER_OLD_FUSED_AR_RMS_HIDDEN = (512, 1024, 2048, 4096) _AITER_OLD_FUSED_AR_RMS_HIDDEN = (512, 1024, 2048, 4096)
if ( if (
aiter_ar is not None not ca_comm.supports_dynamic_hidden_dim
and not hasattr(aiter_ar, "_pool")
and hidden_dim not in _AITER_OLD_FUSED_AR_RMS_HIDDEN and hidden_dim not in _AITER_OLD_FUSED_AR_RMS_HIDDEN
): ):
logger.warning_once( logger.warning_once(
@@ -1515,10 +1498,6 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
_AITER_OLD_FUSED_AR_RMS_HIDDEN, _AITER_OLD_FUSED_AR_RMS_HIDDEN,
hidden_dim, hidden_dim,
) )
# Tear down aiter's custom-allreduce so its IPC handles don't
# race with vllm's ca_comm on the unfused fallback path.
with contextlib.suppress(Exception):
rocm_aiter_ops.destroy_aiter_allreduce()
return return
max_token_num = max_size // (hidden_dim * element_size) max_token_num = max_size // (hidden_dim * element_size)
@@ -1532,9 +1511,7 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
# fall back to the AR+RMS-only fusion paired with PR #41825's # fall back to the AR+RMS-only fusion paired with PR #41825's
# standalone RMS+quant fusion -- still correct, just leaves the # standalone RMS+quant fusion -- still correct, just leaves the
# post-AR quant as a standalone kernel. # post-AR quant as a standalone kernel.
supports_per_group_quant = ( supports_per_group_quant = ca_comm.supports_per_group_quant
rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group()
)
if not supports_per_group_quant: if not supports_per_group_quant:
logger.warning_once( logger.warning_once(
"AITER AR+RMS+per-group-FP8-quant fusion disabled: aiter " "AITER AR+RMS+per-group-FP8-quant fusion disabled: aiter "
@@ -1609,9 +1586,3 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
logger.debug( logger.debug(
"%s Replaced %s patterns", self.__class__.__name__, self.matched_count "%s Replaced %s patterns", self.__class__.__name__, self.matched_count
) )
def __del__(self) -> None:
if getattr(self, "disabled", True):
return
with contextlib.suppress(Exception):
rocm_aiter_ops.destroy_aiter_allreduce()
+7 -2
View File
@@ -1850,8 +1850,13 @@ class VllmConfig:
tp_size = self.parallel_config.tensor_parallel_size tp_size = self.parallel_config.tensor_parallel_size
from vllm._aiter_ops import rocm_aiter_ops from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_enabled(): max_size: int | None = None
max_size = rocm_aiter_ops.get_aiter_allreduce_max_size() if rocm_aiter_ops.is_custom_all_reduce_enabled():
from vllm.distributed.device_communicators.aiter_custom_all_reduce import ( # noqa: E501
AiterCustomAllreduce,
)
max_size = AiterCustomAllreduce.effective_max_size()
else: else:
max_size = compilation_config.pass_config.flashinfer_max_size(tp_size) max_size = compilation_config.pass_config.flashinfer_max_size(tp_size)
if max_size is not None and self.model_config is not None: if max_size is not None and self.model_config is not None:
@@ -0,0 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM-owned wrapper over AITER's ``CustomAllreduce``.
vLLM's ``CudaCommunicator`` stores one of these as ``aiter_ar_comm`` (when
``VLLM_ROCM_USE_AITER_CUSTOM_AR`` is set) so the plain allreduce and
the fused allreduce+RMSNorm path share a single AITER instance with its IPC buffers.
"""
import torch
from torch.distributed import ProcessGroup
from vllm.logger import init_logger
logger = init_logger(__name__)
class AiterCustomAllreduce:
# Default IPC buffer size for AITER's CustomAllreduce.
MAX_SIZE: int = 8192 * 1024 * 8 * 2
@classmethod
def effective_max_size(cls) -> int:
"""
Max input byte size eligible for AITER custom allreduce.
"""
return cls.MAX_SIZE // 2
def __init__(
self,
group: ProcessGroup,
device: int | str | torch.device,
max_size: int | None = None,
):
from aiter.dist.device_communicators.custom_all_reduce import (
CustomAllreduce as _AiterCustomAllreduce,
)
if max_size is None:
max_size = self.MAX_SIZE
self._impl = _AiterCustomAllreduce(group, device, max_size=max_size)
@property
def aiter_ca(self):
return self._impl
@property
def disabled(self) -> bool:
return self._impl.disabled
def should_custom_ar(self, inp: torch.Tensor) -> bool:
return self._impl.should_custom_ar(inp)
def custom_all_reduce(self, inp: torch.Tensor) -> torch.Tensor | None:
return self._impl.custom_all_reduce(inp)
def capture(self):
return self._impl.capture()
def close(self) -> None:
self._impl.close()
@property
def supports_dynamic_hidden_dim(self) -> bool:
"""Aiter's fused_allreduce_rmsnorm kernel dispatches on hidden_dim.
Before aiter v0.1.12 the launcher was template-specialized on HIDDEN_DIM
and silently no-op'd for sizes outside {512, 1024, 2048, 4096}. From v0.1.12
hidden_dim is a runtime argument. Older builds are detected via
AiterCustomAllreduce.supports_dynamic_hidden_dim; This function is used to
skip fusion for unsupported sizes on them.
Ref (old kernel): https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/csrc/include/custom_all_reduce.cuh#L2590
"""
return hasattr(self._impl, "_pool")
@staticmethod
def build_supports_per_group_quant() -> bool:
"""True if the running AITER build exposes the per-group AR+RMS+quant
kernel (added in ROCm/aiter PR #2823).
The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off
this so vLLM degrades to the AR+RMS-only fusion when run against an
older aiter that lacks the per-group launcher.
"""
from aiter.dist.device_communicators.custom_all_reduce import (
CustomAllreduce as _AiterCustomAllreduce,
)
return hasattr(_AiterCustomAllreduce, "fused_ar_rms_per_group_quant")
# TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823).
@property
def supports_per_group_quant(self) -> bool:
return self.build_supports_per_group_quant()
@@ -6,6 +6,7 @@ import torch
from torch.distributed import ProcessGroup from torch.distributed import ProcessGroup
import vllm.envs as envs import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.device_communicators.all_reduce_utils import ( from vllm.distributed.device_communicators.all_reduce_utils import (
NCCL_SYMM_MEM_ALL_REDUCE_CONFIG, NCCL_SYMM_MEM_ALL_REDUCE_CONFIG,
should_nccl_symm_mem_ag_rs, should_nccl_symm_mem_ag_rs,
@@ -19,6 +20,7 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform from vllm.platforms import current_platform
from ..utils import StatelessProcessGroup from ..utils import StatelessProcessGroup
from .aiter_custom_all_reduce import AiterCustomAllreduce
from .base_device_communicator import DeviceCommunicatorBase from .base_device_communicator import DeviceCommunicatorBase
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -48,16 +50,21 @@ class CudaCommunicator(DeviceCommunicatorBase):
use_custom_allreduce = False use_custom_allreduce = False
use_torch_symm_mem = False use_torch_symm_mem = False
use_flashinfer_allreduce = False use_flashinfer_allreduce = False
use_aiter_allreduce = False
else: else:
from vllm.distributed.parallel_state import _ENABLE_CUSTOM_ALL_REDUCE from vllm.distributed.parallel_state import _ENABLE_CUSTOM_ALL_REDUCE
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
use_torch_symm_mem = envs.VLLM_ALLREDUCE_USE_SYMM_MEM use_torch_symm_mem = envs.VLLM_ALLREDUCE_USE_SYMM_MEM
use_flashinfer_allreduce = envs.VLLM_ALLREDUCE_USE_FLASHINFER use_flashinfer_allreduce = envs.VLLM_ALLREDUCE_USE_FLASHINFER
use_aiter_allreduce = use_custom_allreduce and bool(
rocm_aiter_ops.is_custom_all_reduce_enabled()
)
self.use_custom_allreduce = use_custom_allreduce self.use_custom_allreduce = use_custom_allreduce
self.use_torch_symm_mem = use_torch_symm_mem self.use_torch_symm_mem = use_torch_symm_mem
self.use_flashinfer_allreduce = use_flashinfer_allreduce self.use_flashinfer_allreduce = use_flashinfer_allreduce
self.use_aiter_allreduce = use_aiter_allreduce
# lazy import to avoid documentation build error # lazy import to avoid documentation build error
from vllm.distributed.device_communicators.custom_all_reduce import ( from vllm.distributed.device_communicators.custom_all_reduce import (
@@ -85,6 +92,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.qr_comm: QuickAllReduce | None = None self.qr_comm: QuickAllReduce | None = None
self.symm_mem_comm: SymmMemCommunicator | None = None self.symm_mem_comm: SymmMemCommunicator | None = None
self.fi_ar_comm: FlashInferAllReduce | None = None self.fi_ar_comm: FlashInferAllReduce | None = None
self.aiter_ar_comm: AiterCustomAllreduce | None = None
if use_torch_symm_mem and current_platform.is_cuda(): if use_torch_symm_mem and current_platform.is_cuda():
self.symm_mem_comm = SymmMemCommunicator( self.symm_mem_comm = SymmMemCommunicator(
@@ -98,7 +106,13 @@ class CudaCommunicator(DeviceCommunicatorBase):
device=self.device, device=self.device,
) )
if use_custom_allreduce and self.world_size > 1: if self.use_aiter_allreduce and self.world_size > 1:
self.aiter_ar_comm = AiterCustomAllreduce(
group=self.cpu_group,
device=self.device,
)
if use_custom_allreduce and self.aiter_ar_comm is None and self.world_size > 1:
# Initialize a custom fast all-reduce implementation. # Initialize a custom fast all-reduce implementation.
self.ca_comm = CustomAllreduce( self.ca_comm = CustomAllreduce(
group=self.cpu_group, group=self.cpu_group,
@@ -108,13 +122,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
), ),
) )
if current_platform.is_rocm(): if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
# Initialize a custom quick all-reduce implementation for AMD. # Initialize a custom quick all-reduce implementation for AMD.
# Quick reduce is designed as a complement to custom allreduce. # Quick reduce is designed as a complement to custom allreduce
# Based on quickreduce (https://github.com/mk1-project/quickreduce). # (vLLM's or AITER's), so it is initialized for either backend.
# If it's a rocm, 'use_custom_allreduce==True' means it must # Based on quickreduce (https://github.com/mk1-project/quickreduce).
# currently be an MI300 series. # On ROCm, 'use_custom_allreduce==True' means it must currently be
self.qr_comm = QuickAllReduce(group=self.cpu_group, device=self.device) # an MI300 series.
self.qr_comm = QuickAllReduce(group=self.cpu_group, device=self.device)
if self.world_size > 1: if self.world_size > 1:
self._log_all_reduce_backend_selection() self._log_all_reduce_backend_selection()
@@ -203,6 +218,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
"NCCL_SYMM_MEM", "NCCL_SYMM_MEM",
"QUICK_REDUCE", "QUICK_REDUCE",
"FLASHINFER", "FLASHINFER",
"AITER_CUSTOM",
"CUSTOM", "CUSTOM",
"SYMM_MEM", "SYMM_MEM",
"PYNCCL", "PYNCCL",
@@ -236,6 +252,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
enabled_ar_backends.append("QUICK_REDUCE") enabled_ar_backends.append("QUICK_REDUCE")
if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled: if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled:
enabled_ar_backends.append("FLASHINFER") enabled_ar_backends.append("FLASHINFER")
if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled:
enabled_ar_backends.append("AITER_CUSTOM")
if self.ca_comm is not None and not self.ca_comm.disabled: if self.ca_comm is not None and not self.ca_comm.disabled:
enabled_ar_backends.append("CUSTOM") enabled_ar_backends.append("CUSTOM")
if self.symm_mem_comm is not None and not self.symm_mem_comm.disabled: if self.symm_mem_comm is not None and not self.symm_mem_comm.disabled:
@@ -261,8 +279,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
out = torch.ops.vllm.all_reduce_symmetric_with_copy(input_) out = torch.ops.vllm.all_reduce_symmetric_with_copy(input_)
if out is not None: if out is not None:
return out return out
# always try quick reduce first, then flashinfer, then custom allreduce, # always try quick reduce first, then flashinfer, then the AITER or vLLM
# and then pynccl. (quick reduce just for ROCM MI3*) # custom allreduce, and then pynccl. (quick reduce just for ROCM MI3*)
qr_comm = self.qr_comm qr_comm = self.qr_comm
if ( if (
qr_comm is not None qr_comm is not None
@@ -281,6 +299,15 @@ class CudaCommunicator(DeviceCommunicatorBase):
out = fi_ar_comm.all_reduce(input_) out = fi_ar_comm.all_reduce(input_)
assert out is not None assert out is not None
return out return out
aiter_ar_comm = self.aiter_ar_comm
if (
aiter_ar_comm is not None
and not aiter_ar_comm.disabled
and aiter_ar_comm.should_custom_ar(input_)
):
out = aiter_ar_comm.custom_all_reduce(input_)
assert out is not None
return out
ca_comm = self.ca_comm ca_comm = self.ca_comm
if ( if (
ca_comm is not None ca_comm is not None
@@ -509,6 +536,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.pynccl_comm = None self.pynccl_comm = None
if self.ca_comm is not None: if self.ca_comm is not None:
self.ca_comm = None self.ca_comm = None
if self.aiter_ar_comm is not None:
self.aiter_ar_comm.close()
self.aiter_ar_comm = None
if self.fi_ar_comm is not None: if self.fi_ar_comm is not None:
self.fi_ar_comm.destroy() self.fi_ar_comm.destroy()
self.fi_ar_comm = None self.fi_ar_comm = None
+6
View File
@@ -119,6 +119,7 @@ if TYPE_CHECKING:
VLLM_USE_OINK_OPS: bool = False VLLM_USE_OINK_OPS: bool = False
VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD: bool = True VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD: bool = True
VLLM_ROCM_USE_AITER: bool = False VLLM_ROCM_USE_AITER: bool = False
VLLM_ROCM_USE_AITER_CUSTOM_AR: bool = True
VLLM_ROCM_USE_AITER_PAGED_ATTN: bool = False VLLM_ROCM_USE_AITER_PAGED_ATTN: bool = False
VLLM_ROCM_USE_AITER_LINEAR: bool = True VLLM_ROCM_USE_AITER_LINEAR: bool = True
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
@@ -1146,6 +1147,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER": lambda: ( "VLLM_ROCM_USE_AITER": lambda: (
os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1") os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1")
), ),
# Use AITER's CustomAllreduce as the custom-allreduce backend inside vLLM's
# CudaCommunicator on ROCm.
"VLLM_ROCM_USE_AITER_CUSTOM_AR": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "True").lower() in ("true", "1")
),
# Whether to use aiter paged attention. # Whether to use aiter paged attention.
# By default is disabled. # By default is disabled.
"VLLM_ROCM_USE_AITER_PAGED_ATTN": lambda: ( "VLLM_ROCM_USE_AITER_PAGED_ATTN": lambda: (