[ROCm][FEAT] AITER Fused Allreduce + RMSNorm (#37646)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com>
Signed-off-by: junkang1991 <junkangchow@gmail.com>
Co-authored-by: Rita Brugarolas <Rita.BrugarolasBrufau@amd.com>
Co-authored-by: junkang1991 <junkangchow@gmail.com>
Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
vllmellm
2026-05-01 23:07:18 +08:00
committed by GitHub
co-authored by Rita Brugarolas junkang1991 Luka Govedič TJian
parent bc635fad23
commit 529c671e80
9 changed files with 454 additions and 20 deletions
+115
View File
@@ -2,9 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from collections.abc import Callable
from contextlib import contextmanager
from typing import Protocol
import torch
from torch._ops import OpOverload
from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm.platforms import current_platform
@@ -39,6 +42,27 @@ def is_aiter_found() -> bool:
IS_AITER_FOUND = is_aiter_found()
class AiterCustomAllreduceProto(Protocol):
max_size: int
world_size: int
fully_connected: bool
@contextmanager
def capture(self): ...
def close(self) -> None: ...
def fused_ar_rms(
self,
inp: torch.Tensor,
res_inp: torch.Tensor,
*,
w: torch.Tensor,
eps: float,
registered: bool = False,
use_1stage: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]: ...
def should_custom_ar(self, inp: torch.Tensor) -> bool: ...
def is_aiter_found_and_supported() -> bool:
"""Check if AITER library is available and platform supports it.
@@ -750,6 +774,55 @@ def _rocm_aiter_rmsnorm_fused_dynamic_quant_fake(
return out, y_scale
def _rocm_aiter_fused_allreduce_rmsnorm_impl(
input_: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized"
total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1]
token_num = input_.shape[0]
hidden_ok = hidden_dim in (512, 1024, 2048, 4096, 7168)
token_ok = token_num <= 80
world_size = aiter_ar.world_size
full_nvlink = aiter_ar.fully_connected
if world_size == 2:
size_ok = True
elif full_nvlink and world_size <= 4:
size_ok = total_bytes < 256 * 1024
elif full_nvlink and world_size <= 8:
size_ok = total_bytes < 128 * 1024
else:
size_ok = False
use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms(
input_,
residual,
w=weight,
eps=epsilon,
registered=torch.cuda.is_current_stream_capturing(),
use_1stage=use_1stage,
)
assert result is not None
return result[0], result[1]
def _rocm_aiter_fused_allreduce_rmsnorm_fake(
input_: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.empty_like(input_), torch.empty_like(residual)
def _rocm_aiter_per_tensor_quant_impl(
x: torch.Tensor,
quant_dtype: torch.dtype,
@@ -1188,6 +1261,9 @@ class rocm_aiter_ops:
# TODO: Consolidate under _LINEAR_ENABLED
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
_ALL_REDUCE_MAX_SIZE: int = 8192 * 1024 * 8 * 2
_CUSTOM_ALL_REDUCE: AiterCustomAllreduceProto | None = None
@classmethod
def refresh_env_variables(cls):
"""
@@ -1362,6 +1438,35 @@ class rocm_aiter_ops:
return cls.is_linear_enabled() and on_gfx950()
@classmethod
def initialize_aiter_allreduce(
cls, group: ProcessGroup, device: torch.device
) -> None:
try:
from aiter.dist.device_communicators.custom_all_reduce import (
CustomAllreduce as AiterCustomAllreduce,
)
cls._CUSTOM_ALL_REDUCE = AiterCustomAllreduce(group, device)
except Exception:
cls._CUSTOM_ALL_REDUCE = None
@classmethod
def get_aiter_allreduce(cls) -> AiterCustomAllreduceProto | None:
return cls._CUSTOM_ALL_REDUCE
@classmethod
def destroy_aiter_allreduce(cls) -> None:
if cls._CUSTOM_ALL_REDUCE is not None:
cls._CUSTOM_ALL_REDUCE.close()
cls._CUSTOM_ALL_REDUCE = None
@classmethod
def get_aiter_allreduce_max_size(cls) -> int | None:
# effective max input size (based on upstream aiter version: v0.1.10.post3)
# https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/aiter/dist/device_communicators/custom_all_reduce.py#L272-L273
return int(cls._ALL_REDUCE_MAX_SIZE / 2)
@staticmethod
@if_aiter_supported
def register_ops_once() -> None:
@@ -1552,6 +1657,12 @@ class rocm_aiter_ops:
fake_impl=_triton_rotary_embedding_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_fused_allreduce_rmsnorm",
op_func=_rocm_aiter_fused_allreduce_rmsnorm_impl,
fake_impl=_rocm_aiter_fused_allreduce_rmsnorm_fake,
)
direct_register_custom_op(
op_name="fused_mla_dual_rms_norm",
op_func=_fused_mla_dual_rms_norm_impl,
@@ -1605,6 +1716,10 @@ class rocm_aiter_ops:
def get_triton_rotary_embedding_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default
@staticmethod
def get_fused_allreduce_rmsnorm_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm.default
@staticmethod
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm.default