[Perf][ROCm] Enable AITER QuickReduce + RMSNorm fusion (#48249)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Mikko Tukiainen
2026-09-17 15:05:48 +00:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4f9a41ac94
commit 1df336c3ba
2 changed files with 278 additions and 1 deletions
+106 -1
View File
@@ -4,9 +4,10 @@ import ctypes
import functools
import os
from collections.abc import Callable
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
import torch.distributed as dist
from torch._ops import OpOverload
import vllm.envs as envs
@@ -34,6 +35,63 @@ except ImportError:
# which is a host op, so we cache it once here.
FP8_DTYPE = current_platform.fp8_dtype()
_HIPB_MM_INITIALIZED_DEVICES: set[int] = set()
KB = 1024
MB = 1024 * KB
def _get_or_create_aiter_qr_rmsnorm_comm(
device_comm: Any,
) -> tuple[int, int, bool] | None:
"""Create the AITER QR communicator used by fused QR+RMSNorm.
vLLM's normal QuickReduce communicator owns vLLM C++ QR buffers. The fused
QR+RMSNorm kernel added in AITER needs an AITER DeviceComms instance, so we
lazily create one on the same TP CPU group and keep it on the vLLM device
communicator for reuse.
"""
existing = getattr(device_comm, "_aiter_qr_rmsnorm_comm", None)
if existing is False:
return None
if existing is not None:
return existing
try:
import aiter
except Exception:
device_comm._aiter_qr_rmsnorm_comm = False
return None
if not hasattr(aiter, "qr_all_reduce_rmsnorm"):
device_comm._aiter_qr_rmsnorm_comm = False
return None
group = getattr(device_comm, "cpu_group", None)
if group is None:
device_comm._aiter_qr_rmsnorm_comm = False
return None
world_size = dist.get_world_size(group=group)
rank = dist.get_rank(group=group)
if world_size not in (2, 4, 8):
device_comm._aiter_qr_rmsnorm_comm = False
return None
qr_max_size_mb = envs.VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB
qr_max_size = None if qr_max_size_mb is None else qr_max_size_mb * MB
try:
ptr = aiter.init_custom_qr(rank, world_size, qr_max_size)
handle = aiter.qr_get_handle(ptr)
handles = [None] * world_size
dist.all_gather_object(handles, handle, group=group)
aiter.qr_open_handles(ptr, handles)
except Exception:
device_comm._aiter_qr_rmsnorm_comm = False
return None
comm = (ptr, world_size, envs.VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16)
device_comm._aiter_qr_rmsnorm_comm = comm
return comm
def _ensure_hipb_mm_extension_initialized() -> None:
@@ -991,6 +1049,53 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
ca = aiter_ar.aiter_ca
use_1stage = aiter_ar.use_1stage_fused_ar_rms(input_)
if not use_1stage:
from vllm.distributed import get_tp_group
device_comm = get_tp_group().device_communicator
qr_comm = getattr(device_comm, "qr_comm", None)
hidden_dim = input_.shape[-1]
row_size = hidden_dim * input_.element_size()
fused_qr_rmsnorm_ok = (
qr_comm is not None
and not getattr(qr_comm, "disabled", True)
and hasattr(qr_comm, "should_quick_allreduce")
and qr_comm.should_quick_allreduce(input_)
and input_.shape == residual.shape
and input_.dtype in (torch.bfloat16, torch.float16)
and input_.dtype == residual.dtype
and input_.dtype == weight.dtype
and weight.dim() == 1
and weight.numel() == hidden_dim
and input_.numel() % hidden_dim == 0
and row_size > 0
and row_size <= 32 * KB
and (32 * KB) % row_size == 0
)
if fused_qr_rmsnorm_ok:
assert qr_comm is not None
aiter_qr_comm = _get_or_create_aiter_qr_rmsnorm_comm(device_comm)
if aiter_qr_comm is not None:
import aiter
ptr, _, cast_bf2half = aiter_qr_comm
quant_level = qr_comm._get_qr_quant_level(input_)
out = torch.empty_like(input_)
residual_out = torch.empty_like(residual)
aiter.qr_all_reduce_rmsnorm(
ptr,
input_,
residual,
residual_out,
out,
weight,
epsilon,
hidden_dim,
quant_level,
cast_bf2half,
)
return out, residual_out
result = ca.custom_fused_ar_rms(
input_,
residual,