[Perf][ROCm] Enable AITER QuickReduce + RMSNorm fusion (#48249)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4f9a41ac94
commit
1df336c3ba
+106
-1
@@ -4,9 +4,10 @@ import ctypes
|
||||
import functools
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch._ops import OpOverload
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -34,6 +35,63 @@ except ImportError:
|
||||
# which is a host op, so we cache it once here.
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
_HIPB_MM_INITIALIZED_DEVICES: set[int] = set()
|
||||
KB = 1024
|
||||
MB = 1024 * KB
|
||||
|
||||
|
||||
def _get_or_create_aiter_qr_rmsnorm_comm(
|
||||
device_comm: Any,
|
||||
) -> tuple[int, int, bool] | None:
|
||||
"""Create the AITER QR communicator used by fused QR+RMSNorm.
|
||||
|
||||
vLLM's normal QuickReduce communicator owns vLLM C++ QR buffers. The fused
|
||||
QR+RMSNorm kernel added in AITER needs an AITER DeviceComms instance, so we
|
||||
lazily create one on the same TP CPU group and keep it on the vLLM device
|
||||
communicator for reuse.
|
||||
"""
|
||||
existing = getattr(device_comm, "_aiter_qr_rmsnorm_comm", None)
|
||||
if existing is False:
|
||||
return None
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
try:
|
||||
import aiter
|
||||
except Exception:
|
||||
device_comm._aiter_qr_rmsnorm_comm = False
|
||||
return None
|
||||
|
||||
if not hasattr(aiter, "qr_all_reduce_rmsnorm"):
|
||||
device_comm._aiter_qr_rmsnorm_comm = False
|
||||
return None
|
||||
|
||||
group = getattr(device_comm, "cpu_group", None)
|
||||
if group is None:
|
||||
device_comm._aiter_qr_rmsnorm_comm = False
|
||||
return None
|
||||
|
||||
world_size = dist.get_world_size(group=group)
|
||||
rank = dist.get_rank(group=group)
|
||||
if world_size not in (2, 4, 8):
|
||||
device_comm._aiter_qr_rmsnorm_comm = False
|
||||
return None
|
||||
|
||||
qr_max_size_mb = envs.VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB
|
||||
qr_max_size = None if qr_max_size_mb is None else qr_max_size_mb * MB
|
||||
|
||||
try:
|
||||
ptr = aiter.init_custom_qr(rank, world_size, qr_max_size)
|
||||
handle = aiter.qr_get_handle(ptr)
|
||||
handles = [None] * world_size
|
||||
dist.all_gather_object(handles, handle, group=group)
|
||||
aiter.qr_open_handles(ptr, handles)
|
||||
except Exception:
|
||||
device_comm._aiter_qr_rmsnorm_comm = False
|
||||
return None
|
||||
|
||||
comm = (ptr, world_size, envs.VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16)
|
||||
device_comm._aiter_qr_rmsnorm_comm = comm
|
||||
return comm
|
||||
|
||||
|
||||
def _ensure_hipb_mm_extension_initialized() -> None:
|
||||
@@ -991,6 +1049,53 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
ca = aiter_ar.aiter_ca
|
||||
use_1stage = aiter_ar.use_1stage_fused_ar_rms(input_)
|
||||
|
||||
if not use_1stage:
|
||||
from vllm.distributed import get_tp_group
|
||||
|
||||
device_comm = get_tp_group().device_communicator
|
||||
qr_comm = getattr(device_comm, "qr_comm", None)
|
||||
hidden_dim = input_.shape[-1]
|
||||
row_size = hidden_dim * input_.element_size()
|
||||
fused_qr_rmsnorm_ok = (
|
||||
qr_comm is not None
|
||||
and not getattr(qr_comm, "disabled", True)
|
||||
and hasattr(qr_comm, "should_quick_allreduce")
|
||||
and qr_comm.should_quick_allreduce(input_)
|
||||
and input_.shape == residual.shape
|
||||
and input_.dtype in (torch.bfloat16, torch.float16)
|
||||
and input_.dtype == residual.dtype
|
||||
and input_.dtype == weight.dtype
|
||||
and weight.dim() == 1
|
||||
and weight.numel() == hidden_dim
|
||||
and input_.numel() % hidden_dim == 0
|
||||
and row_size > 0
|
||||
and row_size <= 32 * KB
|
||||
and (32 * KB) % row_size == 0
|
||||
)
|
||||
if fused_qr_rmsnorm_ok:
|
||||
assert qr_comm is not None
|
||||
aiter_qr_comm = _get_or_create_aiter_qr_rmsnorm_comm(device_comm)
|
||||
if aiter_qr_comm is not None:
|
||||
import aiter
|
||||
|
||||
ptr, _, cast_bf2half = aiter_qr_comm
|
||||
quant_level = qr_comm._get_qr_quant_level(input_)
|
||||
out = torch.empty_like(input_)
|
||||
residual_out = torch.empty_like(residual)
|
||||
aiter.qr_all_reduce_rmsnorm(
|
||||
ptr,
|
||||
input_,
|
||||
residual,
|
||||
residual_out,
|
||||
out,
|
||||
weight,
|
||||
epsilon,
|
||||
hidden_dim,
|
||||
quant_level,
|
||||
cast_bf2half,
|
||||
)
|
||||
return out, residual_out
|
||||
|
||||
result = ca.custom_fused_ar_rms(
|
||||
input_,
|
||||
residual,
|
||||
|
||||
Reference in New Issue
Block a user