[ROCm][FEAT] AITER Fused Allreduce + RMSNorm (#37646)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com> Signed-off-by: junkang1991 <junkangchow@gmail.com> Co-authored-by: Rita Brugarolas <Rita.BrugarolasBrufau@amd.com> Co-authored-by: junkang1991 <junkangchow@gmail.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Rita Brugarolas
junkang1991
Luka Govedič
TJian
parent
bc635fad23
commit
529c671e80
@@ -2,9 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from typing import Protocol
|
||||
|
||||
import torch
|
||||
from torch._ops import OpOverload
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.platforms import current_platform
|
||||
@@ -39,6 +42,27 @@ def is_aiter_found() -> bool:
|
||||
IS_AITER_FOUND = is_aiter_found()
|
||||
|
||||
|
||||
class AiterCustomAllreduceProto(Protocol):
|
||||
max_size: int
|
||||
world_size: int
|
||||
fully_connected: bool
|
||||
|
||||
@contextmanager
|
||||
def capture(self): ...
|
||||
def close(self) -> None: ...
|
||||
def fused_ar_rms(
|
||||
self,
|
||||
inp: torch.Tensor,
|
||||
res_inp: torch.Tensor,
|
||||
*,
|
||||
w: torch.Tensor,
|
||||
eps: float,
|
||||
registered: bool = False,
|
||||
use_1stage: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]: ...
|
||||
def should_custom_ar(self, inp: torch.Tensor) -> bool: ...
|
||||
|
||||
|
||||
def is_aiter_found_and_supported() -> bool:
|
||||
"""Check if AITER library is available and platform supports it.
|
||||
|
||||
@@ -750,6 +774,55 @@ def _rocm_aiter_rmsnorm_fused_dynamic_quant_fake(
|
||||
return out, y_scale
|
||||
|
||||
|
||||
def _rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
input_: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
assert aiter_ar is not None, "aiter allreduce must be initialized"
|
||||
|
||||
total_bytes = input_.numel() * input_.element_size()
|
||||
hidden_dim = input_.shape[-1]
|
||||
token_num = input_.shape[0]
|
||||
hidden_ok = hidden_dim in (512, 1024, 2048, 4096, 7168)
|
||||
token_ok = token_num <= 80
|
||||
world_size = aiter_ar.world_size
|
||||
full_nvlink = aiter_ar.fully_connected
|
||||
|
||||
if world_size == 2:
|
||||
size_ok = True
|
||||
elif full_nvlink and world_size <= 4:
|
||||
size_ok = total_bytes < 256 * 1024
|
||||
elif full_nvlink and world_size <= 8:
|
||||
size_ok = total_bytes < 128 * 1024
|
||||
else:
|
||||
size_ok = False
|
||||
|
||||
use_1stage = hidden_ok and token_ok and size_ok
|
||||
|
||||
result = aiter_ar.fused_ar_rms(
|
||||
input_,
|
||||
residual,
|
||||
w=weight,
|
||||
eps=epsilon,
|
||||
registered=torch.cuda.is_current_stream_capturing(),
|
||||
use_1stage=use_1stage,
|
||||
)
|
||||
assert result is not None
|
||||
return result[0], result[1]
|
||||
|
||||
|
||||
def _rocm_aiter_fused_allreduce_rmsnorm_fake(
|
||||
input_: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.empty_like(input_), torch.empty_like(residual)
|
||||
|
||||
|
||||
def _rocm_aiter_per_tensor_quant_impl(
|
||||
x: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
@@ -1188,6 +1261,9 @@ class rocm_aiter_ops:
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
|
||||
_ALL_REDUCE_MAX_SIZE: int = 8192 * 1024 * 8 * 2
|
||||
_CUSTOM_ALL_REDUCE: AiterCustomAllreduceProto | None = None
|
||||
|
||||
@classmethod
|
||||
def refresh_env_variables(cls):
|
||||
"""
|
||||
@@ -1362,6 +1438,35 @@ class rocm_aiter_ops:
|
||||
|
||||
return cls.is_linear_enabled() and on_gfx950()
|
||||
|
||||
@classmethod
|
||||
def initialize_aiter_allreduce(
|
||||
cls, group: ProcessGroup, device: torch.device
|
||||
) -> None:
|
||||
try:
|
||||
from aiter.dist.device_communicators.custom_all_reduce import (
|
||||
CustomAllreduce as AiterCustomAllreduce,
|
||||
)
|
||||
|
||||
cls._CUSTOM_ALL_REDUCE = AiterCustomAllreduce(group, device)
|
||||
except Exception:
|
||||
cls._CUSTOM_ALL_REDUCE = None
|
||||
|
||||
@classmethod
|
||||
def get_aiter_allreduce(cls) -> AiterCustomAllreduceProto | None:
|
||||
return cls._CUSTOM_ALL_REDUCE
|
||||
|
||||
@classmethod
|
||||
def destroy_aiter_allreduce(cls) -> None:
|
||||
if cls._CUSTOM_ALL_REDUCE is not None:
|
||||
cls._CUSTOM_ALL_REDUCE.close()
|
||||
cls._CUSTOM_ALL_REDUCE = None
|
||||
|
||||
@classmethod
|
||||
def get_aiter_allreduce_max_size(cls) -> int | None:
|
||||
# effective max input size (based on upstream aiter version: v0.1.10.post3)
|
||||
# https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/aiter/dist/device_communicators/custom_all_reduce.py#L272-L273
|
||||
return int(cls._ALL_REDUCE_MAX_SIZE / 2)
|
||||
|
||||
@staticmethod
|
||||
@if_aiter_supported
|
||||
def register_ops_once() -> None:
|
||||
@@ -1552,6 +1657,12 @@ class rocm_aiter_ops:
|
||||
fake_impl=_triton_rotary_embedding_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_allreduce_rmsnorm",
|
||||
op_func=_rocm_aiter_fused_allreduce_rmsnorm_impl,
|
||||
fake_impl=_rocm_aiter_fused_allreduce_rmsnorm_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_mla_dual_rms_norm",
|
||||
op_func=_fused_mla_dual_rms_norm_impl,
|
||||
@@ -1605,6 +1716,10 @@ class rocm_aiter_ops:
|
||||
def get_triton_rotary_embedding_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_allreduce_rmsnorm_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
|
||||
return torch.ops.vllm.fused_mla_dual_rms_norm.default
|
||||
|
||||
Reference in New Issue
Block a user