[ROCm][AITER] Directly Implement AITER Custom All-reduce in CudaCommunicator (#46065)
Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
+34
-95
@@ -2,12 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from typing import Protocol
|
||||
|
||||
import torch
|
||||
from torch._ops import OpOverload
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.platforms import current_platform
|
||||
@@ -52,42 +49,6 @@ def is_aiter_found() -> bool:
|
||||
IS_AITER_FOUND = is_aiter_found()
|
||||
|
||||
|
||||
class AiterCustomAllreduceProto(Protocol):
|
||||
max_size: int
|
||||
world_size: int
|
||||
fully_connected: bool
|
||||
|
||||
@contextmanager
|
||||
def capture(self): ...
|
||||
def close(self) -> None: ...
|
||||
def fused_ar_rms(
|
||||
self,
|
||||
inp: torch.Tensor,
|
||||
res_inp: torch.Tensor,
|
||||
*,
|
||||
w: torch.Tensor,
|
||||
eps: float,
|
||||
registered: bool = False,
|
||||
use_1stage: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]: ...
|
||||
def fused_ar_rms_per_group_quant(
|
||||
self,
|
||||
inp: torch.Tensor,
|
||||
res_inp: torch.Tensor,
|
||||
*,
|
||||
w: torch.Tensor,
|
||||
eps: float,
|
||||
group_size: int = 128,
|
||||
registered: bool = False,
|
||||
use_1stage: bool = False,
|
||||
emit_bf16: bool = False,
|
||||
) -> (
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
): ...
|
||||
def should_custom_ar(self, inp: torch.Tensor) -> bool: ...
|
||||
|
||||
|
||||
def is_aiter_found_and_supported() -> bool:
|
||||
"""Check if AITER library is available and platform supports it.
|
||||
|
||||
@@ -830,6 +791,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
assert aiter_ar is not None, "aiter allreduce must be initialized"
|
||||
ca = aiter_ar.aiter_ca
|
||||
|
||||
total_bytes = input_.numel() * input_.element_size()
|
||||
hidden_dim = input_.shape[-1]
|
||||
@@ -840,8 +802,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
else:
|
||||
hidden_ok = False
|
||||
token_ok = token_num <= 80
|
||||
world_size = aiter_ar.world_size
|
||||
full_nvlink = aiter_ar.fully_connected
|
||||
world_size = ca.world_size
|
||||
full_nvlink = ca.fully_connected
|
||||
|
||||
if world_size == 2:
|
||||
size_ok = True
|
||||
@@ -854,12 +816,11 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
|
||||
use_1stage = hidden_ok and token_ok and size_ok
|
||||
|
||||
result = aiter_ar.fused_ar_rms(
|
||||
result = ca.custom_fused_ar_rms(
|
||||
input_,
|
||||
residual,
|
||||
w=weight,
|
||||
eps=epsilon,
|
||||
registered=torch.cuda.is_current_stream_capturing(),
|
||||
weight,
|
||||
epsilon,
|
||||
use_1stage=use_1stage,
|
||||
)
|
||||
assert result is not None
|
||||
@@ -890,6 +851,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
|
||||
"""
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
assert aiter_ar is not None, "aiter allreduce must be initialized"
|
||||
ca = aiter_ar.aiter_ca
|
||||
|
||||
total_bytes = input_.numel() * input_.element_size()
|
||||
hidden_dim = input_.shape[-1]
|
||||
@@ -900,8 +862,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
|
||||
else:
|
||||
hidden_ok = False
|
||||
token_ok = token_num <= 80
|
||||
world_size = aiter_ar.world_size
|
||||
full_nvlink = aiter_ar.fully_connected
|
||||
world_size = ca.world_size
|
||||
full_nvlink = ca.fully_connected
|
||||
|
||||
if world_size == 2:
|
||||
size_ok = True
|
||||
@@ -914,7 +876,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
|
||||
|
||||
use_1stage = hidden_ok and token_ok and size_ok
|
||||
|
||||
result = aiter_ar.fused_ar_rms_per_group_quant(
|
||||
result = ca.fused_ar_rms_per_group_quant(
|
||||
input_,
|
||||
residual,
|
||||
w=weight,
|
||||
@@ -962,6 +924,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
|
||||
"""
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
assert aiter_ar is not None, "aiter allreduce must be initialized"
|
||||
ca = aiter_ar.aiter_ca
|
||||
|
||||
total_bytes = input_.numel() * input_.element_size()
|
||||
hidden_dim = input_.shape[-1]
|
||||
@@ -972,8 +935,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
|
||||
else:
|
||||
hidden_ok = False
|
||||
token_ok = token_num <= 80
|
||||
world_size = aiter_ar.world_size
|
||||
full_nvlink = aiter_ar.fully_connected
|
||||
world_size = ca.world_size
|
||||
full_nvlink = ca.fully_connected
|
||||
|
||||
if world_size == 2:
|
||||
size_ok = True
|
||||
@@ -986,7 +949,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
|
||||
|
||||
use_1stage = hidden_ok and token_ok and size_ok
|
||||
|
||||
result = aiter_ar.fused_ar_rms_per_group_quant(
|
||||
result = ca.fused_ar_rms_per_group_quant(
|
||||
input_,
|
||||
residual,
|
||||
w=weight,
|
||||
@@ -1577,6 +1540,7 @@ class rocm_aiter_ops:
|
||||
|
||||
# Check if the env variable is set
|
||||
_AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
|
||||
_CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR
|
||||
_LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
|
||||
_FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE
|
||||
_MLA_ENABLED = envs.VLLM_ROCM_USE_AITER_MLA
|
||||
@@ -1598,9 +1562,6 @@ class rocm_aiter_ops:
|
||||
# num_shared_experts / shared_expert_scoring_func args (7-arg form).
|
||||
_TOPK_SOFTMAX_FUSED_SIGMOID: bool | None = None
|
||||
|
||||
_ALL_REDUCE_MAX_SIZE: int = 8192 * 1024 * 8 * 2
|
||||
_CUSTOM_ALL_REDUCE: AiterCustomAllreduceProto | None = None
|
||||
|
||||
@classmethod
|
||||
def refresh_env_variables(cls):
|
||||
"""
|
||||
@@ -1611,6 +1572,7 @@ class rocm_aiter_ops:
|
||||
you can call this function to reload the env variables.
|
||||
"""
|
||||
cls._AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
|
||||
cls._CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR
|
||||
cls._LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
|
||||
cls._FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE
|
||||
cls._MLA_ENABLED = envs.VLLM_ROCM_USE_AITER_MLA
|
||||
@@ -1770,6 +1732,11 @@ class rocm_aiter_ops:
|
||||
def is_mha_enabled(cls) -> bool:
|
||||
return cls._AITER_ENABLED and cls._MHA_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_custom_all_reduce_enabled(cls) -> bool:
|
||||
return cls._AITER_ENABLED and cls._CUSTOM_ALL_REDUCE_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_shuffle_kv_cache_enabled(cls) -> bool:
|
||||
@@ -1824,33 +1791,20 @@ class rocm_aiter_ops:
|
||||
return cls.is_linear_enabled() and on_gfx950()
|
||||
|
||||
@classmethod
|
||||
def initialize_aiter_allreduce(
|
||||
cls, group: ProcessGroup, device: torch.device
|
||||
) -> None:
|
||||
try:
|
||||
from aiter.dist.device_communicators.custom_all_reduce import (
|
||||
CustomAllreduce as AiterCustomAllreduce,
|
||||
)
|
||||
def get_aiter_allreduce(cls):
|
||||
"""Return the TP device communicator's AITER custom-allreduce if it has
|
||||
one, return None otherwise
|
||||
"""
|
||||
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
|
||||
AiterCustomAllreduce,
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
cls._CUSTOM_ALL_REDUCE = AiterCustomAllreduce(group, device)
|
||||
except Exception:
|
||||
cls._CUSTOM_ALL_REDUCE = None
|
||||
|
||||
@classmethod
|
||||
def get_aiter_allreduce(cls) -> AiterCustomAllreduceProto | None:
|
||||
return cls._CUSTOM_ALL_REDUCE
|
||||
|
||||
@classmethod
|
||||
def destroy_aiter_allreduce(cls) -> None:
|
||||
if cls._CUSTOM_ALL_REDUCE is not None:
|
||||
cls._CUSTOM_ALL_REDUCE.close()
|
||||
cls._CUSTOM_ALL_REDUCE = None
|
||||
|
||||
@classmethod
|
||||
def get_aiter_allreduce_max_size(cls) -> int | None:
|
||||
# effective max input size (based on upstream aiter version: v0.1.10.post3)
|
||||
# https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/aiter/dist/device_communicators/custom_all_reduce.py#L272-L273
|
||||
return int(cls._ALL_REDUCE_MAX_SIZE / 2)
|
||||
device_comm = get_tp_group().device_communicator
|
||||
aiter_ar_comm = getattr(device_comm, "aiter_ar_comm", None)
|
||||
return (
|
||||
aiter_ar_comm if isinstance(aiter_ar_comm, AiterCustomAllreduce) else None
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
@@ -2165,21 +2119,6 @@ class rocm_aiter_ops:
|
||||
def get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() -> OpOverload: # noqa: E501
|
||||
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm.default # noqa: E501
|
||||
|
||||
# TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823).
|
||||
@classmethod
|
||||
def has_fused_allreduce_rmsnorm_quant_per_group(cls) -> bool:
|
||||
"""True if the running AITER build exposes the per-group AR+RMS+quant
|
||||
kernel (added in ROCm/aiter PR #2823).
|
||||
|
||||
The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off
|
||||
this so vLLM degrades to the AR+RMS-only fusion when run against an
|
||||
older aiter that lacks the per-group launcher.
|
||||
"""
|
||||
aiter_ar = cls.get_aiter_allreduce()
|
||||
return aiter_ar is not None and hasattr(
|
||||
aiter_ar, "fused_ar_rms_per_group_quant"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
|
||||
return torch.ops.vllm.fused_mla_dual_rms_norm.default
|
||||
|
||||
Reference in New Issue
Block a user