[ROCm][FEAT] AITER Fused Allreduce + RMSNorm (#37646)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com>
Signed-off-by: junkang1991 <junkangchow@gmail.com>
Co-authored-by: Rita Brugarolas <Rita.BrugarolasBrufau@amd.com>
Co-authored-by: junkang1991 <junkangchow@gmail.com>
Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
vllmellm
2026-05-01 23:07:18 +08:00
committed by GitHub
co-authored by Rita Brugarolas junkang1991 Luka Govedič TJian
parent bc635fad23
commit 529c671e80
9 changed files with 454 additions and 20 deletions
@@ -190,6 +190,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern):
is_scale_transposed: bool = False,
is_e8m0: bool = False,
is_tma_aligned: bool = False,
match_aiter: bool = False,
) -> None:
super().__init__(quant_key)
self.quant_matcher = MatcherQuantFP8(
@@ -12,12 +12,14 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
from torch._inductor.pattern_matcher import PatternMatcherPass
import vllm.ir.ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.passes.fusion.rms_quant_fusion import (
_rms_input_weight_dtype_match,
)
from vllm.config import VllmConfig
from vllm.config.utils import Range
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -32,7 +34,12 @@ from vllm.utils.torch_utils import (
)
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
from ..vllm_inductor_pass import (
VllmFusionPatternMatcherPass,
VllmInductorPass,
VllmPatternMatcherPass,
VllmPatternReplacement,
)
from .matcher_utils import MatcherFusedAddRMSNorm, MatcherQuantFP8
FP8_DTYPE = current_platform.fp8_dtype()
@@ -889,3 +896,204 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
return
with contextlib.suppress(Exception):
destroy_fi_ar_workspace()
# TODO: make BasePattern to inherit from VllmPatternReplacement
class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
def __init__(
self,
epsilon: float,
dtype: torch.dtype,
device: str | None,
use_aiter_rmsnorm: bool = True,
) -> None:
super().__init__(dtype, device)
self.dtype = dtype
self.epsilon = epsilon
self.FUSED_AR_RMSNORM_OP = rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()
def get_inputs(self) -> list[torch.Tensor]:
return [self.empty(5, 16), self.empty(16)]
@property
def pattern(self):
def _pattern(
input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms = vllm.ir.ops.rms_norm(allreduce_output, weight, self.epsilon)
return rms, allreduce_output
return _pattern
@property
def replacement(self):
def _replacement(
input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
residual = torch.empty_like(input)
allreduce = self.FUSED_AR_RMSNORM_OP(
input_=input,
residual=residual,
weight=weight,
epsilon=self.epsilon,
)
return allreduce[0], allreduce[1]
return _replacement
class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
def __init__(
self,
epsilon: float,
dtype: torch.dtype,
device: str | None,
use_aiter_rmsnorm: bool = True,
) -> None:
super().__init__(dtype, device)
self.epsilon = epsilon
self.dtype = dtype
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(
epsilon, match_rocm_aiter=use_aiter_rmsnorm
)
self.FUSED_AR_RMSNORM_OP = rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()
def get_inputs(self) -> list[torch.Tensor]:
input, residual, weight = self.rmsnorm_matcher.inputs()
return [residual, input.to(self.dtype), weight]
@property
def pattern(self):
def _pattern(
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms, residual = self.rmsnorm_matcher(allreduce_output, weight, residual)
return rms, residual
return _pattern
@property
def replacement(self):
def _replacement(
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce = self.FUSED_AR_RMSNORM_OP(
input_=input,
residual=residual,
weight=weight,
epsilon=self.epsilon,
)
return allreduce[0], allreduce[1]
return _replacement
class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "rocm_aiter_allreduce_fusion_pass")
self.disabled = True
self.tp_size = get_tensor_model_parallel_world_size()
if self.tp_size <= 1:
logger.warning_once("AllReduce fusion pass is disabled for tp_size <= 1.")
return
if config.model_config is None:
logger.warning_once(
"AllReduce fusion pass is disabled for missing model_config."
)
return
device_comm = get_tp_group().device_communicator
if device_comm is None:
logger.warning_once("Device communicator is required.")
return
ca_comm = getattr(device_comm, "ca_comm", None)
if ca_comm is None:
logger.warning_once("Custom Allreduce is required.")
return
self.ca_comm = ca_comm
assert isinstance(ca_comm, CustomAllreduce)
group = get_tp_group().cpu_group
rocm_aiter_ops.initialize_aiter_allreduce(group, self.device)
hidden_dim = config.model_config.get_hidden_size()
element_size = torch.tensor([], dtype=self.model_dtype).element_size()
max_size = rocm_aiter_ops.get_aiter_allreduce_max_size()
if max_size is None:
logger.warning("AITER allreduce fusion must be initialized")
return
# Aiter's fused_allreduce_rmsnorm kernel dispatches on hidden_dim.
# Before aiter v0.1.12 the launcher was template-specialized on HIDDEN_DIM
# and silently no-op'd for sizes outside {512, 1024, 2048, 4096}. From v0.1.12
# hidden_dim is a runtime argument. Detect the older API via the missing
# `_pool` attribute and skip fusion for unsupported sizes.
# Ref (old kernel): https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/csrc/include/custom_all_reduce.cuh#L2590
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
_AITER_OLD_FUSED_AR_RMS_HIDDEN = (512, 1024, 2048, 4096)
if (
aiter_ar is not None
and not hasattr(aiter_ar, "_pool")
and hidden_dim not in _AITER_OLD_FUSED_AR_RMS_HIDDEN
):
logger.warning_once(
"AITER allreduce-rmsnorm fusion disabled: aiter<0.1.12 "
"only supports hidden_dim in %s; got %d. Upgrade aiter to "
">=0.1.12 to enable fusion for this model.",
_AITER_OLD_FUSED_AR_RMS_HIDDEN,
hidden_dim,
)
# Tear down aiter's custom-allreduce so its IPC handles don't
# race with vllm's ca_comm on the unfused fallback path.
with contextlib.suppress(Exception):
rocm_aiter_ops.destroy_aiter_allreduce()
return
max_token_num = max_size // (hidden_dim * element_size)
self.max_token_num = min(
max_token_num,
config.scheduler_config.max_num_batched_tokens,
)
for epsilon in [1e-5, 1e-6]:
self.register(
AiterAllreduceFusedRMSNormPattern(
epsilon,
self.model_dtype,
self.device,
)
)
self.register(
AiterAllreduceFusedAddRMSNormPattern(
epsilon,
self.model_dtype,
self.device,
)
)
# WARNING: This is a hack to clear the pattern matcher cache
# and allow multiple values of epsilon.
torch._inductor.pattern_matcher._seen_patterns.clear()
self.disabled = False
self.dump_patterns(config, self.pm_pass)
def is_applicable_for_range(self, compile_range: Range) -> bool:
if self.disabled:
logger.warning_once("AllReduce fusion pass is disabled.")
return False
return bool(compile_range.end <= self.max_token_num)
def __del__(self) -> None:
if getattr(self, "disabled", True):
return
with contextlib.suppress(Exception):
rocm_aiter_ops.destroy_aiter_allreduce()
+7 -1
View File
@@ -18,6 +18,9 @@ from .ir.lowering_pass import VllmIRLoweringPass
from .vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
if rocm_aiter_ops.is_enabled():
from .fusion.allreduce_rms_fusion import (
RocmAiterAllReduceFusionPass,
)
from .fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
RocmAiterRMSNormQuantFusionPass,
@@ -137,7 +140,10 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
self.passes += [AsyncTPPass(config)]
if self.pass_config.fuse_allreduce_rms:
self.passes += [AllReduceFusionPass(config)]
if rocm_aiter_ops.is_enabled():
self.passes += [RocmAiterAllReduceFusionPass(config)]
else:
self.passes += [AllReduceFusionPass(config)]
if self.pass_config.fuse_minimax_qk_norm:
self.passes += [MiniMaxQKNormPass(config)]