[ROCm][Perf][M3] Fused allreduce+GemmaRMSNorm fast path (#54787)

Signed-off-by: Taoyu Zhu <zhutaoyu97@gmail.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
Taoyu Zhu
2026-09-08 07:21:49 +00:00
committed by GitHub
co-authored by Shanshan Shen
parent 81dabe2e5d
commit ce6c241ddc
3 changed files with 77 additions and 73 deletions
@@ -23,6 +23,7 @@ from vllm.distributed.parallel_state import (
get_tp_group,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.platforms import current_platform
MiB = 1024 * 1024
@@ -111,6 +112,30 @@ def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool
return True, max_token_num
def _can_use_aiter_fused_ar_rms(hidden_states: torch.Tensor) -> bool:
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_custom_all_reduce_enabled():
return False
if (
hidden_states.dim() != 2
or not hidden_states.is_contiguous()
or hidden_states.dtype not in _FI_SUPPORTED_DTYPES
):
return False
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
if aiter_ar is None or aiter_ar.disabled:
return False
total_bytes = hidden_states.numel() * hidden_states.element_size()
if total_bytes > aiter_ar.effective_max_size():
return False
if not aiter_ar.should_custom_ar(hidden_states):
return False
return aiter_ar.use_1stage_fused_ar_rms(hidden_states)
def fused_allreduce_gemma_rms_norm(
hidden_states: torch.Tensor,
residual: torch.Tensor,
@@ -149,6 +174,17 @@ def fused_allreduce_gemma_rms_norm(
)
return norm_out, hidden_states
if _can_use_aiter_fused_ar_rms(hidden_states):
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()(
input_=hidden_states,
residual=residual,
weight=norm.weight,
epsilon=norm.variance_epsilon,
gemma_norm=True,
)
# Fallback: explicit all-reduce + GemmaRMSNorm (matches the unfused model).
reduced = tensor_model_parallel_all_reduce(hidden_states)
return norm(reduced, residual)