[ROCm][Perf][M3] Fused allreduce+GemmaRMSNorm fast path (#54787)
Signed-off-by: Taoyu Zhu <zhutaoyu97@gmail.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Shanshan Shen
parent
81dabe2e5d
commit
ce6c241ddc
@@ -23,6 +23,7 @@ from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MiB = 1024 * 1024
|
||||
|
||||
@@ -111,6 +112,30 @@ def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool
|
||||
return True, max_token_num
|
||||
|
||||
|
||||
def _can_use_aiter_fused_ar_rms(hidden_states: torch.Tensor) -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_custom_all_reduce_enabled():
|
||||
return False
|
||||
if (
|
||||
hidden_states.dim() != 2
|
||||
or not hidden_states.is_contiguous()
|
||||
or hidden_states.dtype not in _FI_SUPPORTED_DTYPES
|
||||
):
|
||||
return False
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
if aiter_ar is None or aiter_ar.disabled:
|
||||
return False
|
||||
total_bytes = hidden_states.numel() * hidden_states.element_size()
|
||||
if total_bytes > aiter_ar.effective_max_size():
|
||||
return False
|
||||
if not aiter_ar.should_custom_ar(hidden_states):
|
||||
return False
|
||||
return aiter_ar.use_1stage_fused_ar_rms(hidden_states)
|
||||
|
||||
|
||||
def fused_allreduce_gemma_rms_norm(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -149,6 +174,17 @@ def fused_allreduce_gemma_rms_norm(
|
||||
)
|
||||
return norm_out, hidden_states
|
||||
|
||||
if _can_use_aiter_fused_ar_rms(hidden_states):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()(
|
||||
input_=hidden_states,
|
||||
residual=residual,
|
||||
weight=norm.weight,
|
||||
epsilon=norm.variance_epsilon,
|
||||
gemma_norm=True,
|
||||
)
|
||||
|
||||
# Fallback: explicit all-reduce + GemmaRMSNorm (matches the unfused model).
|
||||
reduced = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return norm(reduced, residual)
|
||||
|
||||
Reference in New Issue
Block a user