[ROCm][FEAT] AITER Fused Allreduce + RMSNorm (#37646)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com> Signed-off-by: junkang1991 <junkangchow@gmail.com> Co-authored-by: Rita Brugarolas <Rita.BrugarolasBrufau@amd.com> Co-authored-by: junkang1991 <junkangchow@gmail.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Rita Brugarolas
junkang1991
Luka Govedič
TJian
parent
bc635fad23
commit
529c671e80
@@ -19,6 +19,8 @@ from .models import (
|
||||
FLASHINFER_ATTN,
|
||||
FLASHINFER_MLA_ATTN,
|
||||
FLASHMLA_SPARSE_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
ROCM_ATTN,
|
||||
TRITON_ATTN,
|
||||
deepseek_coder_v2_lite_fp8,
|
||||
deepseek_r1_fp4,
|
||||
@@ -34,7 +36,9 @@ from .models import (
|
||||
qwen3_a3b_fp8,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only test CUDA/ROCm"
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@@ -55,6 +59,7 @@ pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only tes
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
def test_tp2_ar_rms_fp8_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
@@ -124,6 +129,7 @@ def test_tp2_ar_rms_fp8_fusions(
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
def test_tp2_ar_rms_fp4_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
@@ -176,10 +182,19 @@ def test_tp2_ar_rms_fp4_fusions(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b, qwen3_a3b, gpt_oss_20b],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[
|
||||
TRITON_ATTN,
|
||||
FLASHINFER_ATTN,
|
||||
ROCM_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("custom_ops", tuple(custom_ops_combos("rms_norm")))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Only test CUDA/ROCm")
|
||||
def test_tp2_ar_rms_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
@@ -221,4 +236,5 @@ def test_tp2_ar_rms_fusions(
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
use_aiter=current_platform.is_rocm(),
|
||||
)
|
||||
|
||||
@@ -8,8 +8,12 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.utils import TestFP8Layer, has_module_attribute, multi_gpu_test
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.allreduce_rms_fusion import AllReduceFusionPass
|
||||
from vllm.compilation.passes.fusion.allreduce_rms_fusion import (
|
||||
AllReduceFusionPass,
|
||||
RocmAiterAllReduceFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.fix_functionalization import (
|
||||
FixFunctionalizationPass,
|
||||
)
|
||||
@@ -42,13 +46,19 @@ DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
class TestAllReduceRMSNormModel(torch.nn.Module):
|
||||
def __init__(
|
||||
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
|
||||
self,
|
||||
hidden_size=16,
|
||||
token_num=16,
|
||||
eps=1e-6,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
use_aiter: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
|
||||
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
|
||||
self.use_aiter = use_aiter
|
||||
|
||||
def forward(self, x):
|
||||
# avoid having graph input be an arg to a pattern directly
|
||||
@@ -76,6 +86,8 @@ class TestAllReduceRMSNormModel(torch.nn.Module):
|
||||
return [torch.ops.vllm.all_reduce.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
if self.use_aiter:
|
||||
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
|
||||
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
|
||||
|
||||
|
||||
@@ -194,12 +206,36 @@ class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"test_model, enable_quant_fp8_custom_op",
|
||||
"test_model, enable_quant_fp8_custom_op, use_aiter",
|
||||
[
|
||||
(TestAllReduceRMSNormModel, False),
|
||||
(TestAllReduceRMSNormStaticQuantFP8Model, True),
|
||||
(TestAllReduceRMSNormStaticQuantFP8Model, False),
|
||||
(TestAllReduceFusedAddRMSNormStaticQuantFP4Model, False),
|
||||
(TestAllReduceRMSNormModel, False, IS_AITER_FOUND),
|
||||
pytest.param(
|
||||
TestAllReduceRMSNormStaticQuantFP8Model,
|
||||
True,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Not supported on ROCm platform",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
TestAllReduceRMSNormStaticQuantFP8Model,
|
||||
False,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Not supported on ROCm platform",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
TestAllReduceFusedAddRMSNormStaticQuantFP4Model,
|
||||
False,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Not supported on ROCm platform",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("batch_size", [8])
|
||||
@@ -210,9 +246,18 @@ class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
|
||||
@pytest.mark.parametrize("flashinfer_allreduce_backend", ["trtllm", "mnnvl"])
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
@pytest.mark.skipif(
|
||||
not find_spec("flashinfer")
|
||||
or not has_module_attribute("flashinfer.comm", "allreduce_fusion")
|
||||
or not has_module_attribute("flashinfer.comm", "create_allreduce_fusion_workspace"),
|
||||
current_platform.is_rocm() and not IS_AITER_FOUND,
|
||||
reason="aiter is not found",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cuda()
|
||||
and (
|
||||
not find_spec("flashinfer")
|
||||
or not has_module_attribute("flashinfer.comm", "allreduce_fusion")
|
||||
or not has_module_attribute(
|
||||
"flashinfer.comm", "create_allreduce_fusion_workspace"
|
||||
)
|
||||
),
|
||||
reason="flashinfer is not found or flashinfer "
|
||||
"is not compiled with allreduce_fusion",
|
||||
)
|
||||
@@ -225,7 +270,14 @@ def test_all_reduce_fusion_pass_replace(
|
||||
enable_rms_norm_custom_op,
|
||||
enable_quant_fp8_custom_op,
|
||||
flashinfer_allreduce_backend,
|
||||
use_aiter: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
if use_aiter:
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", str(use_aiter))
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
num_processes = 2
|
||||
if (
|
||||
test_model == TestAllReduceFusedAddRMSNormStaticQuantFP4Model
|
||||
@@ -249,6 +301,8 @@ def test_all_reduce_fusion_pass_replace(
|
||||
enable_rms_norm_custom_op,
|
||||
enable_quant_fp8_custom_op,
|
||||
flashinfer_allreduce_backend,
|
||||
use_aiter,
|
||||
monkeypatch,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
@@ -267,6 +321,8 @@ def all_reduce_fusion_pass_on_test_model(
|
||||
enable_rms_norm_custom_op,
|
||||
enable_quant_fp8_custom_op,
|
||||
flashinfer_allreduce_backend,
|
||||
use_aiter: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
@@ -313,7 +369,11 @@ def all_reduce_fusion_pass_on_test_model(
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
all_reduce_fusion_pass = AllReduceFusionPass(vllm_config)
|
||||
all_reduce_fusion_pass = (
|
||||
RocmAiterAllReduceFusionPass(vllm_config)
|
||||
if use_aiter
|
||||
else AllReduceFusionPass(vllm_config)
|
||||
)
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
func_pass = FixFunctionalizationPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
@@ -323,7 +383,12 @@ def all_reduce_fusion_pass_on_test_model(
|
||||
)
|
||||
|
||||
token_num = batch_size * seq_len
|
||||
model = test_model_cls(hidden_size, token_num, dtype=dtype)
|
||||
if test_model_cls is TestAllReduceRMSNormModel:
|
||||
model = test_model_cls(
|
||||
hidden_size, token_num, dtype=dtype, use_aiter=use_aiter
|
||||
)
|
||||
else:
|
||||
model = test_model_cls(hidden_size, token_num, dtype=dtype)
|
||||
|
||||
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user