[ROCm] [CI] Add new fusion test cases that are relevant to vLLM IR Ops (#34307)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
@@ -13,6 +13,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
class Matches(NamedTuple):
|
||||
# simple pointwise
|
||||
aiter_rms_quant_fusion: int = 0
|
||||
rms_quant_fusion: int = 0
|
||||
act_quant_fusion: int = 0
|
||||
norm_rope_fusion: int = 0
|
||||
@@ -82,6 +83,9 @@ INDUCTOR_GRAPH_PARTITION = [
|
||||
]
|
||||
|
||||
FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
|
||||
"aiter_rms_quant_fusion": re.compile(
|
||||
r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
"norm_rope_fusion": re.compile(
|
||||
|
||||
@@ -63,9 +63,14 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
|
||||
compilation_config: dict,
|
||||
matches_check: list[str],
|
||||
use_deepgemm: bool = False,
|
||||
use_aiter: bool = False,
|
||||
tp_size: int = 1,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1" if use_deepgemm else "0")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
# Disable, compile cache to make sure custom passes run.
|
||||
# Otherwise, we can't verify fusion happened through the logs.
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@@ -24,6 +26,24 @@ TRITON_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.TRITON_ATTN), id="TRITON_ATTN"
|
||||
)
|
||||
|
||||
ROCM_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_ATTN),
|
||||
id="ROCM_ATTN",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm attention only for AMD",
|
||||
),
|
||||
)
|
||||
|
||||
ROCM_AITER_UNIFIED_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN),
|
||||
id="ROCM_AITER_UNIFIED_ATTN",
|
||||
marks=pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="ROCM_AITER_UNIFIED_ATTN only for AMD when AITER is installed",
|
||||
),
|
||||
)
|
||||
|
||||
# Models
|
||||
llama3_8b = ModelFusionInfo(
|
||||
model_name="meta-llama/Llama-3.1-8B-Instruct",
|
||||
@@ -49,7 +69,6 @@ llama3_8b_fp8 = ModelFusionInfo(
|
||||
llama3_8b_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/Llama-3.1-8B-Instruct-FP4",
|
||||
matches=lambda n_layers: Matches(
|
||||
rms_quant_fusion=0,
|
||||
act_quant_fusion=n_layers,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
@@ -79,7 +98,6 @@ llama4_scout_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4",
|
||||
hf_overrides=lambda n_layers: {"text_config": {"num_hidden_layers": n_layers}},
|
||||
matches=lambda n_layers: Matches(
|
||||
rms_quant_fusion=0,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2,
|
||||
sequence_parallel=n_layers * 2,
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Callable
|
||||
import pytest
|
||||
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import is_flashinfer_fp8_blockscale_gemm_supported
|
||||
|
||||
from .common import (
|
||||
@@ -16,6 +17,8 @@ from .common import (
|
||||
)
|
||||
from .models import (
|
||||
FLASHINFER_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
ROCM_ATTN,
|
||||
TRITON_ATTN,
|
||||
llama3_8b_fp4,
|
||||
llama3_8b_fp8,
|
||||
@@ -29,12 +32,33 @@ from .models import (
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides, use_deepgemm",
|
||||
[
|
||||
(*llama3_8b_fp8, False),
|
||||
(*llama4_scout_fp8, False),
|
||||
(*qwen3_a3b_fp8, False),
|
||||
(*qwen3_a3b_fp8, True),
|
||||
pytest.param(
|
||||
*llama4_scout_fp8,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Llama4 Scout FP8 only supported on CUDA",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
*qwen3_a3b_fp8,
|
||||
True,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="DeepGemm only supported on CUDA"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[
|
||||
TRITON_ATTN,
|
||||
FLASHINFER_ATTN,
|
||||
ROCM_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN, FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [6])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@@ -81,6 +105,8 @@ def test_tp1_fp8_fusions(
|
||||
),
|
||||
)
|
||||
|
||||
use_aiter = current_platform.is_rocm() and ("qwen" in model_name.lower())
|
||||
|
||||
matches_check = [
|
||||
"rms_quant_fusion",
|
||||
"act_quant_fusion",
|
||||
@@ -88,6 +114,15 @@ def test_tp1_fp8_fusions(
|
||||
"attn_quant_fusion",
|
||||
]
|
||||
|
||||
if use_aiter:
|
||||
matches_check[0] = "aiter_rms_quant_fusion"
|
||||
|
||||
matches = matches._replace(aiter_rms_quant_fusion=matches.rms_quant_fusion)
|
||||
# TODO: enable the `norm_rope_fusion` test,
|
||||
# On ROCm norm_rope_fusion is only supported without
|
||||
# enabling AITER.
|
||||
matches_check.remove("norm_rope_fusion")
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
@@ -96,6 +131,7 @@ def test_tp1_fp8_fusions(
|
||||
compilation_config,
|
||||
matches_check,
|
||||
use_deepgemm=use_deepgemm,
|
||||
use_aiter=use_aiter,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Callable
|
||||
import pytest
|
||||
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import multi_gpu_test
|
||||
from .common import (
|
||||
@@ -26,6 +27,8 @@ from .models import (
|
||||
qwen3_a3b_fp8,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Callable
|
||||
import pytest
|
||||
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import multi_gpu_test
|
||||
from .common import (
|
||||
@@ -23,6 +24,8 @@ from .models import (
|
||||
qwen3_a3b,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -36,6 +36,8 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
|
||||
@@ -182,8 +182,24 @@ TEST_KERNELS = ROCM_KERNELS if current_platform.is_rocm() else CUDA_KERNELS
|
||||
"model_class, enable_quant_fp8_custom_op, force_kernel",
|
||||
list(itertools.product([TestSiluMulFp8QuantModel], [True, False], TEST_KERNELS))
|
||||
+ [
|
||||
(TestSiluMulNvfp4QuantModel, False, None),
|
||||
(TestSiluMulGroupFp8QuantModel, False, None),
|
||||
pytest.param(
|
||||
TestSiluMulNvfp4QuantModel,
|
||||
False,
|
||||
None,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA only"
|
||||
),
|
||||
),
|
||||
# GroupFP8Quant fusion only works with AITER on ROCm.
|
||||
# and the enable_quant_fp8_custom_op must be True.
|
||||
pytest.param(
|
||||
TestSiluMulGroupFp8QuantModel,
|
||||
True,
|
||||
None,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm only"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
@@ -201,6 +217,7 @@ def test_fusion_silu_and_mul_quant(
|
||||
enable_silu_mul_custom_op: bool,
|
||||
enable_quant_fp8_custom_op: bool,
|
||||
force_kernel: FP8ScaledMMLinearKernel | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
if model_class is TestSiluMulNvfp4QuantModel and not is_nvfp4_supported():
|
||||
pytest.skip("NVFP4 is not supported on this GPU.")
|
||||
@@ -227,13 +244,16 @@ def test_fusion_silu_and_mul_quant(
|
||||
),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
with set_current_vllm_config(config), monkeypatch.context() as m:
|
||||
fusion_passes = [ActivationQuantFusionPass(config)]
|
||||
if IS_AITER_FOUND:
|
||||
if IS_AITER_FOUND and model_class is TestSiluMulGroupFp8QuantModel:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
RocmAiterSiluMulFp8GroupQuantFusionPass,
|
||||
)
|
||||
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
fusion_passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
|
||||
|
||||
passes = [NoOpEliminationPass(config), *fusion_passes, PostCleanupPass(config)]
|
||||
|
||||
Reference in New Issue
Block a user