Signed-off-by: Carl You <4531192+carlyou@users.noreply.github.com> Signed-off-by: Carl Y <4531192+carlyou@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
ee3cf45739
commit
1f5ec2889c
@@ -84,10 +84,14 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
# Filter here to reduce code duplication
|
||||
backend_name = attn_backend.backend.name.lower()
|
||||
requires_mla = "deepseek" in model_name.lower()
|
||||
is_mla = "mla" in attn_backend.backend.name.lower()
|
||||
is_mla = "mla" in backend_name
|
||||
# DeepSeek V3.2 uses sparse MLA
|
||||
requires_sparse = "v3.2" in model_name.lower()
|
||||
is_sparse = "sparse" in backend_name
|
||||
|
||||
if requires_mla != is_mla:
|
||||
if requires_mla != is_mla or requires_sparse != is_sparse:
|
||||
pytest.skip(
|
||||
f"Incompatible model '{model_name}' and "
|
||||
f"attention backend '{attn_backend.backend.name}'"
|
||||
@@ -231,7 +235,9 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
|
||||
)
|
||||
|
||||
elif match_name == "attn_quant_fusion":
|
||||
actual_match = match_table.get(match_name, 0)
|
||||
actual_match = match_table.get(
|
||||
"attn_quant_fusion", 0
|
||||
) + match_table.get("mla_attn_quant_fusion", 0)
|
||||
assert actual_match == expected_matches * n_expected, (
|
||||
f"Could not find {expected_matches * n_expected} "
|
||||
f"{match_name} (found {actual_match})."
|
||||
|
||||
@@ -58,6 +58,15 @@ TRITON_MLA_ATTN = pytest.param(
|
||||
id="TRITON_MLA",
|
||||
)
|
||||
|
||||
FLASHMLA_SPARSE_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.FLASHMLA_SPARSE),
|
||||
id="FLASHMLA_SPARSE",
|
||||
marks=pytest.mark.skipif(
|
||||
not is_blackwell(),
|
||||
reason="FlashMLA Sparse requires Blackwell",
|
||||
),
|
||||
)
|
||||
|
||||
# Models
|
||||
llama3_8b = ModelFusionInfo(
|
||||
model_name="meta-llama/Llama-3.1-8B-Instruct",
|
||||
@@ -141,6 +150,18 @@ qwen3_a3b_fp8 = ModelFusionInfo(
|
||||
),
|
||||
)
|
||||
|
||||
deepseek_coder_v2_lite_fp8 = ModelFusionInfo(
|
||||
model_name="RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8",
|
||||
matches=lambda n_layers: Matches(
|
||||
# first_k_dense_replace=1; MoE hides most rms+quant sites
|
||||
rms_quant_fusion=1,
|
||||
act_quant_fusion=min(1, n_layers), # dense layers only
|
||||
# MLA attn + static FP8 quant
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
deepseek_v3_fp8 = ModelFusionInfo(
|
||||
model_name="deepseek-ai/DeepSeek-V3",
|
||||
matches=lambda n_layers: Matches(
|
||||
@@ -152,7 +173,7 @@ deepseek_v3_fp8 = ModelFusionInfo(
|
||||
rms_quant_fusion=n_layers * 2 + min(3, n_layers), # add for 3 dense layers
|
||||
# silu+block quant
|
||||
act_quant_fusion=min(3, n_layers), # dense layers only
|
||||
# MLA attn + quant not supported yet:
|
||||
# MLA attn + per-group FP8 quant not supported yet:
|
||||
# https://github.com/vllm-project/vllm/issues/35792
|
||||
attn_quant_fusion=0,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
@@ -162,6 +183,16 @@ deepseek_v3_fp8 = ModelFusionInfo(
|
||||
),
|
||||
)
|
||||
|
||||
deepseek_v32_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/DeepSeek-V3.2-NVFP4",
|
||||
matches=lambda n_layers: Matches(
|
||||
rms_quant_fusion=0,
|
||||
act_quant_fusion=0,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
gpt_oss_20b = ModelFusionInfo(
|
||||
model_name="openai/gpt-oss-20b",
|
||||
matches=lambda n_layers: Matches(
|
||||
|
||||
@@ -18,11 +18,14 @@ from .common import (
|
||||
from .models import (
|
||||
FLASHINFER_ATTN,
|
||||
FLASHINFER_MLA_ATTN,
|
||||
FLASHMLA_SPARSE_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
ROCM_ATTN,
|
||||
TRITON_ATTN,
|
||||
TRITON_MLA_ATTN,
|
||||
deepseek_coder_v2_lite_fp8,
|
||||
deepseek_v3_fp8,
|
||||
deepseek_v32_fp4,
|
||||
llama3_8b_fp4,
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp4,
|
||||
@@ -37,6 +40,7 @@ from .models import (
|
||||
(*llama3_8b_fp8, False),
|
||||
(*qwen3_a3b_fp8, False),
|
||||
(*qwen3_a3b_fp8, True),
|
||||
(*deepseek_coder_v2_lite_fp8, False),
|
||||
(*deepseek_v3_fp8, False),
|
||||
(*deepseek_v3_fp8, True),
|
||||
pytest.param(
|
||||
@@ -144,9 +148,12 @@ def test_tp1_fp8_fusions(
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b_fp4, llama4_scout_fp4],
|
||||
[llama3_8b_fp4, llama4_scout_fp4, deepseek_v32_fp4],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[FLASHINFER_ATTN, FLASHMLA_SPARSE_ATTN],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [6])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
|
||||
@@ -18,8 +18,11 @@ from .common import (
|
||||
from .models import (
|
||||
FLASHINFER_ATTN,
|
||||
FLASHINFER_MLA_ATTN,
|
||||
FLASHMLA_SPARSE_ATTN,
|
||||
TRITON_ATTN,
|
||||
deepseek_coder_v2_lite_fp8,
|
||||
deepseek_v3_fp8,
|
||||
deepseek_v32_fp4,
|
||||
gpt_oss_20b,
|
||||
llama3_8b,
|
||||
llama3_8b_fp4,
|
||||
@@ -37,7 +40,13 @@ pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only tes
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet supported
|
||||
[llama3_8b_fp8, llama4_scout_fp8, qwen3_a3b_fp8, deepseek_v3_fp8],
|
||||
[
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp8,
|
||||
qwen3_a3b_fp8,
|
||||
deepseek_coder_v2_lite_fp8,
|
||||
deepseek_v3_fp8,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend", [TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN]
|
||||
@@ -104,9 +113,12 @@ def test_tp2_ar_rms_fp8_fusions(
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b_fp4, llama4_scout_fp4],
|
||||
[llama3_8b_fp4, llama4_scout_fp4, deepseek_v32_fp4],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[FLASHINFER_ATTN, FLASHMLA_SPARSE_ATTN],
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
|
||||
Reference in New Issue
Block a user