[mla] Support fused FP8/NVFP4 output quantization in MLA attention (#35792) (#36205)

Signed-off-by: Carl You <4531192+carlyou@users.noreply.github.com>
Signed-off-by: Carl Y <4531192+carlyou@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Carl Y
2026-04-02 21:16:11 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent ee3cf45739
commit 1f5ec2889c
12 changed files with 928 additions and 17 deletions
+32 -1
View File
@@ -58,6 +58,15 @@ TRITON_MLA_ATTN = pytest.param(
id="TRITON_MLA",
)
FLASHMLA_SPARSE_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.FLASHMLA_SPARSE),
id="FLASHMLA_SPARSE",
marks=pytest.mark.skipif(
not is_blackwell(),
reason="FlashMLA Sparse requires Blackwell",
),
)
# Models
llama3_8b = ModelFusionInfo(
model_name="meta-llama/Llama-3.1-8B-Instruct",
@@ -141,6 +150,18 @@ qwen3_a3b_fp8 = ModelFusionInfo(
),
)
deepseek_coder_v2_lite_fp8 = ModelFusionInfo(
model_name="RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8",
matches=lambda n_layers: Matches(
# first_k_dense_replace=1; MoE hides most rms+quant sites
rms_quant_fusion=1,
act_quant_fusion=min(1, n_layers), # dense layers only
# MLA attn + static FP8 quant
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
),
)
deepseek_v3_fp8 = ModelFusionInfo(
model_name="deepseek-ai/DeepSeek-V3",
matches=lambda n_layers: Matches(
@@ -152,7 +173,7 @@ deepseek_v3_fp8 = ModelFusionInfo(
rms_quant_fusion=n_layers * 2 + min(3, n_layers), # add for 3 dense layers
# silu+block quant
act_quant_fusion=min(3, n_layers), # dense layers only
# MLA attn + quant not supported yet:
# MLA attn + per-group FP8 quant not supported yet:
# https://github.com/vllm-project/vllm/issues/35792
attn_quant_fusion=0,
ar_rms_fusion=n_layers * 2 + 1,
@@ -162,6 +183,16 @@ deepseek_v3_fp8 = ModelFusionInfo(
),
)
deepseek_v32_fp4 = ModelFusionInfo(
model_name="nvidia/DeepSeek-V3.2-NVFP4",
matches=lambda n_layers: Matches(
rms_quant_fusion=0,
act_quant_fusion=0,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
),
)
gpt_oss_20b = ModelFusionInfo(
model_name="openai/gpt-oss-20b",
matches=lambda n_layers: Matches(