[Model][NVIDIA] Route DSA models to the CUDA non-compiled path (#52861)

This commit is contained in:
Woosuk Kwon
2026-08-19 00:13:52 -07:00
committed by GitHub
parent ee11730751
commit b09bd69b5b
16 changed files with 260 additions and 146 deletions
+1 -20
View File
@@ -90,11 +90,8 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
backend_name = attn_backend.backend.name.lower()
requires_mla = "deepseek" in model_name.lower()
is_mla = "mla" in backend_name
# DeepSeek V3.2 uses sparse MLA
requires_sparse = "v3.2" in model_name.lower()
is_sparse = "sparse" in backend_name
if requires_mla != is_mla or requires_sparse != is_sparse:
if requires_mla != is_mla:
pytest.skip(
f"Incompatible model '{model_name}' and "
f"attention backend '{attn_backend.backend.name}'"
@@ -128,22 +125,6 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
# models (e.g. Llama-4-Scout-FP8) at 16384 tokens may trigger OOM.
model_kwargs.setdefault("max_num_batched_tokens", 8192)
# Sparse MLA models (DSv3.2) hit an over-strict inductor assertion in
# decompose_auto_functionalized when +rotary_embedding is forced into
# the compile graph. Disable qk_norm+rope fusion (which auto-enables
# +rotary_embedding) for this combo to avoid the known torch bug.
# TODO: remove once upstream torch fix lands.
if requires_sparse:
if "pass_config" in compilation_config:
compilation_config["pass_config"].enable_qk_norm_rope_fusion = False
matches_check = [m for m in matches_check if m != "norm_rope_fusion"]
# DSv3.2 sparse indexer uses persistent_topk with k=config.index_topk
# (2048 for the default config). max_model_len must be >= index_topk
# or the topk kernel raises "k out of range" at runtime.
model_kwargs["max_model_len"] = max(
model_kwargs.get("max_model_len", 0), 2048
)
# Always compile the full graph instead of piecewise
if not compilation_config["use_inductor_graph_partition"]:
compilation_config["splitting_ops"] = []
-24
View File
@@ -58,18 +58,6 @@ TRITON_MLA_ATTN = pytest.param(
id="TRITON_MLA",
)
FLASHMLA_SPARSE_ATTN = pytest.param(
AttentionBackendCase(
backend=AttentionBackendEnum.FLASHMLA_SPARSE,
model_kwargs=dict(kv_cache_dtype="fp8_ds_mla"),
),
id="FLASHMLA_SPARSE",
marks=pytest.mark.skipif(
not is_blackwell(),
reason="FlashMLA Sparse requires Blackwell",
),
)
# Models
llama3_8b = ModelFusionInfo(
model_name="meta-llama/Llama-3.1-8B-Instruct",
@@ -197,18 +185,6 @@ deepseek_r1_fp4 = ModelFusionInfo(
),
)
deepseek_v32_fp4 = ModelFusionInfo(
model_name="nvidia/DeepSeek-V3.2-NVFP4",
matches=lambda n_layers: Matches(
rms_quant_fusion=0,
# silu+quant on dense layers only; MoE hides the act+quant site
act_quant_fusion=min(3, n_layers),
# MLA attn + NVFP4 output quant fuses on sparse MLA output path
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
),
)
gpt_oss_20b = ModelFusionInfo(
model_name="openai/gpt-oss-20b",
matches=lambda n_layers: Matches(
+2 -4
View File
@@ -18,7 +18,6 @@ from .common import (
from .models import (
FLASHINFER_ATTN,
FLASHINFER_MLA_ATTN,
FLASHMLA_SPARSE_ATTN,
ROCM_AITER_UNIFIED_ATTN,
ROCM_ATTN,
TRITON_ATTN,
@@ -26,7 +25,6 @@ from .models import (
deepseek_coder_v2_lite_fp8,
deepseek_r1_fp4,
deepseek_v3_fp8,
deepseek_v32_fp4,
llama3_8b_fp4,
llama3_8b_fp8,
llama4_scout_fp4,
@@ -149,11 +147,11 @@ def test_tp1_fp8_fusions(
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4, deepseek_v32_fp4],
[llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4],
)
@pytest.mark.parametrize(
"attn_backend",
[FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, FLASHMLA_SPARSE_ATTN],
[FLASHINFER_ATTN, FLASHINFER_MLA_ATTN],
)
@pytest.mark.parametrize("n_layers", [6])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
+2 -4
View File
@@ -18,14 +18,12 @@ from .common import (
from .models import (
FLASHINFER_ATTN,
FLASHINFER_MLA_ATTN,
FLASHMLA_SPARSE_ATTN,
ROCM_AITER_UNIFIED_ATTN,
ROCM_ATTN,
TRITON_ATTN,
deepseek_coder_v2_lite_fp8,
deepseek_r1_fp4,
deepseek_v3_fp8,
deepseek_v32_fp4,
gpt_oss_20b,
llama3_8b,
llama3_8b_fp4,
@@ -120,11 +118,11 @@ def test_tp2_ar_rms_fp8_fusions(
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4, deepseek_v32_fp4],
[llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4],
)
@pytest.mark.parametrize(
"attn_backend",
[FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, FLASHMLA_SPARSE_ATTN],
[FLASHINFER_ATTN, FLASHINFER_MLA_ATTN],
)
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
-13
View File
@@ -132,19 +132,6 @@ MODEL_SPECS = [
),
id="gpt_oss_120b",
),
# NOTE: DeepSeek-V3.2 requires sparse MLA (index_topk) which needs
# Hopper+ GPUs. This test must run on H100 (see pytorch.yaml).
pytest.param(
ModelStartupSpec(
model="deepseek-ai/DeepSeek-V3.2",
hf_overrides=_SMALL_MOE_OVERRIDES,
cold_artifacts_saved=9,
# https://github.com/vllm-project/vllm/issues/38051
warm_artifacts_saved=0 if is_torch_equal_or_newer("2.12.0") else 9,
warm_artifacts_loaded=9 if is_torch_equal_or_newer("2.12.0") else 0,
),
id="deepseek_v3.2",
),
pytest.param(
ModelStartupSpec(
model="moonshotai/Kimi-K2.5",