[Bugfix] [Spec Decode] DeepSeek V4/V4.1 NVFP4: keep ignored MTP/DSpark experts on MXFP4 (#59655)

Signed-off-by: Shobhit Verma <shobhitv@nvidia.com>
Signed-off-by: v-shobhit <161510941+v-shobhit@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
v-shobhit
2026-10-08 00:10:32 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Benjamin Chislett
parent ed825aafc0
commit 8faac3dd2a
6 changed files with 125 additions and 2 deletions
@@ -0,0 +1,15 @@
model_name: "nvidia/DeepSeek-V4.1-Flash-NVFP4"
accuracy_threshold: 0.88
tolerance: 0.03
# The bundled DSpark draft (mtp.*) stays MXFP4 in this checkpoint. Loading it
# as NVFP4 keeps accuracy but drops acceptance length to 1.0 (3.57 when correct).
min_acceptance_length: 2.5
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1800
server_args: >-
--tokenizer-mode deepseek_v41
--trust-remote-code
--max-model-len 8192
--tensor-parallel-size 4
--speculative-config '{"method":"dspark","num_speculative_tokens":5}'
@@ -1,2 +1,3 @@
DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
Kimi-K3-pruned75-DSpark-TP4.yaml
DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml
+76
View File
@@ -318,6 +318,82 @@ def test_deepseek_v41_declines_quark_configs():
)
_DSV41_NVFP4 = {
"moe_quant_algo": "NVFP4",
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head", "mtp.*"],
}
_DSV4_NVFP4_DSPARK = {
"moe_quant_algo": "NVFP4",
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head"],
}
@pytest.mark.parametrize(
("ckpt_quant_cfg", "prefix", "expected"),
[
pytest.param(_DSV41_NVFP4, "model.layers.1.ffn.experts", "nvfp4", id="target"),
pytest.param(
_DSV41_NVFP4,
"language_model.model.layers.0.ffn.experts",
"nvfp4",
id="target-vl-prefix",
),
pytest.param(
_DSV41_NVFP4, "model.layers.40.ffn.experts", "mxfp4", id="ignored-mtp0"
),
pytest.param(
_DSV41_NVFP4, "model.layers.42.ffn.experts", "mxfp4", id="ignored-mtp2"
),
pytest.param(
_DSV4_NVFP4_DSPARK,
"model.layers.41.ffn.experts",
"nvfp4",
id="quantized-mtp1",
),
pytest.param(
{"moe_quant_algo": "NVFP4"},
"model.layers.40.ffn.experts",
"nvfp4",
id="no-ignore-list",
),
pytest.param({}, "model.layers.1.ffn.experts", "mxfp4", id="not-nvfp4"),
],
)
def test_deepseek_v41_nvfp4_export_keeps_ignored_experts_mxfp4(
monkeypatch, ckpt_quant_cfg, prefix, expected
):
"""NVFP4 exports keep the routed experts they ``ignore`` in MXFP4, e.g. the
bundled DSpark draft (``mtp.*``, built as ``layers.{40 + k}``) of
DeepSeek-V4.1-Flash-NVFP4. The ignore list is read from the checkpoint
passed to ``from_config``: the current hf_config below carries none."""
import vllm.model_executor.layers.quantization.modelopt as modelopt
import vllm.models.deepseek_v41.quant_config as dsv41_qc
from vllm.model_executor.layers.fused_moe import RoutedExperts
hf_config = SimpleNamespace(
expert_dtype="fp4",
quantization_config={"moe_quant_algo": ckpt_quant_cfg.get("moe_quant_algo")},
)
model_config = SimpleNamespace(
hf_config=hf_config, hf_text_config=SimpleNamespace(num_hidden_layers=40)
)
monkeypatch.setattr(
dsv41_qc,
"get_current_vllm_config",
lambda: SimpleNamespace(model_config=model_config),
)
monkeypatch.setattr(dsv41_qc, "Mxfp4MoEMethod", lambda moe_config: "mxfp4")
monkeypatch.setattr(modelopt, "ModelOptNvFp4FusedMoE", lambda **kwargs: "nvfp4")
config = dsv41_qc.DeepseekV4FP8Config.from_config(
{"quant_method": "fp8", "activation_scheme": "dynamic", **ckpt_quant_cfg}
)
monkeypatch.setattr(config, "_get_nvfp4_config", lambda: None)
layer = RoutedExperts.__new__(RoutedExperts)
layer.moe_config = None
assert config.get_quant_method(layer, prefix) == expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA")
@pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu])
@pytest.mark.parametrize(