From 8faac3dd2af39fd631f018277387f01c89ad5782 Mon Sep 17 00:00:00 2001 From: v-shobhit <161510941+v-shobhit@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:10:32 -0700 Subject: [PATCH] [Bugfix] [Spec Decode] DeepSeek V4/V4.1 NVFP4: keep ignored MTP/DSpark experts on MXFP4 (#59655) Signed-off-by: Shobhit Verma Signed-off-by: v-shobhit <161510941+v-shobhit@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 Co-authored-by: Benjamin Chislett --- .buildkite/test_areas/lm_eval.yaml | 2 + .buildkite/test_areas/quantization.yaml | 1 + .../DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml | 15 ++++ .../gsm8k/configs/models-spec-decode.txt | 1 + tests/quantization/test_fp8.py | 76 +++++++++++++++++++ vllm/models/deepseek_v41/quant_config.py | 32 +++++++- 6 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index bcb1110848..d1a115a1e1 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -322,9 +322,11 @@ steps: optional: true source_file_dependencies: - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml + - tests/evals/gsm8k/configs/DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml - tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml - tests/evals/gsm8k/configs/models-spec-decode.txt - vllm/models/deepseek_v4/ + - vllm/models/deepseek_v41/ - vllm/models/kimi_k3/ - vllm/model_executor/models/qwen3_dspark.py - vllm/v1/spec_decode/metrics.py diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index e73054a4bc..b49b4b92c7 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -12,6 +12,7 @@ steps: source_file_dependencies: - csrc/ - vllm/model_executor/layers/quantization + - vllm/models/deepseek_v41/quant_config.py - tests/quantization commands: # Pin torchao to the version compatible with the CI PyTorch and CUDA stack. diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml new file mode 100644 index 0000000000..78d8995696 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml @@ -0,0 +1,15 @@ +model_name: "nvidia/DeepSeek-V4.1-Flash-NVFP4" +accuracy_threshold: 0.88 +tolerance: 0.03 +# The bundled DSpark draft (mtp.*) stays MXFP4 in this checkpoint. Loading it +# as NVFP4 keeps accuracy but drops acceptance length to 1.0 (3.57 when correct). +min_acceptance_length: 2.5 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1800 +server_args: >- + --tokenizer-mode deepseek_v41 + --trust-remote-code + --max-model-len 8192 + --tensor-parallel-size 4 + --speculative-config '{"method":"dspark","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/models-spec-decode.txt b/tests/evals/gsm8k/configs/models-spec-decode.txt index 427c35d912..96cead2585 100644 --- a/tests/evals/gsm8k/configs/models-spec-decode.txt +++ b/tests/evals/gsm8k/configs/models-spec-decode.txt @@ -1,2 +1,3 @@ DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml Kimi-K3-pruned75-DSpark-TP4.yaml +DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 1cb53f2239..64341843d9 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -318,6 +318,82 @@ def test_deepseek_v41_declines_quark_configs(): ) +_DSV41_NVFP4 = { + "moe_quant_algo": "NVFP4", + "ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head", "mtp.*"], +} +_DSV4_NVFP4_DSPARK = { + "moe_quant_algo": "NVFP4", + "ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head"], +} + + +@pytest.mark.parametrize( + ("ckpt_quant_cfg", "prefix", "expected"), + [ + pytest.param(_DSV41_NVFP4, "model.layers.1.ffn.experts", "nvfp4", id="target"), + pytest.param( + _DSV41_NVFP4, + "language_model.model.layers.0.ffn.experts", + "nvfp4", + id="target-vl-prefix", + ), + pytest.param( + _DSV41_NVFP4, "model.layers.40.ffn.experts", "mxfp4", id="ignored-mtp0" + ), + pytest.param( + _DSV41_NVFP4, "model.layers.42.ffn.experts", "mxfp4", id="ignored-mtp2" + ), + pytest.param( + _DSV4_NVFP4_DSPARK, + "model.layers.41.ffn.experts", + "nvfp4", + id="quantized-mtp1", + ), + pytest.param( + {"moe_quant_algo": "NVFP4"}, + "model.layers.40.ffn.experts", + "nvfp4", + id="no-ignore-list", + ), + pytest.param({}, "model.layers.1.ffn.experts", "mxfp4", id="not-nvfp4"), + ], +) +def test_deepseek_v41_nvfp4_export_keeps_ignored_experts_mxfp4( + monkeypatch, ckpt_quant_cfg, prefix, expected +): + """NVFP4 exports keep the routed experts they ``ignore`` in MXFP4, e.g. the + bundled DSpark draft (``mtp.*``, built as ``layers.{40 + k}``) of + DeepSeek-V4.1-Flash-NVFP4. The ignore list is read from the checkpoint + passed to ``from_config``: the current hf_config below carries none.""" + import vllm.model_executor.layers.quantization.modelopt as modelopt + import vllm.models.deepseek_v41.quant_config as dsv41_qc + from vllm.model_executor.layers.fused_moe import RoutedExperts + + hf_config = SimpleNamespace( + expert_dtype="fp4", + quantization_config={"moe_quant_algo": ckpt_quant_cfg.get("moe_quant_algo")}, + ) + model_config = SimpleNamespace( + hf_config=hf_config, hf_text_config=SimpleNamespace(num_hidden_layers=40) + ) + monkeypatch.setattr( + dsv41_qc, + "get_current_vllm_config", + lambda: SimpleNamespace(model_config=model_config), + ) + monkeypatch.setattr(dsv41_qc, "Mxfp4MoEMethod", lambda moe_config: "mxfp4") + monkeypatch.setattr(modelopt, "ModelOptNvFp4FusedMoE", lambda **kwargs: "nvfp4") + + config = dsv41_qc.DeepseekV4FP8Config.from_config( + {"quant_method": "fp8", "activation_scheme": "dynamic", **ckpt_quant_cfg} + ) + monkeypatch.setattr(config, "_get_nvfp4_config", lambda: None) + layer = RoutedExperts.__new__(RoutedExperts) + layer.moe_config = None + assert config.get_quant_method(layer, prefix) == expected + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA") @pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu]) @pytest.mark.parametrize( diff --git a/vllm/models/deepseek_v41/quant_config.py b/vllm/models/deepseek_v41/quant_config.py index 384e948879..1d92943db4 100644 --- a/vllm/models/deepseek_v41/quant_config.py +++ b/vllm/models/deepseek_v41/quant_config.py @@ -4,8 +4,11 @@ from __future__ import annotations +import fnmatch from typing import TYPE_CHECKING, cast +import regex as re + from vllm.config import get_current_vllm_config from vllm.config.quantization import QuantSpec from vllm.model_executor.layers.fused_moe import ( @@ -23,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) _DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8") +_EXPERTS_LAYER_RE = re.compile(r"(?:^|\.)layers\.(\d+)\.ffn\.experts$") if TYPE_CHECKING: from vllm.model_executor.layers.quantization.modelopt import ( @@ -54,6 +58,7 @@ class DeepseekV4FP8Config(Fp8Config): super().__init__(*args, **kwargs) self._resolved_expert_dtype: str | None = None self._resolved_moe_quant_algo: str | None = None + self._modelopt_ignore: tuple[str, ...] = () self._nvfp4_config: ModelOptNvFp4Config | None = None # ``is_scale_e8m0`` is a property that resolves on first read, # by which time the current vllm_config has been set. @@ -103,6 +108,21 @@ class DeepseekV4FP8Config(Fp8Config): self._resolve_moe_overrides() return self._resolved_moe_quant_algo or "" + def _is_nvfp4_expert_layer(self, prefix: str) -> bool: + if self.moe_quant_algo != "NVFP4": + return False + match = _EXPERTS_LAYER_RE.search(prefix) + if match is None or not self._modelopt_ignore: + return True + # MTP/draft layers are built as layers.{num_hidden_layers + k} but are + # named mtp.{k} in some checkpoints. + model_config = get_current_vllm_config().model_config + num_layers = model_config.hf_text_config.num_hidden_layers + idx = int(match.group(1)) + layer = f"mtp.{idx - num_layers}" if idx >= num_layers else f"layers.{idx}" + name = f"{layer}.ffn.experts" + return not any(fnmatch.fnmatchcase(name, p) for p in self._modelopt_ignore) + def _get_nvfp4_config(self) -> ModelOptNvFp4Config: if self._nvfp4_config is None: from vllm.model_executor.layers.quantization.modelopt import ( @@ -173,6 +193,9 @@ class DeepseekV4FP8Config(Fp8Config): @classmethod def from_config(cls, config: dict) -> DeepseekV4FP8Config: + # ModelOpt NVFP4 exports keep the experts they ``ignore`` in the native + # MXFP4 format, e.g. the bundled MTP/DSpark layers (``mtp.*``). + ignore = config.get("ignore") # Reroute AMD-Quark fused shared expert MXFP4 checkpoints onto the fp8 # path: the runtime layout matches the DeepSeek-native fp8 checkpoint, # so translate the schema into format Fp8Config.from_config expects. @@ -188,7 +211,12 @@ class DeepseekV4FP8Config(Fp8Config): name for name in quark_exclude if isinstance(name, str) ], } - return cast("DeepseekV4FP8Config", super().from_config(config)) + quant_config = cast("DeepseekV4FP8Config", super().from_config(config)) + if isinstance(ignore, list): + quant_config._modelopt_ignore = tuple( + p for p in ignore if isinstance(p, str) + ) + return quant_config def get_quant_method(self, layer, prefix): if ( @@ -221,7 +249,7 @@ class DeepseekV4FP8Config(Fp8Config): ): return UnquantizedFusedMoEMethod(layer.moe_config) if self.expert_dtype == "fp4": - if self.moe_quant_algo == "NVFP4": + if self._is_nvfp4_expert_layer(prefix): from vllm.model_executor.layers.quantization.modelopt import ( ModelOptNvFp4FusedMoE, )