[Bugfix] [Spec Decode] DeepSeek V4/V4.1 NVFP4: keep ignored MTP/DSpark experts on MXFP4 (#59655)

Signed-off-by: Shobhit Verma <shobhitv@nvidia.com>
Signed-off-by: v-shobhit <161510941+v-shobhit@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
v-shobhit
2026-10-08 00:10:32 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Benjamin Chislett
parent ed825aafc0
commit 8faac3dd2a
6 changed files with 125 additions and 2 deletions
+2
View File
@@ -322,9 +322,11 @@ steps:
optional: true
source_file_dependencies:
- tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
- tests/evals/gsm8k/configs/DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml
- tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml
- tests/evals/gsm8k/configs/models-spec-decode.txt
- vllm/models/deepseek_v4/
- vllm/models/deepseek_v41/
- vllm/models/kimi_k3/
- vllm/model_executor/models/qwen3_dspark.py
- vllm/v1/spec_decode/metrics.py
+1
View File
@@ -12,6 +12,7 @@ steps:
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- vllm/models/deepseek_v41/quant_config.py
- tests/quantization
commands:
# Pin torchao to the version compatible with the CI PyTorch and CUDA stack.
@@ -0,0 +1,15 @@
model_name: "nvidia/DeepSeek-V4.1-Flash-NVFP4"
accuracy_threshold: 0.88
tolerance: 0.03
# The bundled DSpark draft (mtp.*) stays MXFP4 in this checkpoint. Loading it
# as NVFP4 keeps accuracy but drops acceptance length to 1.0 (3.57 when correct).
min_acceptance_length: 2.5
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1800
server_args: >-
--tokenizer-mode deepseek_v41
--trust-remote-code
--max-model-len 8192
--tensor-parallel-size 4
--speculative-config '{"method":"dspark","num_speculative_tokens":5}'
@@ -1,2 +1,3 @@
DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
Kimi-K3-pruned75-DSpark-TP4.yaml
DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml
+76
View File
@@ -318,6 +318,82 @@ def test_deepseek_v41_declines_quark_configs():
)
_DSV41_NVFP4 = {
"moe_quant_algo": "NVFP4",
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head", "mtp.*"],
}
_DSV4_NVFP4_DSPARK = {
"moe_quant_algo": "NVFP4",
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head"],
}
@pytest.mark.parametrize(
("ckpt_quant_cfg", "prefix", "expected"),
[
pytest.param(_DSV41_NVFP4, "model.layers.1.ffn.experts", "nvfp4", id="target"),
pytest.param(
_DSV41_NVFP4,
"language_model.model.layers.0.ffn.experts",
"nvfp4",
id="target-vl-prefix",
),
pytest.param(
_DSV41_NVFP4, "model.layers.40.ffn.experts", "mxfp4", id="ignored-mtp0"
),
pytest.param(
_DSV41_NVFP4, "model.layers.42.ffn.experts", "mxfp4", id="ignored-mtp2"
),
pytest.param(
_DSV4_NVFP4_DSPARK,
"model.layers.41.ffn.experts",
"nvfp4",
id="quantized-mtp1",
),
pytest.param(
{"moe_quant_algo": "NVFP4"},
"model.layers.40.ffn.experts",
"nvfp4",
id="no-ignore-list",
),
pytest.param({}, "model.layers.1.ffn.experts", "mxfp4", id="not-nvfp4"),
],
)
def test_deepseek_v41_nvfp4_export_keeps_ignored_experts_mxfp4(
monkeypatch, ckpt_quant_cfg, prefix, expected
):
"""NVFP4 exports keep the routed experts they ``ignore`` in MXFP4, e.g. the
bundled DSpark draft (``mtp.*``, built as ``layers.{40 + k}``) of
DeepSeek-V4.1-Flash-NVFP4. The ignore list is read from the checkpoint
passed to ``from_config``: the current hf_config below carries none."""
import vllm.model_executor.layers.quantization.modelopt as modelopt
import vllm.models.deepseek_v41.quant_config as dsv41_qc
from vllm.model_executor.layers.fused_moe import RoutedExperts
hf_config = SimpleNamespace(
expert_dtype="fp4",
quantization_config={"moe_quant_algo": ckpt_quant_cfg.get("moe_quant_algo")},
)
model_config = SimpleNamespace(
hf_config=hf_config, hf_text_config=SimpleNamespace(num_hidden_layers=40)
)
monkeypatch.setattr(
dsv41_qc,
"get_current_vllm_config",
lambda: SimpleNamespace(model_config=model_config),
)
monkeypatch.setattr(dsv41_qc, "Mxfp4MoEMethod", lambda moe_config: "mxfp4")
monkeypatch.setattr(modelopt, "ModelOptNvFp4FusedMoE", lambda **kwargs: "nvfp4")
config = dsv41_qc.DeepseekV4FP8Config.from_config(
{"quant_method": "fp8", "activation_scheme": "dynamic", **ckpt_quant_cfg}
)
monkeypatch.setattr(config, "_get_nvfp4_config", lambda: None)
layer = RoutedExperts.__new__(RoutedExperts)
layer.moe_config = None
assert config.get_quant_method(layer, prefix) == expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA")
@pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu])
@pytest.mark.parametrize(
+30 -2
View File
@@ -4,8 +4,11 @@
from __future__ import annotations
import fnmatch
from typing import TYPE_CHECKING, cast
import regex as re
from vllm.config import get_current_vllm_config
from vllm.config.quantization import QuantSpec
from vllm.model_executor.layers.fused_moe import (
@@ -23,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
_EXPERTS_LAYER_RE = re.compile(r"(?:^|\.)layers\.(\d+)\.ffn\.experts$")
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.modelopt import (
@@ -54,6 +58,7 @@ class DeepseekV4FP8Config(Fp8Config):
super().__init__(*args, **kwargs)
self._resolved_expert_dtype: str | None = None
self._resolved_moe_quant_algo: str | None = None
self._modelopt_ignore: tuple[str, ...] = ()
self._nvfp4_config: ModelOptNvFp4Config | None = None
# ``is_scale_e8m0`` is a property that resolves on first read,
# by which time the current vllm_config has been set.
@@ -103,6 +108,21 @@ class DeepseekV4FP8Config(Fp8Config):
self._resolve_moe_overrides()
return self._resolved_moe_quant_algo or ""
def _is_nvfp4_expert_layer(self, prefix: str) -> bool:
if self.moe_quant_algo != "NVFP4":
return False
match = _EXPERTS_LAYER_RE.search(prefix)
if match is None or not self._modelopt_ignore:
return True
# MTP/draft layers are built as layers.{num_hidden_layers + k} but are
# named mtp.{k} in some checkpoints.
model_config = get_current_vllm_config().model_config
num_layers = model_config.hf_text_config.num_hidden_layers
idx = int(match.group(1))
layer = f"mtp.{idx - num_layers}" if idx >= num_layers else f"layers.{idx}"
name = f"{layer}.ffn.experts"
return not any(fnmatch.fnmatchcase(name, p) for p in self._modelopt_ignore)
def _get_nvfp4_config(self) -> ModelOptNvFp4Config:
if self._nvfp4_config is None:
from vllm.model_executor.layers.quantization.modelopt import (
@@ -173,6 +193,9 @@ class DeepseekV4FP8Config(Fp8Config):
@classmethod
def from_config(cls, config: dict) -> DeepseekV4FP8Config:
# ModelOpt NVFP4 exports keep the experts they ``ignore`` in the native
# MXFP4 format, e.g. the bundled MTP/DSpark layers (``mtp.*``).
ignore = config.get("ignore")
# Reroute AMD-Quark fused shared expert MXFP4 checkpoints onto the fp8
# path: the runtime layout matches the DeepSeek-native fp8 checkpoint,
# so translate the schema into format Fp8Config.from_config expects.
@@ -188,7 +211,12 @@ class DeepseekV4FP8Config(Fp8Config):
name for name in quark_exclude if isinstance(name, str)
],
}
return cast("DeepseekV4FP8Config", super().from_config(config))
quant_config = cast("DeepseekV4FP8Config", super().from_config(config))
if isinstance(ignore, list):
quant_config._modelopt_ignore = tuple(
p for p in ignore if isinstance(p, str)
)
return quant_config
def get_quant_method(self, layer, prefix):
if (
@@ -221,7 +249,7 @@ class DeepseekV4FP8Config(Fp8Config):
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if self.expert_dtype == "fp4":
if self.moe_quant_algo == "NVFP4":
if self._is_nvfp4_expert_layer(prefix):
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4FusedMoE,
)