[Bugfix] [Spec Decode] DeepSeek V4/V4.1 NVFP4: keep ignored MTP/DSpark experts on MXFP4 (#59655)
Signed-off-by: Shobhit Verma <shobhitv@nvidia.com> Signed-off-by: v-shobhit <161510941+v-shobhit@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Benjamin Chislett
parent
ed825aafc0
commit
8faac3dd2a
@@ -322,9 +322,11 @@ steps:
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
|
||||
- tests/evals/gsm8k/configs/DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml
|
||||
- tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml
|
||||
- tests/evals/gsm8k/configs/models-spec-decode.txt
|
||||
- vllm/models/deepseek_v4/
|
||||
- vllm/models/deepseek_v41/
|
||||
- vllm/models/kimi_k3/
|
||||
- vllm/model_executor/models/qwen3_dspark.py
|
||||
- vllm/v1/spec_decode/metrics.py
|
||||
|
||||
@@ -12,6 +12,7 @@ steps:
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- vllm/model_executor/layers/quantization
|
||||
- vllm/models/deepseek_v41/quant_config.py
|
||||
- tests/quantization
|
||||
commands:
|
||||
# Pin torchao to the version compatible with the CI PyTorch and CUDA stack.
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
model_name: "nvidia/DeepSeek-V4.1-Flash-NVFP4"
|
||||
accuracy_threshold: 0.88
|
||||
tolerance: 0.03
|
||||
# The bundled DSpark draft (mtp.*) stays MXFP4 in this checkpoint. Loading it
|
||||
# as NVFP4 keeps accuracy but drops acceptance length to 1.0 (3.57 when correct).
|
||||
min_acceptance_length: 2.5
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1800
|
||||
server_args: >-
|
||||
--tokenizer-mode deepseek_v41
|
||||
--trust-remote-code
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 4
|
||||
--speculative-config '{"method":"dspark","num_speculative_tokens":5}'
|
||||
@@ -1,2 +1,3 @@
|
||||
DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
|
||||
Kimi-K3-pruned75-DSpark-TP4.yaml
|
||||
DeepSeek-V4.1-Flash-NVFP4-DSpark-TP4.yaml
|
||||
|
||||
@@ -318,6 +318,82 @@ def test_deepseek_v41_declines_quark_configs():
|
||||
)
|
||||
|
||||
|
||||
_DSV41_NVFP4 = {
|
||||
"moe_quant_algo": "NVFP4",
|
||||
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head", "mtp.*"],
|
||||
}
|
||||
_DSV4_NVFP4_DSPARK = {
|
||||
"moe_quant_algo": "NVFP4",
|
||||
"ignore": ["*.attn.*", "*.ffn.shared_experts.*", "head"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ckpt_quant_cfg", "prefix", "expected"),
|
||||
[
|
||||
pytest.param(_DSV41_NVFP4, "model.layers.1.ffn.experts", "nvfp4", id="target"),
|
||||
pytest.param(
|
||||
_DSV41_NVFP4,
|
||||
"language_model.model.layers.0.ffn.experts",
|
||||
"nvfp4",
|
||||
id="target-vl-prefix",
|
||||
),
|
||||
pytest.param(
|
||||
_DSV41_NVFP4, "model.layers.40.ffn.experts", "mxfp4", id="ignored-mtp0"
|
||||
),
|
||||
pytest.param(
|
||||
_DSV41_NVFP4, "model.layers.42.ffn.experts", "mxfp4", id="ignored-mtp2"
|
||||
),
|
||||
pytest.param(
|
||||
_DSV4_NVFP4_DSPARK,
|
||||
"model.layers.41.ffn.experts",
|
||||
"nvfp4",
|
||||
id="quantized-mtp1",
|
||||
),
|
||||
pytest.param(
|
||||
{"moe_quant_algo": "NVFP4"},
|
||||
"model.layers.40.ffn.experts",
|
||||
"nvfp4",
|
||||
id="no-ignore-list",
|
||||
),
|
||||
pytest.param({}, "model.layers.1.ffn.experts", "mxfp4", id="not-nvfp4"),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v41_nvfp4_export_keeps_ignored_experts_mxfp4(
|
||||
monkeypatch, ckpt_quant_cfg, prefix, expected
|
||||
):
|
||||
"""NVFP4 exports keep the routed experts they ``ignore`` in MXFP4, e.g. the
|
||||
bundled DSpark draft (``mtp.*``, built as ``layers.{40 + k}``) of
|
||||
DeepSeek-V4.1-Flash-NVFP4. The ignore list is read from the checkpoint
|
||||
passed to ``from_config``: the current hf_config below carries none."""
|
||||
import vllm.model_executor.layers.quantization.modelopt as modelopt
|
||||
import vllm.models.deepseek_v41.quant_config as dsv41_qc
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
|
||||
hf_config = SimpleNamespace(
|
||||
expert_dtype="fp4",
|
||||
quantization_config={"moe_quant_algo": ckpt_quant_cfg.get("moe_quant_algo")},
|
||||
)
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=hf_config, hf_text_config=SimpleNamespace(num_hidden_layers=40)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dsv41_qc,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=model_config),
|
||||
)
|
||||
monkeypatch.setattr(dsv41_qc, "Mxfp4MoEMethod", lambda moe_config: "mxfp4")
|
||||
monkeypatch.setattr(modelopt, "ModelOptNvFp4FusedMoE", lambda **kwargs: "nvfp4")
|
||||
|
||||
config = dsv41_qc.DeepseekV4FP8Config.from_config(
|
||||
{"quant_method": "fp8", "activation_scheme": "dynamic", **ckpt_quant_cfg}
|
||||
)
|
||||
monkeypatch.setattr(config, "_get_nvfp4_config", lambda: None)
|
||||
layer = RoutedExperts.__new__(RoutedExperts)
|
||||
layer.moe_config = None
|
||||
assert config.get_quant_method(layer, prefix) == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA")
|
||||
@pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu])
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -4,8 +4,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.quantization import QuantSpec
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -23,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
|
||||
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
|
||||
_EXPERTS_LAYER_RE = re.compile(r"(?:^|\.)layers\.(\d+)\.ffn\.experts$")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
@@ -54,6 +58,7 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._resolved_expert_dtype: str | None = None
|
||||
self._resolved_moe_quant_algo: str | None = None
|
||||
self._modelopt_ignore: tuple[str, ...] = ()
|
||||
self._nvfp4_config: ModelOptNvFp4Config | None = None
|
||||
# ``is_scale_e8m0`` is a property that resolves on first read,
|
||||
# by which time the current vllm_config has been set.
|
||||
@@ -103,6 +108,21 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
self._resolve_moe_overrides()
|
||||
return self._resolved_moe_quant_algo or ""
|
||||
|
||||
def _is_nvfp4_expert_layer(self, prefix: str) -> bool:
|
||||
if self.moe_quant_algo != "NVFP4":
|
||||
return False
|
||||
match = _EXPERTS_LAYER_RE.search(prefix)
|
||||
if match is None or not self._modelopt_ignore:
|
||||
return True
|
||||
# MTP/draft layers are built as layers.{num_hidden_layers + k} but are
|
||||
# named mtp.{k} in some checkpoints.
|
||||
model_config = get_current_vllm_config().model_config
|
||||
num_layers = model_config.hf_text_config.num_hidden_layers
|
||||
idx = int(match.group(1))
|
||||
layer = f"mtp.{idx - num_layers}" if idx >= num_layers else f"layers.{idx}"
|
||||
name = f"{layer}.ffn.experts"
|
||||
return not any(fnmatch.fnmatchcase(name, p) for p in self._modelopt_ignore)
|
||||
|
||||
def _get_nvfp4_config(self) -> ModelOptNvFp4Config:
|
||||
if self._nvfp4_config is None:
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
@@ -173,6 +193,9 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict) -> DeepseekV4FP8Config:
|
||||
# ModelOpt NVFP4 exports keep the experts they ``ignore`` in the native
|
||||
# MXFP4 format, e.g. the bundled MTP/DSpark layers (``mtp.*``).
|
||||
ignore = config.get("ignore")
|
||||
# Reroute AMD-Quark fused shared expert MXFP4 checkpoints onto the fp8
|
||||
# path: the runtime layout matches the DeepSeek-native fp8 checkpoint,
|
||||
# so translate the schema into format Fp8Config.from_config expects.
|
||||
@@ -188,7 +211,12 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
name for name in quark_exclude if isinstance(name, str)
|
||||
],
|
||||
}
|
||||
return cast("DeepseekV4FP8Config", super().from_config(config))
|
||||
quant_config = cast("DeepseekV4FP8Config", super().from_config(config))
|
||||
if isinstance(ignore, list):
|
||||
quant_config._modelopt_ignore = tuple(
|
||||
p for p in ignore if isinstance(p, str)
|
||||
)
|
||||
return quant_config
|
||||
|
||||
def get_quant_method(self, layer, prefix):
|
||||
if (
|
||||
@@ -221,7 +249,7 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if self.expert_dtype == "fp4":
|
||||
if self.moe_quant_algo == "NVFP4":
|
||||
if self._is_nvfp4_expert_layer(prefix):
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4FusedMoE,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user