Add NVFP4 MOE support for Deepseek V4. (#42209)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-05-22 07:21:51 -07:00
committed by GitHub
parent a377631d21
commit fb21d8b4f9
9 changed files with 217 additions and 17 deletions
+53 -1
View File
@@ -2,6 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Quantization config for DeepSeek V4."""
from __future__ import annotations
from typing import TYPE_CHECKING
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod
@@ -14,6 +18,11 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
class DeepseekV4FP8Config(Fp8Config):
"""FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch.
@@ -38,6 +47,8 @@ class DeepseekV4FP8Config(Fp8Config):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._resolved_expert_dtype: str | None = None
self._resolved_moe_quant_algo: str | None = None
self._nvfp4_config: ModelOptNvFp4Config | None = None
# ``is_scale_e8m0`` is a property that resolves on first read,
# by which time the current vllm_config has been set.
@@ -70,6 +81,36 @@ class DeepseekV4FP8Config(Fp8Config):
# checkpoints (Flash-Base) store them as float32.
return self.expert_dtype == "fp4"
def _resolve_moe_overrides(self) -> None:
if self._resolved_moe_quant_algo is not None:
return
try:
hf_config = get_current_vllm_config().model_config.hf_config
except Exception:
return
quant_cfg = getattr(hf_config, "quantization_config", None) or {}
algo = (quant_cfg.get("moe_quant_algo") or "").upper() or None
self._resolved_moe_quant_algo = algo or ""
@property
def moe_quant_algo(self) -> str:
self._resolve_moe_overrides()
return self._resolved_moe_quant_algo or ""
def _get_nvfp4_config(self) -> ModelOptNvFp4Config:
if self._nvfp4_config is None:
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
)
self._nvfp4_config = ModelOptNvFp4Config(
is_checkpoint_nvfp4_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=[],
group_size=16,
)
return self._nvfp4_config
@classmethod
def get_name(cls) -> QuantizationMethods:
return "deepseek_v4_fp8"
@@ -97,10 +138,21 @@ class DeepseekV4FP8Config(Fp8Config):
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if self.expert_dtype == "fp4":
if self.moe_quant_algo == "NVFP4":
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4FusedMoE,
)
return ModelOptNvFp4FusedMoE(
quant_config=self._get_nvfp4_config(),
moe_config=layer.moe_config,
)
return Mxfp4MoEMethod(layer.moe_config)
# expert_dtype == "fp8": fall through to Fp8Config which
# returns Fp8MoEMethod with block-wise float32 scales.
return super().get_quant_method(layer, prefix)
def is_mxfp4_quant(self, prefix, layer):
return isinstance(layer, FusedMoE) and self.expert_dtype == "fp4"
if not isinstance(layer, FusedMoE) or self.expert_dtype != "fp4":
return False
return self.moe_quant_algo != "NVFP4"