Add NVFP4 MOE support for Deepseek V4. (#42209)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -150,9 +150,12 @@ steps:
|
||||
- csrc/quantization/fp4/
|
||||
- csrc/attention/mla/
|
||||
- csrc/quantization/cutlass_w8a8/moe/
|
||||
- vllm/model_executor/layers/fused_moe/cutlass_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/flashinfer_cutlass_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/flashinfer_a2a_prepare_finalize.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
|
||||
- vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_one_sided.py
|
||||
- vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_two_sided.py
|
||||
- vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
|
||||
- vllm/v1/attention/backends/flashinfer.py
|
||||
- vllm/v1/attention/backends/mla/cutlass_mla.py
|
||||
@@ -185,6 +188,7 @@ steps:
|
||||
- pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py
|
||||
- pytest -v -s tests/kernels/moe/test_flashinfer.py
|
||||
- pytest -v -s tests/kernels/moe/test_flashinfer_moe.py
|
||||
- pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py
|
||||
- pytest -v -s tests/kernels/moe/test_cutedsl_moe.py
|
||||
# e2e
|
||||
- pytest -v -s tests/models/quantization/test_nvfp4.py
|
||||
|
||||
@@ -22,6 +22,8 @@ from tests.kernels.quantization.nvfp4_utils import (
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.custom_op import CustomOp, op_registry
|
||||
from vllm.model_executor.layers.activation import SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
@@ -58,15 +60,45 @@ MNK_FACTORS = [
|
||||
(64, 704, 4096),
|
||||
]
|
||||
|
||||
_SWIGLU_LIMIT = 0.1
|
||||
_LARGE_OUTPUT1_SCALE = 32768.0
|
||||
_CLAMP_OP_NAME = "test_silu_and_mul_with_clamp"
|
||||
|
||||
# Test-only fixed-limit clamp. ``custom_op_name`` makes the class itself
|
||||
# valid as an ``activation=`` argument to ``torch_moe`` (which only looks
|
||||
# up ``activation.custom_op_name`` in ``op_registry``), so no
|
||||
# ``MoEActivation`` enum extension is needed.
|
||||
if _CLAMP_OP_NAME not in op_registry:
|
||||
|
||||
@CustomOp.register(_CLAMP_OP_NAME)
|
||||
class _SiluAndMulWithClampTest(SiluAndMulWithClamp):
|
||||
custom_op_name = _CLAMP_OP_NAME
|
||||
|
||||
def __init__(self, *, compile_native: bool = True) -> None:
|
||||
super().__init__(_SWIGLU_LIMIT, compile_native=compile_native)
|
||||
|
||||
|
||||
SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME]
|
||||
|
||||
|
||||
ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"),
|
||||
pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"),
|
||||
pytest.param(
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
None,
|
||||
id="relu2_no_mul",
|
||||
),
|
||||
pytest.param(MoEActivation.GELU, MoEActivation.GELU, None, id="gelu"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [128])
|
||||
@pytest.mark.parametrize("topk", [8])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[MoEActivation.SILU, MoEActivation.RELU2_NO_MUL, MoEActivation.GELU],
|
||||
)
|
||||
@pytest.mark.parametrize("activation,torch_activation,swiglu_limit", ACTIVATION_CASES)
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_fp4_moe_no_graph(
|
||||
m: int,
|
||||
@@ -76,6 +108,8 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
activation: MoEActivation,
|
||||
torch_activation: MoEActivation | type[SiluAndMulWithClamp],
|
||||
swiglu_limit: float | None,
|
||||
workspace_init,
|
||||
):
|
||||
# FlashInfer's trtllm_batched_gemm_runner has no precompiled tile
|
||||
@@ -113,6 +147,15 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
# Match what oracle/nvfp4.py does for this backend.
|
||||
is_scale_swizzled=False,
|
||||
)
|
||||
quant_config.gemm1_clamp_limit = swiglu_limit
|
||||
if swiglu_limit is not None:
|
||||
assert quant_config.g1_alphas is not None
|
||||
assert quant_config.a2_gscale is not None
|
||||
assert torch.all(quant_config.a2_gscale == 1)
|
||||
# With a2_gscale == 1, g1_alphas is the TRTLLM
|
||||
# output1_scale_gate_scalar. Make it large enough to catch
|
||||
# clamp/output-scale coupling in the FlashInfer kernel wrapper.
|
||||
quant_config.g1_alphas.fill_(_LARGE_OUTPUT1_SCALE)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
@@ -133,6 +176,23 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
trtllm_inner = TrtLlmNvFp4ExpertsModular(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
)
|
||||
# Mimic the production weight-loader path so per-expert tensors that
|
||||
# are normally precomputed in process_weights_after_loading (g1_scale_c
|
||||
# and the rescaled gemm1_clamp_limit) get materialized. The test's
|
||||
# synthetic quant_config has g1_alphas/g2_alphas already at their
|
||||
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
|
||||
# (same tensor) and use input_scale=1 to make the in-place
|
||||
# weight_scale_2 *= input_scale step a no-op.
|
||||
fake_layer = torch.nn.Module()
|
||||
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
|
||||
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
|
||||
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
|
||||
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
|
||||
trtllm_inner.process_weights_after_loading(fake_layer)
|
||||
|
||||
trtllm_experts = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
@@ -140,7 +200,7 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
TrtLlmNvFp4ExpertsModular(moe_config=moe_config, quant_config=quant_config),
|
||||
trtllm_inner,
|
||||
inplace=False,
|
||||
)
|
||||
|
||||
@@ -195,7 +255,7 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
)
|
||||
|
||||
torch_output = torch_moe(
|
||||
a_in_dtype, w1_d, w2_d, score, topk, activation=activation
|
||||
a_in_dtype, w1_d, w2_d, score, topk, activation=torch_activation
|
||||
)
|
||||
|
||||
torch.testing.assert_close(torch_output, trtllm_output, atol=2e-1, rtol=2e-1)
|
||||
@@ -203,5 +263,14 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_trtllm_fp4_moe_no_graph(
|
||||
64, 704, 4096, 128, 8, torch.bfloat16, MoEActivation.GELU, None
|
||||
64,
|
||||
704,
|
||||
4096,
|
||||
128,
|
||||
8,
|
||||
torch.bfloat16,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
@@ -805,6 +805,7 @@ def nvfp4_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
is_scale_swizzled: bool = True,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for mxfp4 activations and nvp4 weights.
|
||||
@@ -823,6 +824,7 @@ def nvfp4_moe_quant_config(
|
||||
per_out_ch_quant=False,
|
||||
block_shape=None,
|
||||
is_scale_swizzled=is_scale_swizzled,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
@@ -1271,6 +1273,11 @@ class FusedMoEConfig:
|
||||
is_act_and_mul: bool = True
|
||||
is_lora_enabled: bool = False
|
||||
|
||||
# SwiGLU clamp limit. When set, backends that do not implement the clamp
|
||||
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
|
||||
# cannot silently select one and drop the clamp.
|
||||
swiglu_limit: float | None = None
|
||||
|
||||
# This flag is used to disable the inplace optimization
|
||||
# in MoE kernels. If this flag is True then the kernel
|
||||
# should not be using inplace. If the flag is false, the
|
||||
|
||||
@@ -66,6 +66,17 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
else:
|
||||
self.g1_scale_c = self.quant_config.a2_gscale.clone()
|
||||
|
||||
if moe_config.is_act_and_mul and quant_config.gemm1_clamp_limit is not None:
|
||||
device = torch.accelerator.current_device_index()
|
||||
self.gemm1_clamp_limit = torch.full(
|
||||
(self.local_num_experts,),
|
||||
quant_config.gemm1_clamp_limit,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
self.gemm1_clamp_limit = None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
|
||||
layer.w2_weight_scale_2.data.mul_(layer.w2_input_scale)
|
||||
@@ -84,6 +95,20 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
)
|
||||
self.g1_scale_c = layer.g1_scale_c
|
||||
|
||||
# Pre-fold the per-expert g1_alphas (= output1_scale_gate_scalar)
|
||||
# division so the TRTLLM kernel receives the raw-GEMM-space clamp
|
||||
# directly. g1_alphas is set once here in process_weights_after_loading
|
||||
# (via the in-place mul above) and never changes again, so this is a
|
||||
# static, per-expert constant. Register on the layer so EPLB
|
||||
# rearranges it alongside the other expert tensors.
|
||||
if self.gemm1_clamp_limit is not None:
|
||||
gemm1_clamp_limit = self.gemm1_clamp_limit / self.quant_config.g1_alphas
|
||||
layer.register_parameter(
|
||||
"gemm1_clamp_limit",
|
||||
torch.nn.Parameter(gemm1_clamp_limit, requires_grad=False),
|
||||
)
|
||||
self.gemm1_clamp_limit = layer.gemm1_clamp_limit
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
"""Supports only Blackwell-family GPUs."""
|
||||
@@ -201,6 +226,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
|
||||
# Pack topk ids and weights into format expected by the kernel.
|
||||
packed_tensor = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
|
||||
output1_scale_gate_scalar = self.quant_config.g1_alphas
|
||||
|
||||
# Invoke kernel.
|
||||
flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
|
||||
@@ -215,12 +241,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
gemm1_bias=None,
|
||||
gemm1_alpha=None,
|
||||
gemm1_beta=None,
|
||||
gemm1_clamp_limit=None,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn),
|
||||
gemm2_bias=None,
|
||||
output1_scale_scalar=self.g1_scale_c,
|
||||
output1_scale_gate_scalar=self.quant_config.g1_alphas,
|
||||
output1_scale_gate_scalar=output1_scale_gate_scalar,
|
||||
output2_scale_scalar=self.quant_config.g2_alphas,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
@@ -313,6 +339,8 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
if e_score_correction_bias is not None:
|
||||
e_score_correction_bias = e_score_correction_bias.to(torch.bfloat16)
|
||||
|
||||
output1_scale_gate_scalar = self.quant_config.g1_alphas
|
||||
|
||||
# Invoke kernel.
|
||||
# NOTE: Activation padding and output
|
||||
# truncation are handled by the MoE runner's
|
||||
@@ -328,12 +356,12 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
gemm1_bias=None,
|
||||
gemm1_alpha=None,
|
||||
gemm1_beta=None,
|
||||
gemm1_clamp_limit=None,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.quant_config.w2_scale.view(torch.float8_e4m3fn),
|
||||
gemm2_bias=None,
|
||||
output1_scale_scalar=self.g1_scale_c,
|
||||
output1_scale_gate_scalar=self.quant_config.g1_alphas,
|
||||
output1_scale_gate_scalar=output1_scale_gate_scalar,
|
||||
output2_scale_scalar=self.quant_config.g2_alphas,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
|
||||
@@ -335,6 +335,7 @@ class FusedMoE(PluggableLayer):
|
||||
activation=self.activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=self.routing_method_type,
|
||||
swiglu_limit=swiglu_limit,
|
||||
# TODO: in_dtype == out_dtype?
|
||||
disable_inplace=disable_inplace() or shared_experts is not None,
|
||||
)
|
||||
|
||||
@@ -181,6 +181,15 @@ def select_nvfp4_moe_backend(
|
||||
NvFp4MoeBackend.EMULATION,
|
||||
]
|
||||
|
||||
NVFP4_BACKENDS_WITH_CLAMP = {
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
}
|
||||
|
||||
if config.swiglu_limit is not None:
|
||||
AVAILABLE_BACKENDS = [
|
||||
b for b in AVAILABLE_BACKENDS if b in NVFP4_BACKENDS_WITH_CLAMP
|
||||
]
|
||||
|
||||
use_batched = config.moe_parallel_config.use_batched_activation_format
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
@@ -234,6 +243,16 @@ def select_nvfp4_moe_backend(
|
||||
and requested_backend == NvFp4MoeBackend.FLASHINFER_CUTEDSL
|
||||
):
|
||||
requested_backend = NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED
|
||||
if (
|
||||
config.swiglu_limit is not None
|
||||
and requested_backend not in NVFP4_BACKENDS_WITH_CLAMP
|
||||
):
|
||||
raise ValueError(
|
||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
||||
f"explicitly requested moe_backend={runner_backend!r} does "
|
||||
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm' or "
|
||||
f"'flashinfer_cutlass' instead."
|
||||
)
|
||||
return _return_or_raise(
|
||||
requested_backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
@@ -242,17 +261,32 @@ def select_nvfp4_moe_backend(
|
||||
if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
|
||||
# If the user rejects FlashInfer remove those backends.
|
||||
for b in FLASHINFER_NVFP4_MOE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(b)
|
||||
if b in AVAILABLE_BACKENDS:
|
||||
AVAILABLE_BACKENDS.remove(b)
|
||||
|
||||
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
|
||||
# If user is explicit about backend, validate it.
|
||||
backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
|
||||
if (
|
||||
config.swiglu_limit is not None
|
||||
and backend not in NVFP4_BACKENDS_WITH_CLAMP
|
||||
):
|
||||
raise ValueError(
|
||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
||||
f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
|
||||
f"({backend.value}) does not apply the SwiGLU clamp."
|
||||
)
|
||||
return _return_or_raise(
|
||||
backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
else:
|
||||
# If the user is not explicit about the backend, try each.
|
||||
for backend in FLASHINFER_NVFP4_MOE_BACKENDS:
|
||||
fi_backends = [
|
||||
b
|
||||
for b in FLASHINFER_NVFP4_MOE_BACKENDS
|
||||
if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
|
||||
]
|
||||
for backend in fi_backends:
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls,
|
||||
@@ -439,6 +473,7 @@ def make_nvfp4_moe_quant_config(
|
||||
w2_scale_2: torch.Tensor,
|
||||
a13_scale: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
swiglu_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
if backend == NvFp4MoeBackend.MARLIN:
|
||||
return nvfp4_w4a16_moe_quant_config(
|
||||
@@ -455,6 +490,7 @@ def make_nvfp4_moe_quant_config(
|
||||
a2_gscale=a2_scale,
|
||||
w1_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
# Pass w13_scale_2 / w2_scale_2 directly as g1/g2_alphas.
|
||||
@@ -478,6 +514,7 @@ def make_nvfp4_moe_quant_config(
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
)
|
||||
),
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+1
@@ -258,6 +258,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
@@ -1601,6 +1601,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Quantization config for DeepSeek V4."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod
|
||||
@@ -14,6 +18,11 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
|
||||
_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4FP8Config(Fp8Config):
|
||||
"""FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch.
|
||||
@@ -38,6 +47,8 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._resolved_expert_dtype: str | None = None
|
||||
self._resolved_moe_quant_algo: str | None = None
|
||||
self._nvfp4_config: ModelOptNvFp4Config | None = None
|
||||
# ``is_scale_e8m0`` is a property that resolves on first read,
|
||||
# by which time the current vllm_config has been set.
|
||||
|
||||
@@ -70,6 +81,36 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
# checkpoints (Flash-Base) store them as float32.
|
||||
return self.expert_dtype == "fp4"
|
||||
|
||||
def _resolve_moe_overrides(self) -> None:
|
||||
if self._resolved_moe_quant_algo is not None:
|
||||
return
|
||||
try:
|
||||
hf_config = get_current_vllm_config().model_config.hf_config
|
||||
except Exception:
|
||||
return
|
||||
quant_cfg = getattr(hf_config, "quantization_config", None) or {}
|
||||
algo = (quant_cfg.get("moe_quant_algo") or "").upper() or None
|
||||
self._resolved_moe_quant_algo = algo or ""
|
||||
|
||||
@property
|
||||
def moe_quant_algo(self) -> str:
|
||||
self._resolve_moe_overrides()
|
||||
return self._resolved_moe_quant_algo or ""
|
||||
|
||||
def _get_nvfp4_config(self) -> ModelOptNvFp4Config:
|
||||
if self._nvfp4_config is None:
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
|
||||
self._nvfp4_config = ModelOptNvFp4Config(
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=[],
|
||||
group_size=16,
|
||||
)
|
||||
return self._nvfp4_config
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "deepseek_v4_fp8"
|
||||
@@ -97,10 +138,21 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if self.expert_dtype == "fp4":
|
||||
if self.moe_quant_algo == "NVFP4":
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4FusedMoE,
|
||||
)
|
||||
|
||||
return ModelOptNvFp4FusedMoE(
|
||||
quant_config=self._get_nvfp4_config(),
|
||||
moe_config=layer.moe_config,
|
||||
)
|
||||
return Mxfp4MoEMethod(layer.moe_config)
|
||||
# expert_dtype == "fp8": fall through to Fp8Config which
|
||||
# returns Fp8MoEMethod with block-wise float32 scales.
|
||||
return super().get_quant_method(layer, prefix)
|
||||
|
||||
def is_mxfp4_quant(self, prefix, layer):
|
||||
return isinstance(layer, FusedMoE) and self.expert_dtype == "fp4"
|
||||
if not isinstance(layer, FusedMoE) or self.expert_dtype != "fp4":
|
||||
return False
|
||||
return self.moe_quant_algo != "NVFP4"
|
||||
|
||||
Reference in New Issue
Block a user