From 287471b99442b44c5a16c4d70b0f3e178dd52732 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Mon, 18 May 2026 19:50:02 -0700 Subject: [PATCH] [Model Refactoring] Migrate DeepSeek V4 to vllm/models/ [1/N] (#43004) Signed-off-by: Woosuk Kwon --- tests/models/test_deepseek_v4_mega_moe.py | 2 +- .../layers/quantization/__init__.py | 2 +- vllm/model_executor/models/registry.py | 31 ++++- vllm/models/__init__.py | 2 + vllm/models/deepseek_v4/__init__.py | 30 +++++ vllm/models/deepseek_v4/amd/__init__.py | 2 + vllm/models/deepseek_v4/amd/deepseek_v4.py | 1 + .../models/deepseek_v4/amd/deepseek_v4_mtp.py | 1 + vllm/models/deepseek_v4/nvidia/__init__.py | 2 + .../deepseek_v4/nvidia}/deepseek_v4.py | 125 ++---------------- .../deepseek_v4/nvidia}/deepseek_v4_mtp.py | 11 +- vllm/models/deepseek_v4/quant_config.py | 106 +++++++++++++++ 12 files changed, 189 insertions(+), 126 deletions(-) create mode 100644 vllm/models/__init__.py create mode 100644 vllm/models/deepseek_v4/__init__.py create mode 100644 vllm/models/deepseek_v4/amd/__init__.py create mode 120000 vllm/models/deepseek_v4/amd/deepseek_v4.py create mode 120000 vllm/models/deepseek_v4/amd/deepseek_v4_mtp.py create mode 100644 vllm/models/deepseek_v4/nvidia/__init__.py rename vllm/{model_executor/models => models/deepseek_v4/nvidia}/deepseek_v4.py (92%) rename vllm/{model_executor/models => models/deepseek_v4/nvidia}/deepseek_v4_mtp.py (98%) create mode 100644 vllm/models/deepseek_v4/quant_config.py diff --git a/tests/models/test_deepseek_v4_mega_moe.py b/tests/models/test_deepseek_v4_mega_moe.py index 304f044868..24a8701e70 100644 --- a/tests/models/test_deepseek_v4_mega_moe.py +++ b/tests/models/test_deepseek_v4_mega_moe.py @@ -6,7 +6,7 @@ from types import SimpleNamespace import pytest import torch -from vllm.model_executor.models.deepseek_v4 import ( +from vllm.models.deepseek_v4.nvidia.deepseek_v4 import ( DeepseekV4MegaMoEExperts, _stage_deepseek_v4_mega_moe_inputs, make_deepseek_v4_expert_params_mapping, diff --git a/vllm/model_executor/layers/quantization/__init__.py b/vllm/model_executor/layers/quantization/__init__.py index fa583c3a45..11489feb9d 100644 --- a/vllm/model_executor/layers/quantization/__init__.py +++ b/vllm/model_executor/layers/quantization/__init__.py @@ -113,7 +113,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: # lazy import to avoid triggering `torch.compile` too early from vllm.config.quantization import _ONLINE_SHORTHANDS from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig - from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config + from vllm.models.deepseek_v4 import DeepseekV4FP8Config from .auto_gptq import AutoGPTQConfig from .awq import AWQConfig diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index d3d009ff81..b6773503cb 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -6,6 +6,7 @@ Whenever you add an architecture to this page, please also update """ import importlib +import importlib.util import json import os import pickle @@ -97,7 +98,7 @@ _TEXT_GENERATION_MODELS = { "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"), "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"), "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"), - "DeepseekV4ForCausalLM": ("deepseek_v4", "DeepseekV4ForCausalLM"), + "DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"), "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"), "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"), "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"), @@ -611,7 +612,7 @@ _SPECULATIVE_DECODING_MODELS = { "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"), "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"), "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"), - "DeepSeekV4MTPModel": ("deepseek_v4_mtp", "DeepSeekV4MTP"), + "DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"), "Gemma4MTPModel": ("gemma4_mtp", "Gemma4MTP"), "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"), "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"), @@ -870,10 +871,21 @@ class _LazyRegisteredModel(_BaseRegisteredModel): @logtime(logger=logger, msg="Registry inspect model class") def inspect_model_cls(self) -> _ModelInfo: - model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py" + # Modules registered with a non-default location (e.g. the + # hardware-isolated ``vllm.models.`` layout) live outside + # ``vllm/model_executor/models``. Resolve the module spec directly + # so the file-hash cache stays warm for them. + if self.module_name.startswith("vllm.model_executor.models."): + model_path = Path(__file__).parent / f"{self.module_name.split('.')[-1]}.py" + else: + try: + spec = importlib.util.find_spec(self.module_name) + except (ImportError, ValueError): + spec = None + model_path = Path(spec.origin) if spec is not None and spec.origin else None module_hash = None - if model_path.exists(): + if model_path is not None and model_path.exists(): with open(model_path, "rb") as f: module_hash = safe_hash(f.read(), usedforsecurity=False).hexdigest() @@ -1328,10 +1340,19 @@ class _ModelRegistry: return model_cls.supports_transcription_only +def _resolve_module_name(mod_relname: str) -> str: + # Allow registry entries to point at fully-qualified module paths (e.g. + # ``vllm.models.deepseek_v4``) for models that live outside the legacy + # ``vllm.model_executor.models`` flat layout. + if mod_relname.startswith("vllm."): + return mod_relname + return f"vllm.model_executor.models.{mod_relname}" + + ModelRegistry = _ModelRegistry( { model_arch: _LazyRegisteredModel( - module_name=f"vllm.model_executor.models.{mod_relname}", + module_name=_resolve_module_name(mod_relname), class_name=cls_name, ) for model_arch, (mod_relname, cls_name) in _VLLM_MODELS.items() diff --git a/vllm/models/__init__.py b/vllm/models/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/deepseek_v4/__init__.py b/vllm/models/deepseek_v4/__init__.py new file mode 100644 index 0000000000..a8c0fc2224 --- /dev/null +++ b/vllm/models/deepseek_v4/__init__.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DeepSeek V4 model — hardware-isolated entry point. + +The actual implementation lives under ``nvidia/`` and ``amd/``; this module +picks the right one for the current platform and re-exports the public +classes used by the model registry and quantization config lookup. +""" + +from typing import TYPE_CHECKING + +from vllm.platforms import current_platform + +from .quant_config import DeepseekV4FP8Config + +# Pick the per-platform implementation. The NVIDIA branch is the static +# default that mypy sees; the ROCm branch overrides it at runtime and is +# kept type-compatible via ``# type: ignore[assignment]``. +if TYPE_CHECKING or not current_platform.is_rocm(): + from .nvidia.deepseek_v4 import DeepseekV4ForCausalLM + from .nvidia.deepseek_v4_mtp import DeepSeekV4MTP +else: + from .amd.deepseek_v4 import DeepseekV4ForCausalLM # type: ignore[assignment] + from .amd.deepseek_v4_mtp import DeepSeekV4MTP # type: ignore[assignment] + +__all__ = [ + "DeepSeekV4MTP", + "DeepseekV4FP8Config", + "DeepseekV4ForCausalLM", +] diff --git a/vllm/models/deepseek_v4/amd/__init__.py b/vllm/models/deepseek_v4/amd/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/deepseek_v4/amd/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/deepseek_v4/amd/deepseek_v4.py b/vllm/models/deepseek_v4/amd/deepseek_v4.py new file mode 120000 index 0000000000..b16a47b66d --- /dev/null +++ b/vllm/models/deepseek_v4/amd/deepseek_v4.py @@ -0,0 +1 @@ +../nvidia/deepseek_v4.py \ No newline at end of file diff --git a/vllm/models/deepseek_v4/amd/deepseek_v4_mtp.py b/vllm/models/deepseek_v4/amd/deepseek_v4_mtp.py new file mode 120000 index 0000000000..db212511d8 --- /dev/null +++ b/vllm/models/deepseek_v4/amd/deepseek_v4_mtp.py @@ -0,0 +1 @@ +../nvidia/deepseek_v4_mtp.py \ No newline at end of file diff --git a/vllm/models/deepseek_v4/nvidia/__init__.py b/vllm/models/deepseek_v4/nvidia/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/deepseek_v4/nvidia/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/model_executor/models/deepseek_v4.py b/vllm/models/deepseek_v4/nvidia/deepseek_v4.py similarity index 92% rename from vllm/model_executor/models/deepseek_v4.py rename to vllm/models/deepseek_v4/nvidia/deepseek_v4.py index 05878eeacb..b08900df9e 100644 --- a/vllm/model_executor/models/deepseek_v4.py +++ b/vllm/models/deepseek_v4/nvidia/deepseek_v4.py @@ -9,7 +9,7 @@ import torch import torch.nn as nn from vllm.compilation.decorators import support_torch_compile -from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config import VllmConfig from vllm.distributed import ( get_ep_group, get_pp_group, @@ -24,7 +24,6 @@ from vllm.model_executor.layers.deepseek_v4_attention import ( DeepseekV4MultiHeadLatentAttentionWrapper, ) from vllm.model_executor.layers.fused_moe import FusedMoE -from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( fused_topk_bias, ) @@ -44,15 +43,7 @@ from vllm.model_executor.layers.mhc import ( MHCPostOp, MHCPreOp, ) -from vllm.model_executor.layers.quantization import ( - QuantizationConfig, - QuantizationMethods, -) -from vllm.model_executor.layers.quantization.fp8 import Fp8Config -from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - is_layer_skipped, -) +from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -60,13 +51,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import SupportsPP -from vllm.model_executor.utils import set_weight_attrs -from vllm.platforms import current_platform -from vllm.sequence import IntermediateTensors -from vllm.triton_utils import tl, triton -from vllm.utils.torch_utils import direct_register_custom_op - -from .utils import ( +from vllm.model_executor.models.utils import ( AutoWeightsLoader, PPMissingLayer, WeightsMapper, @@ -75,8 +60,11 @@ from .utils import ( make_layers, maybe_prefix, ) - -_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8") +from vllm.model_executor.utils import set_weight_attrs +from vllm.platforms import current_platform +from vllm.sequence import IntermediateTensors +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op class DeepseekV4MLP(nn.Module): @@ -130,97 +118,6 @@ class DeepseekV4MLP(nn.Module): return x -class DeepseekV4FP8Config(Fp8Config): - """FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch. - - DeepSeek V4 checkpoints always use FP8 block quantization for - linear/attention layers. The MoE expert weights vary by checkpoint: - - ``expert_dtype="fp4"`` (e.g. DeepSeek-V4-Flash): MXFP4 experts - with ue8m0 (e8m0fnu) FP8 linear scales. - - ``expert_dtype="fp8"`` (e.g. DeepSeek-V4-Flash-Base): FP8 block - experts with float32 FP8 linear scales. - - The dispatch and the linear scale dtype are both keyed off - ``expert_dtype`` from the model's hf_config; missing values default - to ``"fp4"`` so existing FP4 checkpoints stay unchanged. - - NOTE: ``expert_dtype`` is resolved lazily because this config is - constructed during VllmConfig setup, before ``set_current_vllm_config`` - is active. Reading hf_config eagerly in ``__init__`` would always see - the default ``"fp4"`` and silently misroute Flash-Base checkpoints. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._resolved_expert_dtype: str | None = None - # ``is_scale_e8m0`` is a property that resolves on first read, - # by which time the current vllm_config has been set. - - @property - def expert_dtype(self) -> str: - if self._resolved_expert_dtype is None: - try: - hf_config = get_current_vllm_config().model_config.hf_config - except Exception: - # vllm_config not yet set; defer the decision until a - # later call lands inside set_current_vllm_config. - return "fp4" - expert_dtype = getattr(hf_config, "expert_dtype", "fp4") - if expert_dtype not in _DEEPSEEK_V4_EXPERT_DTYPES: - raise ValueError( - f"Unsupported DeepSeek V4 expert_dtype={expert_dtype!r}; " - f"expected one of {_DEEPSEEK_V4_EXPERT_DTYPES}." - ) - self._resolved_expert_dtype = expert_dtype - from vllm.logger import init_logger - - init_logger(__name__).info_once( - "DeepSeek V4 expert_dtype resolved to %r", expert_dtype - ) - return self._resolved_expert_dtype - - @property - def is_scale_e8m0(self) -> bool: - # FP4 checkpoints store FP8 linear scales as e8m0fnu; FP8 expert - # checkpoints (Flash-Base) store them as float32. - return self.expert_dtype == "fp4" - - @classmethod - def get_name(cls) -> QuantizationMethods: - return "deepseek_v4_fp8" - - @classmethod - def override_quantization_method( - cls, hf_quant_cfg, user_quant, hf_config=None - ) -> QuantizationMethods | None: - if not ( - isinstance(hf_quant_cfg, dict) - and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") - ): - return None - model_type = getattr(hf_config, "model_type", None) - if model_type == "deepseek_v4" or user_quant == "deepseek_v4_fp8": - return "deepseek_v4_fp8" - return None - - def get_quant_method(self, layer, prefix): - if isinstance(layer, FusedMoE): - if is_layer_skipped( - prefix=prefix, - ignored_layers=self.ignored_layers, - fused_mapping=self.packed_modules_mapping, - ): - return UnquantizedFusedMoEMethod(layer.moe_config) - if self.expert_dtype == "fp4": - return Mxfp4MoEMethod(layer.moe_config) - # expert_dtype == "fp8": fall through to Fp8Config which - # returns Fp8MoEMethod with block-wise float32 scales. - return super().get_quant_method(layer, prefix) - - def is_mxfp4_quant(self, prefix, layer): - return isinstance(layer, FusedMoE) and self.expert_dtype == "fp4" - - @triton.jit def _deepseek_v4_stage_mega_moe_inputs_kernel( hidden_states, @@ -1535,7 +1432,7 @@ class DeepseekV4Model(nn.Module): ): loaded_weight = loaded_weight.view(torch.uint8) for mapping in expert_mapping: - param_name, weight_name, expert_id, shard_id = mapping + param_name, weight_name, expert_id, expert_shard_id = mapping if weight_name not in name: continue name_mapped = name.replace(weight_name, param_name) @@ -1552,7 +1449,7 @@ class DeepseekV4Model(nn.Module): param, loaded_weight, name_mapped, - shard_id=shard_id, + shard_id=expert_shard_id, expert_id=expert_id, return_success=True, ) @@ -1673,7 +1570,7 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP): else: self.lm_head = PPMissingLayer() self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( + self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] self.model.make_empty_intermediate_tensors ) diff --git a/vllm/model_executor/models/deepseek_v4_mtp.py b/vllm/models/deepseek_v4/nvidia/deepseek_v4_mtp.py similarity index 98% rename from vllm/model_executor/models/deepseek_v4_mtp.py rename to vllm/models/deepseek_v4/nvidia/deepseek_v4_mtp.py index ad93fd5882..41060243eb 100644 --- a/vllm/model_executor/models/deepseek_v4_mtp.py +++ b/vllm/models/deepseek_v4/nvidia/deepseek_v4_mtp.py @@ -34,16 +34,16 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.deepseek_mtp import SharedHead +from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name +from vllm.model_executor.models.utils import maybe_prefix from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from .deepseek_mtp import SharedHead -from .deepseek_v2 import get_spec_layer_idx_from_weight_name from .deepseek_v4 import ( DeepseekV4DecoderLayer, make_deepseek_v4_expert_params_mapping, ) -from .utils import maybe_prefix logger = init_logger(__name__) @@ -68,6 +68,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): ) -> None: super().__init__() + assert vllm_config.speculative_config is not None config = vllm_config.speculative_config.draft_model_config.hf_config self.config = config quant_config = vllm_config.quant_config @@ -407,7 +408,7 @@ class DeepSeekV4MTP(nn.Module): ): loaded_weight = loaded_weight.view(torch.uint8) for mapping in expert_mapping: - param_name, weight_name, expert_id, shard_id = mapping + param_name, weight_name, expert_id, expert_shard_id = mapping if weight_name not in name: continue name_mapped = name.replace(weight_name, param_name) @@ -422,7 +423,7 @@ class DeepSeekV4MTP(nn.Module): param, loaded_weight, name_mapped, - shard_id=shard_id, + shard_id=expert_shard_id, expert_id=expert_id, return_success=True, ) diff --git a/vllm/models/deepseek_v4/quant_config.py b/vllm/models/deepseek_v4/quant_config.py new file mode 100644 index 0000000000..2d844b04ac --- /dev/null +++ b/vllm/models/deepseek_v4/quant_config.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Quantization config for DeepSeek V4.""" + +from vllm.config import get_current_vllm_config +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe.layer import UnquantizedFusedMoEMethod +from vllm.model_executor.layers.quantization import QuantizationMethods +from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + is_layer_skipped, +) + +_DEEPSEEK_V4_EXPERT_DTYPES = ("fp4", "fp8") + + +class DeepseekV4FP8Config(Fp8Config): + """FP8 config for DeepSeek V4 with expert-dtype-aware MoE dispatch. + + DeepSeek V4 checkpoints always use FP8 block quantization for + linear/attention layers. The MoE expert weights vary by checkpoint: + - ``expert_dtype="fp4"`` (e.g. DeepSeek-V4-Flash): MXFP4 experts + with ue8m0 (e8m0fnu) FP8 linear scales. + - ``expert_dtype="fp8"`` (e.g. DeepSeek-V4-Flash-Base): FP8 block + experts with float32 FP8 linear scales. + + The dispatch and the linear scale dtype are both keyed off + ``expert_dtype`` from the model's hf_config; missing values default + to ``"fp4"`` so existing FP4 checkpoints stay unchanged. + + NOTE: ``expert_dtype`` is resolved lazily because this config is + constructed during VllmConfig setup, before ``set_current_vllm_config`` + is active. Reading hf_config eagerly in ``__init__`` would always see + the default ``"fp4"`` and silently misroute Flash-Base checkpoints. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._resolved_expert_dtype: str | None = None + # ``is_scale_e8m0`` is a property that resolves on first read, + # by which time the current vllm_config has been set. + + @property + def expert_dtype(self) -> str: + if self._resolved_expert_dtype is None: + try: + hf_config = get_current_vllm_config().model_config.hf_config + except Exception: + # vllm_config not yet set; defer the decision until a + # later call lands inside set_current_vllm_config. + return "fp4" + expert_dtype = getattr(hf_config, "expert_dtype", "fp4") + if expert_dtype not in _DEEPSEEK_V4_EXPERT_DTYPES: + raise ValueError( + f"Unsupported DeepSeek V4 expert_dtype={expert_dtype!r}; " + f"expected one of {_DEEPSEEK_V4_EXPERT_DTYPES}." + ) + self._resolved_expert_dtype = expert_dtype + from vllm.logger import init_logger + + init_logger(__name__).info_once( + "DeepSeek V4 expert_dtype resolved to %r", expert_dtype + ) + return self._resolved_expert_dtype + + @property + def is_scale_e8m0(self) -> bool: + # FP4 checkpoints store FP8 linear scales as e8m0fnu; FP8 expert + # checkpoints (Flash-Base) store them as float32. + return self.expert_dtype == "fp4" + + @classmethod + def get_name(cls) -> QuantizationMethods: + return "deepseek_v4_fp8" + + @classmethod + def override_quantization_method( + cls, hf_quant_cfg, user_quant, hf_config=None + ) -> QuantizationMethods | None: + if not ( + isinstance(hf_quant_cfg, dict) + and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") + ): + return None + model_type = getattr(hf_config, "model_type", None) + if model_type == "deepseek_v4" or user_quant == "deepseek_v4_fp8": + return "deepseek_v4_fp8" + return None + + def get_quant_method(self, layer, prefix): + if isinstance(layer, FusedMoE): + if is_layer_skipped( + prefix=prefix, + ignored_layers=self.ignored_layers, + fused_mapping=self.packed_modules_mapping, + ): + return UnquantizedFusedMoEMethod(layer.moe_config) + if self.expert_dtype == "fp4": + return Mxfp4MoEMethod(layer.moe_config) + # expert_dtype == "fp8": fall through to Fp8Config which + # returns Fp8MoEMethod with block-wise float32 scales. + return super().get_quant_method(layer, prefix) + + def is_mxfp4_quant(self, prefix, layer): + return isinstance(layer, FusedMoE) and self.expert_dtype == "fp4"