[Quantization] [Refactor] Create special "GptOssMxfp4MoeMethod" (#39604)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Yongye Zhu
2026-04-13 12:53:58 -04:00
committed by GitHub
parent 4d042ed85f
commit 739e5945dc
18 changed files with 129 additions and 34 deletions
@@ -30,6 +30,7 @@ QuantizationMethods = Literal[
"torchao",
"inc",
"mxfp4",
"gpt_oss_mxfp4",
"mxfp8",
"cpu_awq",
"online",
@@ -133,7 +134,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
ModelOptNvFp4Config,
)
from .moe_wna16 import MoeWNA16Config
from .mxfp4 import Mxfp4Config
from .mxfp4 import GptOssMxfp4Config, Mxfp4Config
from .mxfp8 import Mxfp8Config
from .online.base import OnlineQuantizationConfig
from .torchao import TorchAOConfig
@@ -160,6 +161,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"auto-round": INCConfig,
"inc": INCConfig,
"mxfp4": Mxfp4Config,
"gpt_oss_mxfp4": GptOssMxfp4Config,
"mxfp8": Mxfp8Config,
"cpu_awq": CPUAWQConfig,
"online": OnlineQuantizationConfig,
@@ -232,7 +232,7 @@ class AWQMarlinConfig(QuantizationConfig):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> "QuantizationMethods | None":
# Skip override to marlin kernels, as they are not
# batch invariant
@@ -110,13 +110,22 @@ class QuantizationConfig(ABC):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls,
hf_quant_cfg: dict[str, Any],
user_quant: str | None,
hf_config: Any = None,
) -> QuantizationMethods | None:
"""
Detects if this quantization method can support a given checkpoint
format by overriding the user specified quantization method --
this method should only be overwritten by subclasses in exceptional
circumstances
circumstances.
Args:
hf_quant_cfg: The checkpoint's quantization config dict.
user_quant: The user-specified quantization method string.
hf_config: The HuggingFace model config object (e.g. for
model_type checks). May be None if not available.
"""
return None
@@ -104,7 +104,7 @@ class CPUAWQConfig(QuantizationConfig):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> "QuantizationMethods | None":
quant_method = hf_quant_cfg.get("quant_method", "").lower()
if current_platform.is_cpu() and (quant_method == "awq"):
@@ -84,7 +84,7 @@ class GGUFConfig(QuantizationConfig):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg: dict[str, Any], user_quant: str | None
cls, hf_quant_cfg: dict[str, Any], user_quant: str | None, hf_config=None
) -> "QuantizationMethods | None":
# When user explicitly specifies --quantization gguf, override
# whatever quantization method is in the HF model config (e.g. fp8).
@@ -214,7 +214,7 @@ class GPTQMarlinConfig(QuantizationConfig):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
@@ -453,7 +453,7 @@ class INCConfig(QuantizationConfig):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> "QuantizationMethods | None":
"""Override the `auto-round` method to `inc`."""
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
@@ -406,7 +406,7 @@ class ModelOptFp8Config(ModelOptQuantConfigBase):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
algo = cls._extract_modelopt_quant_algo(hf_quant_cfg)
if algo is not None and algo == "FP8":
@@ -1028,7 +1028,7 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
algo = cls._extract_modelopt_quant_algo(hf_quant_cfg)
if algo is not None and ("NVFP4" in algo or "FP4" in algo):
@@ -1525,7 +1525,7 @@ class ModelOptMxFp8Config(ModelOptQuantConfigBase):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
algo = cls._extract_modelopt_quant_algo(hf_quant_cfg)
if algo is not None and "MXFP8" in algo:
@@ -2052,7 +2052,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
algo = cls._extract_modelopt_quant_algo(hf_quant_cfg)
if algo is not None and algo == "MIXED_PRECISION":
@@ -130,7 +130,7 @@ class MoeWNA16Config(QuantizationConfig):
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
can_convert = cls.is_moe_wna16_compatible(hf_quant_cfg)
if can_convert and user_quant == "moe_wna16":
@@ -19,11 +19,11 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
convert_to_mxfp4_moe_kernel_format,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
mxfp4_round_up_hidden_size_and_intermediate_size,
select_mxfp4_moe_backend,
select_gpt_oss_mxfp4_moe_backend,
)
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
from vllm.model_executor.layers.quantization import QuantizationMethods
@@ -38,6 +38,12 @@ logger = init_logger(__name__)
class Mxfp4Config(QuantizationConfig):
"""Canonical base config for MXFP4 quantization.
Subclasses override get_name() and override_quantization_method() to
register themselves as the handler for a specific checkpoint format.
"""
def __init__(self, ignored_layers: list[str] | None = None):
super().__init__()
self.ignored_layers = ignored_layers
@@ -62,6 +68,8 @@ class Mxfp4Config(QuantizationConfig):
def get_config_filenames(cls) -> list[str]:
return []
# TODO (zyongye) This is only temporaty fallback.
# We should have `Mxfp4MoEMethod` after this migration is complete.
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
@@ -79,7 +87,7 @@ class Mxfp4Config(QuantizationConfig):
)
return UnquantizedLinearMethod()
elif isinstance(layer, FusedMoE):
return Mxfp4MoEMethod(layer.moe_config)
return GptOssMxfp4MoEMethod(layer.moe_config)
elif isinstance(layer, Attention):
logger.debug_once(
"MXFP4 attention layer is not implemented. "
@@ -93,13 +101,46 @@ class Mxfp4Config(QuantizationConfig):
return True
class Mxfp4MoEMethod(FusedMoEMethodBase):
class GptOssMxfp4Config(Mxfp4Config):
"""MXFP4 config for GPT-OSS checkpoints.
Checkpoints carry ``"quant_method": "mxfp4"`` in their JSON config.
override_quantization_method() maps that to the canonical internal name
so that the rest of the loading path uses "gpt_oss_mxfp4" consistently.
"""
@classmethod
def get_name(cls) -> QuantizationMethods:
return "gpt_oss_mxfp4"
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
# Match both "mxfp4" (original checkpoint value) and "gpt_oss_mxfp4"
# (already normalized by verify_and_update_model_config) so that
# explicit --quantization mxfp4 from the user doesn't cause a mismatch.
if not (
isinstance(hf_quant_cfg, dict)
and hf_quant_cfg.get("quant_method") in ("mxfp4", "gpt_oss_mxfp4")
):
return None
# Require explicit confirmation that this is a GPT-OSS model.
# Do NOT fall back to returning the override when hf_config is None,
# as that would silently claim all mxfp4 checkpoints.
model_type = getattr(hf_config, "model_type", None)
if model_type != "gpt_oss":
return None
return "gpt_oss_mxfp4"
class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_dtype = "mxfp4"
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
self.weight_dtype = "gpt_oss_mxfp4"
self.mxfp4_backend, self.experts_cls = select_gpt_oss_mxfp4_moe_backend(moe)
self.max_capture_size = (
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
@@ -281,7 +322,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
# Convert weights to kernel format
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
convert_to_mxfp4_moe_kernel_format(
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=w13,
@@ -30,11 +30,11 @@ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_m
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
convert_to_mxfp4_moe_kernel_format,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
mxfp4_round_up_hidden_size_and_intermediate_size,
select_mxfp4_moe_backend,
select_gpt_oss_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_fp8_moe_layer_for_marlin,
@@ -995,7 +995,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
self.w2_precision_config = None
if self.ocp_mx_scheme == "w_mxfp4":
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
self.mxfp4_backend, self.experts_cls = select_gpt_oss_mxfp4_moe_backend(moe)
elif self.ocp_mx_scheme.startswith("w_mxfp4"):
# TODO(bowenbao): refactor and introduce backends for other OCP MX schemes.
self.mxfp4_backend = Mxfp4MoeBackend.NONE
@@ -1300,7 +1300,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
# Convert weights to kernel format
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
convert_to_mxfp4_moe_kernel_format(
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=w13,