[Quantization] [Refactor] Create special "GptOssMxfp4MoeMethod" (#39604)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -30,6 +30,7 @@ QuantizationMethods = Literal[
|
||||
"torchao",
|
||||
"inc",
|
||||
"mxfp4",
|
||||
"gpt_oss_mxfp4",
|
||||
"mxfp8",
|
||||
"cpu_awq",
|
||||
"online",
|
||||
@@ -133,7 +134,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
ModelOptNvFp4Config,
|
||||
)
|
||||
from .moe_wna16 import MoeWNA16Config
|
||||
from .mxfp4 import Mxfp4Config
|
||||
from .mxfp4 import GptOssMxfp4Config, Mxfp4Config
|
||||
from .mxfp8 import Mxfp8Config
|
||||
from .online.base import OnlineQuantizationConfig
|
||||
from .torchao import TorchAOConfig
|
||||
@@ -160,6 +161,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"auto-round": INCConfig,
|
||||
"inc": INCConfig,
|
||||
"mxfp4": Mxfp4Config,
|
||||
"gpt_oss_mxfp4": GptOssMxfp4Config,
|
||||
"mxfp8": Mxfp8Config,
|
||||
"cpu_awq": CPUAWQConfig,
|
||||
"online": OnlineQuantizationConfig,
|
||||
|
||||
@@ -232,7 +232,7 @@ class AWQMarlinConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
# Skip override to marlin kernels, as they are not
|
||||
# batch invariant
|
||||
|
||||
@@ -110,13 +110,22 @@ class QuantizationConfig(ABC):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls,
|
||||
hf_quant_cfg: dict[str, Any],
|
||||
user_quant: str | None,
|
||||
hf_config: Any = None,
|
||||
) -> QuantizationMethods | None:
|
||||
"""
|
||||
Detects if this quantization method can support a given checkpoint
|
||||
format by overriding the user specified quantization method --
|
||||
this method should only be overwritten by subclasses in exceptional
|
||||
circumstances
|
||||
circumstances.
|
||||
|
||||
Args:
|
||||
hf_quant_cfg: The checkpoint's quantization config dict.
|
||||
user_quant: The user-specified quantization method string.
|
||||
hf_config: The HuggingFace model config object (e.g. for
|
||||
model_type checks). May be None if not available.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
@@ -104,7 +104,7 @@ class CPUAWQConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
quant_method = hf_quant_cfg.get("quant_method", "").lower()
|
||||
if current_platform.is_cpu() and (quant_method == "awq"):
|
||||
|
||||
@@ -84,7 +84,7 @@ class GGUFConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg: dict[str, Any], user_quant: str | None
|
||||
cls, hf_quant_cfg: dict[str, Any], user_quant: str | None, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
# When user explicitly specifies --quantization gguf, override
|
||||
# whatever quantization method is in the HF model config (e.g. fp8).
|
||||
|
||||
@@ -214,7 +214,7 @@ class GPTQMarlinConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
|
||||
|
||||
|
||||
@@ -453,7 +453,7 @@ class INCConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
"""Override the `auto-round` method to `inc`."""
|
||||
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
|
||||
|
||||
@@ -406,7 +406,7 @@ class ModelOptFp8Config(ModelOptQuantConfigBase):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
algo = cls._extract_modelopt_quant_algo(hf_quant_cfg)
|
||||
if algo is not None and algo == "FP8":
|
||||
@@ -1028,7 +1028,7 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
algo = cls._extract_modelopt_quant_algo(hf_quant_cfg)
|
||||
if algo is not None and ("NVFP4" in algo or "FP4" in algo):
|
||||
@@ -1525,7 +1525,7 @@ class ModelOptMxFp8Config(ModelOptQuantConfigBase):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
algo = cls._extract_modelopt_quant_algo(hf_quant_cfg)
|
||||
if algo is not None and "MXFP8" in algo:
|
||||
@@ -2052,7 +2052,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
algo = cls._extract_modelopt_quant_algo(hf_quant_cfg)
|
||||
if algo is not None and algo == "MIXED_PRECISION":
|
||||
|
||||
@@ -130,7 +130,7 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
can_convert = cls.is_moe_wna16_compatible(hf_quant_cfg)
|
||||
if can_convert and user_quant == "moe_wna16":
|
||||
|
||||
@@ -19,11 +19,11 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
convert_to_mxfp4_moe_kernel_format,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size,
|
||||
select_mxfp4_moe_backend,
|
||||
select_gpt_oss_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
@@ -38,6 +38,12 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Mxfp4Config(QuantizationConfig):
|
||||
"""Canonical base config for MXFP4 quantization.
|
||||
|
||||
Subclasses override get_name() and override_quantization_method() to
|
||||
register themselves as the handler for a specific checkpoint format.
|
||||
"""
|
||||
|
||||
def __init__(self, ignored_layers: list[str] | None = None):
|
||||
super().__init__()
|
||||
self.ignored_layers = ignored_layers
|
||||
@@ -62,6 +68,8 @@ class Mxfp4Config(QuantizationConfig):
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return []
|
||||
|
||||
# TODO (zyongye) This is only temporaty fallback.
|
||||
# We should have `Mxfp4MoEMethod` after this migration is complete.
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
@@ -79,7 +87,7 @@ class Mxfp4Config(QuantizationConfig):
|
||||
)
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, FusedMoE):
|
||||
return Mxfp4MoEMethod(layer.moe_config)
|
||||
return GptOssMxfp4MoEMethod(layer.moe_config)
|
||||
elif isinstance(layer, Attention):
|
||||
logger.debug_once(
|
||||
"MXFP4 attention layer is not implemented. "
|
||||
@@ -93,13 +101,46 @@ class Mxfp4Config(QuantizationConfig):
|
||||
return True
|
||||
|
||||
|
||||
class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
class GptOssMxfp4Config(Mxfp4Config):
|
||||
"""MXFP4 config for GPT-OSS checkpoints.
|
||||
|
||||
Checkpoints carry ``"quant_method": "mxfp4"`` in their JSON config.
|
||||
override_quantization_method() maps that to the canonical internal name
|
||||
so that the rest of the loading path uses "gpt_oss_mxfp4" consistently.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "gpt_oss_mxfp4"
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
# Match both "mxfp4" (original checkpoint value) and "gpt_oss_mxfp4"
|
||||
# (already normalized by verify_and_update_model_config) so that
|
||||
# explicit --quantization mxfp4 from the user doesn't cause a mismatch.
|
||||
if not (
|
||||
isinstance(hf_quant_cfg, dict)
|
||||
and hf_quant_cfg.get("quant_method") in ("mxfp4", "gpt_oss_mxfp4")
|
||||
):
|
||||
return None
|
||||
# Require explicit confirmation that this is a GPT-OSS model.
|
||||
# Do NOT fall back to returning the override when hf_config is None,
|
||||
# as that would silently claim all mxfp4 checkpoints.
|
||||
model_type = getattr(hf_config, "model_type", None)
|
||||
if model_type != "gpt_oss":
|
||||
return None
|
||||
return "gpt_oss_mxfp4"
|
||||
|
||||
|
||||
class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
|
||||
self.weight_dtype = "gpt_oss_mxfp4"
|
||||
self.mxfp4_backend, self.experts_cls = select_gpt_oss_mxfp4_moe_backend(moe)
|
||||
|
||||
self.max_capture_size = (
|
||||
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
|
||||
@@ -281,7 +322,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
# Convert weights to kernel format
|
||||
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
|
||||
convert_to_mxfp4_moe_kernel_format(
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=w13,
|
||||
|
||||
@@ -30,11 +30,11 @@ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_m
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
convert_to_mxfp4_moe_kernel_format,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size,
|
||||
select_mxfp4_moe_backend,
|
||||
select_gpt_oss_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_fp8_moe_layer_for_marlin,
|
||||
@@ -995,7 +995,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
self.w2_precision_config = None
|
||||
|
||||
if self.ocp_mx_scheme == "w_mxfp4":
|
||||
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
|
||||
self.mxfp4_backend, self.experts_cls = select_gpt_oss_mxfp4_moe_backend(moe)
|
||||
elif self.ocp_mx_scheme.startswith("w_mxfp4"):
|
||||
# TODO(bowenbao): refactor and introduce backends for other OCP MX schemes.
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.NONE
|
||||
@@ -1300,7 +1300,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
|
||||
# Convert weights to kernel format
|
||||
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
|
||||
convert_to_mxfp4_moe_kernel_format(
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=w13,
|
||||
|
||||
Reference in New Issue
Block a user