[Quantization] Support compressed-tensors MXFP4 MoE with FP8 activations via Mxfp4MoEMethod (#59815)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Misha Goin
2026-10-14 13:33:35 -07:00
committed by GitHub
co-authored by Claude Kevin H. Luu
parent fe98d90a9d
commit 984c242ede
12 changed files with 311 additions and 10 deletions
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsW4A4Mxfp4,
CompressedTensorsW4A8Fp8,
CompressedTensorsW4A8Int,
CompressedTensorsW4A8Mxfp4,
CompressedTensorsW8A8Fp8,
CompressedTensorsW8A8Int8,
CompressedTensorsW8A8Mxfp8,
@@ -63,6 +64,11 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
create_fp8_quant_key,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -449,6 +455,39 @@ class CompressedTensorsConfig(QuantizationConfig):
and is_symmetric
)
@classmethod
def _is_mxfp4_w4a8_fp8(
cls, weight_quant: QuantizationArgs, input_quant: QuantizationArgs
) -> bool:
if not cls._is_mxfp4(weight_quant) or input_quant is None:
return False
is_float_type = input_quant.type == QuantizationType.FLOAT
is_8_bits = input_quant.num_bits == 8
is_static_weight = not weight_quant.dynamic
is_dynamic_input = bool(input_quant.dynamic)
is_token_or_group = input_quant.strategy in (
QuantizationStrategy.TOKEN,
QuantizationStrategy.GROUP,
)
return (
is_float_type
and is_8_bits
and is_static_weight
and is_dynamic_input
and is_token_or_group
and input_quant.symmetric
)
@staticmethod
def _fp8_dynamic_input_key(input_quant: QuantizationArgs) -> QuantKey:
if input_quant.strategy == QuantizationStrategy.GROUP:
group_shape = GroupShape(1, input_quant.group_size)
else:
group_shape = GroupShape.PER_TOKEN
return create_fp8_quant_key(static=False, group_shape=group_shape)
@staticmethod
def _is_mxfp8(quant_args: QuantizationArgs) -> bool:
if quant_args is None:
@@ -764,6 +803,11 @@ class CompressedTensorsConfig(QuantizationConfig):
)
return CompressedTensorsW4A4Fp4()
if self._is_mxfp4_w4a8_fp8(weight_quant, input_quant):
return CompressedTensorsW4A8Mxfp4(
activation_quant_key=self._fp8_dynamic_input_key(input_quant)
)
if self._is_mxfp4(weight_quant):
return CompressedTensorsW4A4Mxfp4()
@@ -34,8 +34,12 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
# make sure quantization config for Linear can target it
quant_config._add_fused_moe_to_target_scheme_map()
unfused_names = [
layer_name + proj_name
for proj_name in [".0.gate_proj", ".0.up_proj", ".0.down_proj"]
f"{layer_name}.0.{proj_name}"
for proj_name in (
layer.ckpt_gate_proj_name,
layer.ckpt_up_proj_name,
layer.ckpt_down_proj_name,
)
]
# TODO: refactor this to use expert_mapping and check all layer numbers
all_scheme_dicts = [
@@ -59,6 +63,16 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
input_quant = scheme_dict.get("input_activations")
format = scheme_dict.get("format")
if quant_config._is_mxfp4_w4a8_fp8(weight_quant, input_quant):
from .compressed_tensors_moe_w4a8_mxfp4 import (
CompressedTensorsW4A8Mxfp4MoEMethod,
)
return CompressedTensorsW4A8Mxfp4MoEMethod(
layer.moe_config,
activation_key=quant_config._fp8_dynamic_input_key(input_quant),
)
if quant_config._is_mxfp4(weight_quant):
from .compressed_tensors_moe_w4a4_mxfp4 import (
CompressedTensorsW4A4Mxfp4MoEMethod,
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
_PACKED_WEIGHTS = ("w13_weight", "w2_weight")
class CompressedTensorsW4A8Mxfp4MoEMethod(Mxfp4MoEMethod):
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
Only the checkpoint naming and the declared activation scheme differ from
the native MXFP4 path, so backend selection, weight conversion and
execution are inherited.
"""
# The packed -> unpacked rename means a processed state dict no longer
# matches the parameters registered by create_weights.
supports_pre_processed_weights = False
def create_weights(self, layer: RoutedExperts, *args, **kwargs):
super().create_weights(layer, *args, **kwargs)
for name in _PACKED_WEIGHTS:
param = getattr(layer, name)
delattr(layer, name)
layer.register_parameter(f"{name}_packed", param)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
for name in _PACKED_WEIGHTS:
param = getattr(layer, f"{name}_packed")
delattr(layer, f"{name}_packed")
layer.register_parameter(name, param)
super().process_weights_after_loading(layer)
@@ -6,6 +6,7 @@ from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
from .compressed_tensors_w4a8_mxfp4 import CompressedTensorsW4A8Mxfp4
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
@@ -23,6 +24,7 @@ __all__ = [
"CompressedTensorsW8A8Int8",
"CompressedTensorsW8A8Fp8",
"CompressedTensorsW4A4Mxfp4",
"CompressedTensorsW4A8Mxfp4",
"CompressedTensorsW4A4Fp4",
"CompressedTensorsW4A8Int",
"CompressedTensorsW4A8Fp8",
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsScheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
)
from vllm.model_executor.parameter import (
@@ -35,10 +36,10 @@ class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
Otherwise: W4A16 weight-only via Marlin.
"""
def __init__(self):
def __init__(self, activation_quant_key: QuantKey = kMxfp4Dynamic):
self.group_size = 32
self.kernel = init_mxfp4_linear_kernel(
activation_quant_key=kMxfp4Dynamic,
activation_quant_key=activation_quant_key,
)
@classmethod
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w4a4_mxfp4 import ( # noqa: E501
CompressedTensorsW4A4Mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
logger = init_logger(__name__)
__all__ = ["CompressedTensorsW4A8Mxfp4"]
class CompressedTensorsW4A8Mxfp4(CompressedTensorsW4A4Mxfp4):
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
The weight layout matches W4A4; only the activation quantization differs.
The checkpoint's FP8 activations are used when a kernel supports them,
otherwise this falls back to the W4A4 activation format.
"""
def __init__(self, activation_quant_key: QuantKey):
try:
super().__init__(activation_quant_key=activation_quant_key)
except ValueError as e:
logger.warning_once(
"No MXFP4 linear kernel supports %s activations on this "
"platform; falling back to MXFP4 activations. %s",
activation_quant_key,
e,
)
super().__init__()
@@ -38,7 +38,10 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
is_layer_skipped,
)
from vllm.model_executor.utils import (
is_weights_pre_processed,
replace_parameter,
@@ -541,9 +544,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
supports_pre_processed_weights = True
def __init__(self, moe: FusedMoEConfig):
def __init__(self, moe: FusedMoEConfig, activation_key: QuantKey | None = None):
super().__init__(moe)
self.weight_dtype = "mxfp4"
# Checkpoint-declared activation quantization. Only backends that pick
# their activation format from the layer (Humming) consume it.
self.activation_key = activation_key
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
self.experts_cls: type[mk.FusedMoEExperts] | None
@@ -822,6 +828,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
_cache_permute_indices=self._cache_permute_indices,
activation=self.moe.activation,
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
activation_key=self.activation_key,
)
)
@@ -59,6 +59,7 @@ def prepare_humming_linear_layer_config(
input_quant_config: dict | None = None,
*,
input_schema: "BaseInputSchema | None" = None,
allow_input_schema_fallback: bool = True,
) -> "LayerConfig":
from vllm.utils.humming import (
BaseInputSchema,
@@ -119,6 +120,7 @@ def prepare_humming_linear_layer_config(
weight_schema=weight_schema,
input_schema=input_schema,
param_dtype=layer.params_dtype,
allow_fallback=allow_input_schema_fallback,
)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name: