[Quantization] Support compressed-tensors MXFP4 MoE with FP8 activations via Mxfp4MoEMethod (#59815)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Misha Goin
2026-10-14 13:33:35 -07:00
committed by GitHub
co-authored by Claude Kevin H. Luu
parent fe98d90a9d
commit 984c242ede
12 changed files with 311 additions and 10 deletions
+143 -1
View File
@@ -18,7 +18,7 @@ from compressed_tensors.quantization import (
from tests.models.utils import check_logprobs_close
from tests.quantization.utils import load_model_without_vllm_runner
from vllm.config import set_current_vllm_config
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.kernel import KernelConfig
from vllm.forward_context import set_forward_context
from vllm.model_executor.kernels.linear import (
@@ -54,6 +54,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
ScaleDesc,
kFp8Dynamic128Sym,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.platforms import current_platform
@@ -1020,3 +1021,144 @@ def test_compressed_tensors_mxfp4(vllm_runner):
llm.apply_model(check_model)
output = llm.generate_greedy("Hello my name is", max_tokens=4)
assert output
def _mxfp4_moe_layer(monkeypatch, input_activations):
from types import SimpleNamespace
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization import mxfp4
# Exercise scheme routing and allocation without selecting a GPU kernel.
monkeypatch.setattr(
mxfp4,
"select_deepseek_v4_mxfp4_moe_backend",
lambda moe: (Mxfp4MoeBackend.NONE, None),
)
weights = {
"num_bits": 4,
"type": "float",
"strategy": "group",
"group_size": 32,
"symmetric": True,
"dynamic": False,
}
quant_config = CompressedTensorsConfig.from_config(
{
"format": "mxfp4-pack-quantized",
"config_groups": {
"group_0": {
# Name targets only match if the layer's own checkpoint
# projection names are used for the lookup.
"targets": [r"re:.*experts\.\d+\.w\d+$"],
"weights": weights,
"input_activations": input_activations,
}
},
}
)
return RoutedExperts(
"model.layers.0.ffn.experts",
torch.bfloat16,
make_dummy_moe_config(num_experts=2, hidden_dim=64, intermediate_size=32),
quant_config,
expert_map_manager=SimpleNamespace(
local_num_experts=2,
placement_strategy="linear",
expert_map=None,
expert_mask=None,
routing_tables=None,
map_global_to_local=lambda index: index,
),
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
)
def test_mxfp4_moe_with_fp8_inputs_shares_native_mxfp4_method(monkeypatch):
"""MXFP4 weights with dynamic FP8 inputs load compressed-tensors names but
otherwise behave as the native MXFP4 method, so its backends, padding and
monolithic support are not reimplemented."""
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
layer = _mxfp4_moe_layer(
monkeypatch,
{
"num_bits": 8,
"type": "float",
"strategy": "group",
"group_size": 128,
"symmetric": True,
"dynamic": True,
},
)
assert isinstance(layer.quant_method, Mxfp4MoEMethod)
assert not layer.quant_method.supports_pre_processed_weights
# The checkpoint's FP8 activations reach backends such as Humming that
# would otherwise default to BF16 activations.
assert layer.quant_method.activation_key == kFp8Dynamic128Sym
assert {"w13_weight_packed", "w2_weight_packed"} <= dict(
layer.named_parameters()
).keys()
w13 = layer.w13_weight_packed
layer.quant_method.process_weights_after_loading(layer)
assert layer.w13_weight is w13
assert not hasattr(layer, "w13_weight_packed")
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="FP8 x MXFP4 linear kernels need CUDA"
)
def test_mxfp4_linear_with_fp8_inputs_requests_fp8_activations():
"""Dense MXFP4 layers with dynamic FP8 inputs (e.g. GLM-5.3 dense MLPs)
must not be routed as W4A4."""
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsW4A8Mxfp4,
)
weights = QuantizationArgs(
num_bits=4, type="float", strategy="group", group_size=32, symmetric=True
)
inputs = QuantizationArgs(
num_bits=8,
type="float",
strategy="group",
group_size=128,
symmetric=True,
dynamic=True,
)
config = CompressedTensorsConfig.from_config(
{"format": "mxfp4-pack-quantized", "config_groups": {}}
)
with set_current_vllm_config(VllmConfig()):
scheme = config._get_scheme_from_parts(weights, inputs)
assert isinstance(scheme, CompressedTensorsW4A8Mxfp4)
assert scheme.kernel.config.activation_quant_key == kFp8Dynamic128Sym
def test_mxfp4_moe_with_mxfp4_inputs_keeps_w4a4_method(monkeypatch):
"""Only FP8 inputs take the shared path; W4A4 keeps its own kernels."""
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_w4a4_mxfp4 import ( # noqa: E501
CompressedTensorsW4A4Mxfp4MoEMethod,
)
layer = _mxfp4_moe_layer(
monkeypatch,
{
"num_bits": 4,
"type": "float",
"strategy": "group",
"group_size": 32,
"symmetric": True,
"dynamic": True,
},
)
assert isinstance(layer.quant_method, CompressedTensorsW4A4Mxfp4MoEMethod)