[Quantization] Support compressed-tensors MXFP4 MoE with FP8 activations via Mxfp4MoEMethod (#59815)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude
Kevin H. Luu
parent
fe98d90a9d
commit
984c242ede
@@ -18,7 +18,7 @@ from compressed_tensors.quantization import (
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import KernelConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
@@ -54,6 +54,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
kFp8Dynamic128Sym,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.platforms import current_platform
|
||||
@@ -1020,3 +1021,144 @@ def test_compressed_tensors_mxfp4(vllm_runner):
|
||||
llm.apply_model(check_model)
|
||||
output = llm.generate_greedy("Hello my name is", max_tokens=4)
|
||||
assert output
|
||||
|
||||
|
||||
def _mxfp4_moe_layer(monkeypatch, input_activations):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization import mxfp4
|
||||
|
||||
# Exercise scheme routing and allocation without selecting a GPU kernel.
|
||||
monkeypatch.setattr(
|
||||
mxfp4,
|
||||
"select_deepseek_v4_mxfp4_moe_backend",
|
||||
lambda moe: (Mxfp4MoeBackend.NONE, None),
|
||||
)
|
||||
weights = {
|
||||
"num_bits": 4,
|
||||
"type": "float",
|
||||
"strategy": "group",
|
||||
"group_size": 32,
|
||||
"symmetric": True,
|
||||
"dynamic": False,
|
||||
}
|
||||
quant_config = CompressedTensorsConfig.from_config(
|
||||
{
|
||||
"format": "mxfp4-pack-quantized",
|
||||
"config_groups": {
|
||||
"group_0": {
|
||||
# Name targets only match if the layer's own checkpoint
|
||||
# projection names are used for the lookup.
|
||||
"targets": [r"re:.*experts\.\d+\.w\d+$"],
|
||||
"weights": weights,
|
||||
"input_activations": input_activations,
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
return RoutedExperts(
|
||||
"model.layers.0.ffn.experts",
|
||||
torch.bfloat16,
|
||||
make_dummy_moe_config(num_experts=2, hidden_dim=64, intermediate_size=32),
|
||||
quant_config,
|
||||
expert_map_manager=SimpleNamespace(
|
||||
local_num_experts=2,
|
||||
placement_strategy="linear",
|
||||
expert_map=None,
|
||||
expert_mask=None,
|
||||
routing_tables=None,
|
||||
map_global_to_local=lambda index: index,
|
||||
),
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
)
|
||||
|
||||
|
||||
def test_mxfp4_moe_with_fp8_inputs_shares_native_mxfp4_method(monkeypatch):
|
||||
"""MXFP4 weights with dynamic FP8 inputs load compressed-tensors names but
|
||||
otherwise behave as the native MXFP4 method, so its backends, padding and
|
||||
monolithic support are not reimplemented."""
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
|
||||
layer = _mxfp4_moe_layer(
|
||||
monkeypatch,
|
||||
{
|
||||
"num_bits": 8,
|
||||
"type": "float",
|
||||
"strategy": "group",
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
"dynamic": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert isinstance(layer.quant_method, Mxfp4MoEMethod)
|
||||
assert not layer.quant_method.supports_pre_processed_weights
|
||||
# The checkpoint's FP8 activations reach backends such as Humming that
|
||||
# would otherwise default to BF16 activations.
|
||||
assert layer.quant_method.activation_key == kFp8Dynamic128Sym
|
||||
assert {"w13_weight_packed", "w2_weight_packed"} <= dict(
|
||||
layer.named_parameters()
|
||||
).keys()
|
||||
w13 = layer.w13_weight_packed
|
||||
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer.w13_weight is w13
|
||||
assert not hasattr(layer, "w13_weight_packed")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="FP8 x MXFP4 linear kernels need CUDA"
|
||||
)
|
||||
def test_mxfp4_linear_with_fp8_inputs_requests_fp8_activations():
|
||||
"""Dense MXFP4 layers with dynamic FP8 inputs (e.g. GLM-5.3 dense MLPs)
|
||||
must not be routed as W4A4."""
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsW4A8Mxfp4,
|
||||
)
|
||||
|
||||
weights = QuantizationArgs(
|
||||
num_bits=4, type="float", strategy="group", group_size=32, symmetric=True
|
||||
)
|
||||
inputs = QuantizationArgs(
|
||||
num_bits=8,
|
||||
type="float",
|
||||
strategy="group",
|
||||
group_size=128,
|
||||
symmetric=True,
|
||||
dynamic=True,
|
||||
)
|
||||
config = CompressedTensorsConfig.from_config(
|
||||
{"format": "mxfp4-pack-quantized", "config_groups": {}}
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
scheme = config._get_scheme_from_parts(weights, inputs)
|
||||
|
||||
assert isinstance(scheme, CompressedTensorsW4A8Mxfp4)
|
||||
assert scheme.kernel.config.activation_quant_key == kFp8Dynamic128Sym
|
||||
|
||||
|
||||
def test_mxfp4_moe_with_mxfp4_inputs_keeps_w4a4_method(monkeypatch):
|
||||
"""Only FP8 inputs take the shared path; W4A4 keeps its own kernels."""
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_w4a4_mxfp4 import ( # noqa: E501
|
||||
CompressedTensorsW4A4Mxfp4MoEMethod,
|
||||
)
|
||||
|
||||
layer = _mxfp4_moe_layer(
|
||||
monkeypatch,
|
||||
{
|
||||
"num_bits": 4,
|
||||
"type": "float",
|
||||
"strategy": "group",
|
||||
"group_size": 32,
|
||||
"symmetric": True,
|
||||
"dynamic": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert isinstance(layer.quant_method, CompressedTensorsW4A4Mxfp4MoEMethod)
|
||||
|
||||
Reference in New Issue
Block a user