[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)

Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Xuanteng Huang
2026-10-07 22:01:09 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 OpenAI Codex
parent b803edbc9f
commit f38c6778ce
18 changed files with 450 additions and 61 deletions
+25 -7
View File
@@ -103,6 +103,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_tp_weight_quant,
kMxfp8Dynamic,
kMxfp8Static,
kNvfp4DynamicToken,
kNvfp4Static,
weight_amax,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -211,15 +213,26 @@ def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
assert args.ignore == []
@pytest.mark.parametrize(
("is_act_and_mul", "expected_hidden_alignment"),
[(True, 256), (False, 512)],
)
def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
monkeypatch,
monkeypatch, is_act_and_mul, expected_hidden_alignment
) -> None:
method = object.__new__(Nvfp4OnlineMoEMethod)
method.moe = SimpleNamespace(is_act_and_mul=True)
method.nvfp4_backend = object()
method.experts_cls = object
method.moe_quant_config = None
method.moe_kernel = None
monkeypatch.setattr(current_platform, "is_device_capability_family", lambda _: True)
select_backend = Mock(return_value=(object(), object))
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.online.nvfp4.select_nvfp4_moe_backend",
select_backend,
)
moe_config = SimpleNamespace(is_act_and_mul=is_act_and_mul)
method = Nvfp4OnlineMoEMethod(moe=moe_config)
select_backend.assert_called_once_with(
config=moe_config,
weight_key=kNvfp4Static,
activation_key=kNvfp4DynamicToken,
)
layer = Mock()
converted_weights = [
@@ -274,7 +287,12 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
assert method.moe_kernel is kernel
assert convert_weights.call_count == 3
assert all(
call.kwargs["trtllm_hidden_alignment"] == expected_hidden_alignment
for call in convert_weights.call_args_list
)
make_kernel.assert_called_once()
assert make_kernel.call_args.kwargs["per_token_activation"] is True
get_quant_config.assert_called_once()
assert process_weights.call_count == 3