[Quantization] Select per-token NVFP4 MoE backends explicitly (#57176)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matej Sirovatka
2026-09-23 07:30:58 +00:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex aoshen02 Claude Opus 5.5
parent 0f2a15c927
commit 9127170b88
6 changed files with 97 additions and 21 deletions
+71
View File
@@ -36,6 +36,7 @@ from vllm.model_executor.kernels.linear.mxfp8.marlin import (
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
LinearBase,
@@ -503,6 +504,76 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None:
ModelConfig(model=str(tmp_path), quantization="fp8_per_channel")
@pytest.mark.parametrize("per_token_activation", [False, True])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
def test_nvfp4_one_sided_sizes_dispatched_activations(
monkeypatch, per_token_activation: bool, input_dtype: torch.dtype
) -> None:
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.all2all_utils import (
flashinfer_one_sided_dispatch_layout,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
TrtLlmNvFp4ExpertsModular,
)
config = make_dummy_moe_config(hidden_dim=6144, in_dtype=input_dtype)
config.moe_parallel_config.dp_size = 2
config.moe_parallel_config.ep_size = 2
config.moe_parallel_config.use_ep = True
config.moe_parallel_config.all2all_backend = "flashinfer_nvlink_one_sided"
expert_quant_config = FusedMoEQuantConfig.make("nvfp4")
def check_dispatch_layout(*, quant_config, input_dtype, **kwargs):
assert quant_config is expert_quant_config
layout = flashinfer_one_sided_dispatch_layout(
config.hidden_dim, quant_config, input_dtype=input_dtype
)
if per_token_activation:
assert input_dtype == config.in_dtype
assert (
layout.x_bytes_per_token == config.hidden_dim * config.in_dtype.itemsize
)
assert layout.x_sf_bytes_per_token == 0
else:
assert input_dtype is None
assert layout.x_bytes_per_token == config.hidden_dim // 2
assert layout.x_sf_bytes_per_token == config.hidden_dim // 16
assert expert_quant_config.quant_dtype == "nvfp4"
raise RuntimeError("dispatch layout verified before allocation")
monkeypatch.setattr(
nvfp4_oracle, "maybe_make_prepare_finalize", check_dispatch_layout
)
with pytest.raises(
RuntimeError, match="dispatch layout verified before allocation"
):
nvfp4_oracle.make_nvfp4_moe_kernel(
moe_quant_config=expert_quant_config,
moe_config=config,
experts_cls=TrtLlmNvFp4ExpertsModular,
backend=nvfp4_oracle.NvFp4MoeBackend.FLASHINFER_TRTLLM,
per_token_activation=per_token_activation,
)
@pytest.mark.parametrize(
"input_dtype",
[torch.float32, torch.float8_e4m3fn, torch.float8_e5m2, torch.int8],
)
def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtype):
from vllm.model_executor.layers.fused_moe.all2all_utils import (
flashinfer_one_sided_dispatch_layout,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
with pytest.raises(ValueError, match="unpacked inputs must be float16 or bfloat16"):
flashinfer_one_sided_dispatch_layout(
6144, FusedMoEQuantConfig.make("nvfp4"), input_dtype=input_dtype
)
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",