[Quantization] Add per-token NVFP4 CuTe-DSL MoE backend (#50030)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
Matej Sirovatka
2026-09-30 10:42:03 +00:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex aoshen02
parent d5ee309314
commit 0103a9b96f
9 changed files with 186 additions and 66 deletions
@@ -29,14 +29,20 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import (
FlashInferCuteDSLExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
NvFp4MoeBackend,
make_nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
quantize_nvfp4_per_token_input,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_cutedsl_moe_nvfp4
from vllm.utils.math_utils import next_power_of_2
@@ -159,8 +165,12 @@ def _dequantize_nvfp4_linear(
return (tensor_f32 * tensor_sf.unsqueeze(-1)).reshape(m, k).to(dtype)
@pytest.mark.parametrize("m,n,k,e,topk", [(16, 128, 512, 4, 2)])
@pytest.mark.parametrize(
"m,n,k,e,topk",
[(16, 128, 512, 4, 2), (16, 192, 512, 4, 2), (16, 192, 2688, 4, 2)],
)
@pytest.mark.parametrize("activation,alpha,beta,limit", _ACT_CASES)
@pytest.mark.parametrize("per_token_activation", [False, True])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_flashinfer_cutedsl_fp4_moe(
@@ -173,6 +183,7 @@ def test_flashinfer_cutedsl_fp4_moe(
alpha: float | None,
beta: float | None,
limit: float | None,
per_token_activation: bool,
dtype: torch.dtype,
workspace_init,
):
@@ -181,6 +192,10 @@ def test_flashinfer_cutedsl_fp4_moe(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
hidden_states = torch.randn((m, k), device="cuda", dtype=dtype) / 10
# Keep GEMM1 activation variance stable as K changes, so the fixed
# tolerance against the BF16 GEMM2 reference measures comparable FP4
# quantization error at each hidden width.
hidden_states *= (512 / k) ** 0.5
w1_rows = 2 * n if activation.is_gated else n
w1 = torch.randn((e, w1_rows, k), device="cuda", dtype=dtype) / 15
@@ -193,7 +208,6 @@ def test_flashinfer_cutedsl_fp4_moe(
hidden_states, score, topk, renormalize=False
)
activation = MoEActivation.RELU2_NO_MUL
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
@@ -231,19 +245,30 @@ def test_flashinfer_cutedsl_fp4_moe(
w2_scale_2=(1.0 / w2_global_scale),
a2_scale=a2_scale,
)
quant_config = nvfp4_moe_quant_config(
g1_alphas=w1_alpha,
g2_alphas=w2_alpha,
a1_gscale=(1.0 / a1_scale),
a2_gscale=(1.0 / a2_scale),
w1_scale=w1_scale_cutedsl,
# I=192 must remain unpadded for gated experts, but become 256 for
# non-gated experts. The reference below uses the checkpoint shapes.
expected_n = n if activation.is_gated else (n + 127) // 128 * 128
assert moe_config.intermediate_size_per_partition == expected_n
assert w1_cutedsl.shape[1] == expected_n * (2 if activation.is_gated else 1)
assert w2_cutedsl.shape[2] * 2 == expected_n
expected_k = (k + 255) // 256 * 256
assert moe_config.hidden_dim == expected_k
assert moe_config.hidden_dim_unpadded == k
assert w1_cutedsl.shape[2] * 2 == expected_k
assert w2_cutedsl.shape[1] == expected_k
quant_config = make_nvfp4_moe_quant_config(
backend=NvFp4MoeBackend.FLASHINFER_CUTEDSL,
w13_scale_2=w1_alpha,
w2_scale_2=w2_alpha,
a13_scale=a1_scale,
a2_scale=a2_scale,
w13_scale=w1_scale_cutedsl,
w2_scale=w2_scale_cutedsl,
is_scale_swizzled=False,
# Unset params must be omitted rather than forwarded as None into
# the kernel's float-typed SwiGLU arguments.
gemm1_alpha=alpha,
gemm1_beta=beta,
gemm1_clamp_limit=limit,
swiglu_alpha=alpha,
swiglu_beta=beta,
swiglu_limit=limit,
)
cutedsl_experts = mk.FusedMoEKernel(
@@ -256,11 +281,13 @@ def test_flashinfer_cutedsl_fp4_moe(
FlashInferCuteDSLExperts(
moe_config=moe_config,
quant_config=quant_config,
per_token_activation=per_token_activation,
),
)
cutedsl_output = cutedsl_experts.apply(
hidden_states=hidden_states,
# The model runner pads inputs and trims outputs around this kernel.
hidden_states=torch.nn.functional.pad(hidden_states, (0, expected_k - k)),
w1=w1_cutedsl,
w2=w2_cutedsl,
topk_weights=topk_weights,
@@ -270,19 +297,25 @@ def test_flashinfer_cutedsl_fp4_moe(
expert_map=None,
apply_router_weight_on_input=False,
)
cutedsl_output = cutedsl_output[:, :k]
a_global_scale = torch.ones(1, device="cuda", dtype=torch.float32)
a_q, a_scale = ops.scaled_fp4_quant(
hidden_states,
a_global_scale,
is_sf_swizzled_layout=False,
)
if per_token_activation:
a_q, a_scale, a_row_scale = quantize_nvfp4_per_token_input(hidden_states)
else:
a_q, a_scale = ops.scaled_fp4_quant(
hidden_states,
a_global_scale,
is_sf_swizzled_layout=False,
)
a_in_dtype = _dequantize_nvfp4_linear(
a_q,
a_scale,
a_global_scale,
dtype=dtype,
)
if per_token_activation:
a_in_dtype = (a_in_dtype.float() * a_row_scale.view(-1, 1)).to(dtype)
w1_d = torch.empty((e, w1_rows, k), device="cuda", dtype=dtype)
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
+24
View File
@@ -605,6 +605,30 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None:
ModelConfig(model=str(tmp_path), quantization="fp8_per_channel")
def test_nvfp4_per_token_backend_contract() -> None:
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import (
FlashInferCuteDSLExperts,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
TrtLlmNvFp4ExpertsModular,
TrtLlmNvFp4ExpertsMonolithic,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kNvfp4DynamicToken,
kNvfp4Static,
)
scheme = (kNvfp4Static, kNvfp4DynamicToken)
assert TrtLlmNvFp4ExpertsMonolithic._supports_quant_scheme(*scheme)
assert TrtLlmNvFp4ExpertsModular._supports_quant_scheme(*scheme)
assert FlashInferCuteDSLExperts._supports_quant_scheme(*scheme)
assert FlashInferCuteDSLExperts._supports_no_act_and_mul()
assert not FlashInferExperts._supports_quant_scheme(*scheme)
@pytest.mark.parametrize("per_token_activation", [False, True])
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
def test_nvfp4_one_sided_sizes_dispatched_activations(
@@ -11,8 +11,8 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_hidden_dim_for_fi,
align_fp4_moe_weights_for_fi,
align_trtllm_fp4_moe_hidden_dim_for_fi,
)
@@ -63,7 +63,7 @@ def test_align_trtllm_fp4_moe_hidden_dim_noop():
w2_scale = torch.arange(2 * 512 * 1, dtype=torch.uint8).reshape(2, 512, 1)
out_w13, out_w13_scale, out_w2, out_w2_scale, padded_hidden = (
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
)
assert padded_hidden == 512
@@ -90,7 +90,7 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
)
out_w13, out_w13_scale, out_w2, out_w2_scale, out_hidden_dim = (
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
)
assert out_hidden_dim == padded_hidden_dim