[Quantization] Add per-token NVFP4 CuTe-DSL MoE backend (#50030)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
aoshen02
parent
d5ee309314
commit
0103a9b96f
@@ -29,14 +29,20 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import (
|
||||
FlashInferCuteDSLExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
make_nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
quantize_nvfp4_per_token_input,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutedsl_moe_nvfp4
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
@@ -159,8 +165,12 @@ def _dequantize_nvfp4_linear(
|
||||
return (tensor_f32 * tensor_sf.unsqueeze(-1)).reshape(m, k).to(dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k,e,topk", [(16, 128, 512, 4, 2)])
|
||||
@pytest.mark.parametrize(
|
||||
"m,n,k,e,topk",
|
||||
[(16, 128, 512, 4, 2), (16, 192, 512, 4, 2), (16, 192, 2688, 4, 2)],
|
||||
)
|
||||
@pytest.mark.parametrize("activation,alpha,beta,limit", _ACT_CASES)
|
||||
@pytest.mark.parametrize("per_token_activation", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_cutedsl_fp4_moe(
|
||||
@@ -173,6 +183,7 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
alpha: float | None,
|
||||
beta: float | None,
|
||||
limit: float | None,
|
||||
per_token_activation: bool,
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
@@ -181,6 +192,10 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
hidden_states = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
# Keep GEMM1 activation variance stable as K changes, so the fixed
|
||||
# tolerance against the BF16 GEMM2 reference measures comparable FP4
|
||||
# quantization error at each hidden width.
|
||||
hidden_states *= (512 / k) ** 0.5
|
||||
|
||||
w1_rows = 2 * n if activation.is_gated else n
|
||||
w1 = torch.randn((e, w1_rows, k), device="cuda", dtype=dtype) / 15
|
||||
@@ -193,7 +208,6 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
hidden_states, score, topk, renormalize=False
|
||||
)
|
||||
|
||||
activation = MoEActivation.RELU2_NO_MUL
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
@@ -231,19 +245,30 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
w2_scale_2=(1.0 / w2_global_scale),
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=w1_alpha,
|
||||
g2_alphas=w2_alpha,
|
||||
a1_gscale=(1.0 / a1_scale),
|
||||
a2_gscale=(1.0 / a2_scale),
|
||||
w1_scale=w1_scale_cutedsl,
|
||||
# I=192 must remain unpadded for gated experts, but become 256 for
|
||||
# non-gated experts. The reference below uses the checkpoint shapes.
|
||||
expected_n = n if activation.is_gated else (n + 127) // 128 * 128
|
||||
assert moe_config.intermediate_size_per_partition == expected_n
|
||||
assert w1_cutedsl.shape[1] == expected_n * (2 if activation.is_gated else 1)
|
||||
assert w2_cutedsl.shape[2] * 2 == expected_n
|
||||
expected_k = (k + 255) // 256 * 256
|
||||
assert moe_config.hidden_dim == expected_k
|
||||
assert moe_config.hidden_dim_unpadded == k
|
||||
assert w1_cutedsl.shape[2] * 2 == expected_k
|
||||
assert w2_cutedsl.shape[1] == expected_k
|
||||
quant_config = make_nvfp4_moe_quant_config(
|
||||
backend=NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
w13_scale_2=w1_alpha,
|
||||
w2_scale_2=w2_alpha,
|
||||
a13_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
w13_scale=w1_scale_cutedsl,
|
||||
w2_scale=w2_scale_cutedsl,
|
||||
is_scale_swizzled=False,
|
||||
# Unset params must be omitted rather than forwarded as None into
|
||||
# the kernel's float-typed SwiGLU arguments.
|
||||
gemm1_alpha=alpha,
|
||||
gemm1_beta=beta,
|
||||
gemm1_clamp_limit=limit,
|
||||
swiglu_alpha=alpha,
|
||||
swiglu_beta=beta,
|
||||
swiglu_limit=limit,
|
||||
)
|
||||
|
||||
cutedsl_experts = mk.FusedMoEKernel(
|
||||
@@ -256,11 +281,13 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
FlashInferCuteDSLExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
per_token_activation=per_token_activation,
|
||||
),
|
||||
)
|
||||
|
||||
cutedsl_output = cutedsl_experts.apply(
|
||||
hidden_states=hidden_states,
|
||||
# The model runner pads inputs and trims outputs around this kernel.
|
||||
hidden_states=torch.nn.functional.pad(hidden_states, (0, expected_k - k)),
|
||||
w1=w1_cutedsl,
|
||||
w2=w2_cutedsl,
|
||||
topk_weights=topk_weights,
|
||||
@@ -270,19 +297,25 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
cutedsl_output = cutedsl_output[:, :k]
|
||||
|
||||
a_global_scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
a_q, a_scale = ops.scaled_fp4_quant(
|
||||
hidden_states,
|
||||
a_global_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
if per_token_activation:
|
||||
a_q, a_scale, a_row_scale = quantize_nvfp4_per_token_input(hidden_states)
|
||||
else:
|
||||
a_q, a_scale = ops.scaled_fp4_quant(
|
||||
hidden_states,
|
||||
a_global_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
a_in_dtype = _dequantize_nvfp4_linear(
|
||||
a_q,
|
||||
a_scale,
|
||||
a_global_scale,
|
||||
dtype=dtype,
|
||||
)
|
||||
if per_token_activation:
|
||||
a_in_dtype = (a_in_dtype.float() * a_row_scale.view(-1, 1)).to(dtype)
|
||||
|
||||
w1_d = torch.empty((e, w1_rows, k), device="cuda", dtype=dtype)
|
||||
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
|
||||
|
||||
Reference in New Issue
Block a user