[Kernel][Performance] Add FlashInfer cutedsl NVFP4 GEMM backend (#42235)

Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
Mohammad Miadh Angkad
2026-06-22 16:17:18 -04:00
committed by GitHub
parent 2b4a7491ec
commit d1a38c2762
6 changed files with 93 additions and 1 deletions
@@ -75,7 +75,7 @@ def get_ref_results(
@pytest.mark.parametrize("shape", SHAPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("backend", ["cutlass", "cudnn", "trtllm", "b12x"])
@pytest.mark.parametrize("backend", ["cute-dsl", "cutlass", "cudnn", "trtllm", "b12x"])
@pytest.mark.parametrize("autotune", [False, True])
@torch.inference_mode()
def test_flashinfer_nvfp4_gemm(
@@ -88,6 +88,8 @@ def test_flashinfer_nvfp4_gemm(
) -> None:
if "trtllm" in backend and dtype == torch.float16:
pytest.skip("Only torch.bfloat16 is supported for TRTLLM FP4 GEMM operations")
if backend == "cute-dsl" and not current_platform.is_device_capability_family(100):
pytest.skip("FlashInfer cutedsl backend is only supported on SM10x")
if backend == "b12x" and not current_platform.has_device_capability(120):
pytest.skip("b12x FP4 GEMM requires SM120+ (CC 12.0+)")
if backend == "b12x" and not has_flashinfer_b12x_gemm():
+7
View File
@@ -90,6 +90,7 @@ def test_models(example_prompts, model_name) -> None:
EAGER = [True, False]
SM_100_NVFP4_BACKENDS = [
"flashinfer_cutedsl",
"flashinfer_cudnn",
"flashinfer_trtllm",
"flashinfer_cutlass",
@@ -102,12 +103,18 @@ SM_100_NVFP4_BACKENDS = [
"backend",
[
"emulation",
"flashinfer_cutedsl",
"flashinfer_cudnn",
"flashinfer_trtllm", # the small seq_len ensures trtllm_8x4_layout backend is used
"flashinfer_cutlass",
],
)
def test_nvfp4(vllm_runner, model, eager, backend):
if backend == "flashinfer_cutedsl" and not (
current_platform.is_device_capability_family(100)
):
pytest.skip("The flashinfer_cutedsl backend is only supported on SM10x")
if (
not current_platform.has_device_capability(100)
and backend in SM_100_NVFP4_BACKENDS
@@ -956,6 +956,15 @@ class AsyncTPPass(VllmFusionPatternMatcherPass):
a_scale_view=a_scale_view,
)
)
self.register(
FlashInferAllGatherFP4Pattern(
self.model_dtype,
self.device,
"cute-dsl",
use_8x4_sf_layout=False,
a_scale_view="float8",
)
)
# NVFP4 reduce-scatter does not need scale communication: FP4
# scales are consumed by the local GEMM and only BF16 partial
# outputs are reduced. Keep this PR scoped to the all-gather
+2
View File
@@ -141,6 +141,7 @@ LinearBackend = Literal[
"auto",
"cutlass",
"flashinfer_cutlass",
"flashinfer_cutedsl",
"flashinfer_trtllm",
"flashinfer_cudnn",
"flashinfer_b12x",
@@ -198,6 +199,7 @@ class KernelConfig:
- "auto": Automatically select the best backend based on model and hardware
- "cutlass": Use CUTLASS-based kernels
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
- "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
- "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+)
@@ -115,6 +115,7 @@ from vllm.model_executor.kernels.linear.nvfp4.fbgemm import (
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferB12xNvFp4LinearKernel,
FlashInferCudnnNvFp4LinearKernel,
FlashInferCuteDslNvFp4LinearKernel,
FlashInferCutlassNvFp4LinearKernel,
FlashInferTrtllmNvFp4LinearKernel,
)
@@ -209,6 +210,9 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
FlashInferCutlassNvFp4LinearKernel,
FlashInferMxFp4LinearKernel,
},
"flashinfer_cutedsl": {
FlashInferCuteDslNvFp4LinearKernel,
},
"flashinfer_trtllm": {
FlashInferTrtllmNvFp4LinearKernel,
},
@@ -399,6 +403,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
PlatformEnum.CUDA: [
FlashInferCuteDslNvFp4LinearKernel,
# FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
# upstream CUTLASS SM121 MMA op guard is resolved; use
# --linear-backend flashinfer_b12x to opt in explicitly.
@@ -1038,6 +1043,7 @@ __all__ = [
"CutlassNvFp4LinearKernel",
"EmulationNvFp4LinearKernel",
"FbgemmNvFp4LinearKernel",
"FlashInferCuteDslNvFp4LinearKernel",
"FlashInferB12xNvFp4LinearKernel",
"FlashInferCutlassNvFp4LinearKernel",
"FlashInferTrtllmNvFp4LinearKernel",
@@ -28,6 +28,72 @@ from vllm.utils.flashinfer import (
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
class FlashInferCuteDslNvFp4LinearKernel(NvFp4LinearKernel):
"""NVFP4 GEMM via FlashInfer's cutedsl backend."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_device_capability_family(100):
return False, "FlashInfer cutedsl requires sm_10x"
if not has_flashinfer():
return False, "FlashInfer required"
return True, None
@classmethod
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# cutedsl uses the same swizzled + padded layout as cutlass.
layer.weight_scale = torch.nn.Parameter(
swizzle_blockscale(layer.weight_scale.data), requires_grad=False
)
padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass(
layer.weight.data
)
layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False)
layer.weights_padding_cols = weights_padding_cols
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
output_size = layer.output_size_per_partition
output_dtype = x.dtype
output_shape = [*x.shape[:-1], output_size]
x_fp4, x_blockscale = scaled_fp4_quant(
x,
layer.input_global_scale_inv,
is_sf_swizzled_layout=True,
backend="flashinfer-cutedsl",
)
x_fp4 = pad_nvfp4_activation_for_cutlass(
x_fp4, getattr(layer, "weights_padding_cols", 0)
)
out = flashinfer_scaled_fp4_mm(
x_fp4,
layer.weight,
x_blockscale,
layer.weight_scale,
layer.alpha,
output_dtype,
backend="cute-dsl",
)
out = slice_nvfp4_output(out, output_size)
if bias is not None:
out = out + bias
return out.view(*output_shape)
class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
"""NVFP4 GEMM via FlashInfer's CUTLASS wrapper."""