From d1a38c276202b23b5dc8a7bfc7f0b3b83a1ac913 Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Tue, 23 Jun 2026 04:17:18 +0800 Subject: [PATCH] [Kernel][Performance] Add FlashInfer cutedsl NVFP4 GEMM backend (#42235) Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> --- .../test_flashinfer_nvfp4_scaled_mm.py | 4 +- tests/models/quantization/test_nvfp4.py | 7 ++ .../passes/fusion/collective_fusion.py | 9 +++ vllm/config/kernel.py | 2 + .../model_executor/kernels/linear/__init__.py | 6 ++ .../kernels/linear/nvfp4/flashinfer.py | 66 +++++++++++++++++++ 6 files changed, 93 insertions(+), 1 deletion(-) diff --git a/tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py b/tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py index 698c679a20..a1e76d73a1 100644 --- a/tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py +++ b/tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py @@ -75,7 +75,7 @@ def get_ref_results( @pytest.mark.parametrize("shape", SHAPES) @pytest.mark.parametrize("seed", SEEDS) @pytest.mark.parametrize("device", CUDA_DEVICES) -@pytest.mark.parametrize("backend", ["cutlass", "cudnn", "trtllm", "b12x"]) +@pytest.mark.parametrize("backend", ["cute-dsl", "cutlass", "cudnn", "trtllm", "b12x"]) @pytest.mark.parametrize("autotune", [False, True]) @torch.inference_mode() def test_flashinfer_nvfp4_gemm( @@ -88,6 +88,8 @@ def test_flashinfer_nvfp4_gemm( ) -> None: if "trtllm" in backend and dtype == torch.float16: pytest.skip("Only torch.bfloat16 is supported for TRTLLM FP4 GEMM operations") + if backend == "cute-dsl" and not current_platform.is_device_capability_family(100): + pytest.skip("FlashInfer cutedsl backend is only supported on SM10x") if backend == "b12x" and not current_platform.has_device_capability(120): pytest.skip("b12x FP4 GEMM requires SM120+ (CC 12.0+)") if backend == "b12x" and not has_flashinfer_b12x_gemm(): diff --git a/tests/models/quantization/test_nvfp4.py b/tests/models/quantization/test_nvfp4.py index 660643eeab..f5022a461b 100644 --- a/tests/models/quantization/test_nvfp4.py +++ b/tests/models/quantization/test_nvfp4.py @@ -90,6 +90,7 @@ def test_models(example_prompts, model_name) -> None: EAGER = [True, False] SM_100_NVFP4_BACKENDS = [ + "flashinfer_cutedsl", "flashinfer_cudnn", "flashinfer_trtllm", "flashinfer_cutlass", @@ -102,12 +103,18 @@ SM_100_NVFP4_BACKENDS = [ "backend", [ "emulation", + "flashinfer_cutedsl", "flashinfer_cudnn", "flashinfer_trtllm", # the small seq_len ensures trtllm_8x4_layout backend is used "flashinfer_cutlass", ], ) def test_nvfp4(vllm_runner, model, eager, backend): + if backend == "flashinfer_cutedsl" and not ( + current_platform.is_device_capability_family(100) + ): + pytest.skip("The flashinfer_cutedsl backend is only supported on SM10x") + if ( not current_platform.has_device_capability(100) and backend in SM_100_NVFP4_BACKENDS diff --git a/vllm/compilation/passes/fusion/collective_fusion.py b/vllm/compilation/passes/fusion/collective_fusion.py index 3658877c67..7a413665ff 100644 --- a/vllm/compilation/passes/fusion/collective_fusion.py +++ b/vllm/compilation/passes/fusion/collective_fusion.py @@ -956,6 +956,15 @@ class AsyncTPPass(VllmFusionPatternMatcherPass): a_scale_view=a_scale_view, ) ) + self.register( + FlashInferAllGatherFP4Pattern( + self.model_dtype, + self.device, + "cute-dsl", + use_8x4_sf_layout=False, + a_scale_view="float8", + ) + ) # NVFP4 reduce-scatter does not need scale communication: FP4 # scales are consumed by the local GEMM and only BF16 partial # outputs are reduced. Keep this PR scoped to the all-gather diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index 46dad3aa44..cd1408c3e7 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -141,6 +141,7 @@ LinearBackend = Literal[ "auto", "cutlass", "flashinfer_cutlass", + "flashinfer_cutedsl", "flashinfer_trtllm", "flashinfer_cudnn", "flashinfer_b12x", @@ -198,6 +199,7 @@ class KernelConfig: - "auto": Automatically select the best backend based on model and hardware - "cutlass": Use CUTLASS-based kernels - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels + - "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels - "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels - "flashinfer_cudnn": Use FlashInfer with cuDNN kernels - "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 919d71fb8e..58ba7c8cb4 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -115,6 +115,7 @@ from vllm.model_executor.kernels.linear.nvfp4.fbgemm import ( from vllm.model_executor.kernels.linear.nvfp4.flashinfer import ( FlashInferB12xNvFp4LinearKernel, FlashInferCudnnNvFp4LinearKernel, + FlashInferCuteDslNvFp4LinearKernel, FlashInferCutlassNvFp4LinearKernel, FlashInferTrtllmNvFp4LinearKernel, ) @@ -209,6 +210,9 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { FlashInferCutlassNvFp4LinearKernel, FlashInferMxFp4LinearKernel, }, + "flashinfer_cutedsl": { + FlashInferCuteDslNvFp4LinearKernel, + }, "flashinfer_trtllm": { FlashInferTrtllmNvFp4LinearKernel, }, @@ -399,6 +403,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = { _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = { PlatformEnum.CUDA: [ + FlashInferCuteDslNvFp4LinearKernel, # FlashInferB12xNvFp4LinearKernel excluded from auto-selection until # upstream CUTLASS SM121 MMA op guard is resolved; use # --linear-backend flashinfer_b12x to opt in explicitly. @@ -1038,6 +1043,7 @@ __all__ = [ "CutlassNvFp4LinearKernel", "EmulationNvFp4LinearKernel", "FbgemmNvFp4LinearKernel", + "FlashInferCuteDslNvFp4LinearKernel", "FlashInferB12xNvFp4LinearKernel", "FlashInferCutlassNvFp4LinearKernel", "FlashInferTrtllmNvFp4LinearKernel", diff --git a/vllm/model_executor/kernels/linear/nvfp4/flashinfer.py b/vllm/model_executor/kernels/linear/nvfp4/flashinfer.py index 84c695693f..b721a135e2 100644 --- a/vllm/model_executor/kernels/linear/nvfp4/flashinfer.py +++ b/vllm/model_executor/kernels/linear/nvfp4/flashinfer.py @@ -28,6 +28,72 @@ from vllm.utils.flashinfer import ( from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig +class FlashInferCuteDslNvFp4LinearKernel(NvFp4LinearKernel): + """NVFP4 GEMM via FlashInfer's cutedsl backend.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_device_capability_family(100): + return False, "FlashInfer cutedsl requires sm_10x" + if not has_flashinfer(): + return False, "FlashInfer required" + return True, None + + @classmethod + def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]: + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # cutedsl uses the same swizzled + padded layout as cutlass. + layer.weight_scale = torch.nn.Parameter( + swizzle_blockscale(layer.weight_scale.data), requires_grad=False + ) + padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass( + layer.weight.data + ) + layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False) + layer.weights_padding_cols = weights_padding_cols + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + output_size = layer.output_size_per_partition + output_dtype = x.dtype + output_shape = [*x.shape[:-1], output_size] + + x_fp4, x_blockscale = scaled_fp4_quant( + x, + layer.input_global_scale_inv, + is_sf_swizzled_layout=True, + backend="flashinfer-cutedsl", + ) + + x_fp4 = pad_nvfp4_activation_for_cutlass( + x_fp4, getattr(layer, "weights_padding_cols", 0) + ) + + out = flashinfer_scaled_fp4_mm( + x_fp4, + layer.weight, + x_blockscale, + layer.weight_scale, + layer.alpha, + output_dtype, + backend="cute-dsl", + ) + + out = slice_nvfp4_output(out, output_size) + + if bias is not None: + out = out + bias + return out.view(*output_shape) + + class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel): """NVFP4 GEMM via FlashInfer's CUTLASS wrapper."""