From bd8865a299c4a68cff9b6443b9fd795f4c4735f6 Mon Sep 17 00:00:00 2001 From: Seonjin Date: Thu, 20 Aug 2026 07:07:22 -0700 Subject: [PATCH] [Kernel] Add FlashInfer TRTLLM MXFP8 linear backend (#52204) Signed-off-by: seonjinn Signed-off-by: Misha Goin Co-authored-by: Misha Goin --- docs/features/quantization/modelopt.md | 5 + .../test_flashinfer_mxfp8_trtllm.py | 123 ++++++++++++++++++ .../model_executor/kernels/linear/__init__.py | 4 + .../kernels/linear/mxfp8/flashinfer.py | 100 ++++++++++++++ vllm/utils/flashinfer.py | 33 +++++ 5 files changed, 265 insertions(+) create mode 100644 tests/kernels/quantization/test_flashinfer_mxfp8_trtllm.py diff --git a/docs/features/quantization/modelopt.md b/docs/features/quantization/modelopt.md index 7850dc75cc..139497abab 100644 --- a/docs/features/quantization/modelopt.md +++ b/docs/features/quantization/modelopt.md @@ -33,6 +33,11 @@ following `quantization.quant_algo` values: [Engine Arguments](../../configuration/engine_args.md) page and shown by `vllm serve --help=KernelConfig`. +!!! note + For models quantized to MXFP8 with BF16 activations on SM100-family GPUs, + use `--linear-backend flashinfer_trtllm` to select FlashInfer's TensorRT-LLM + GEMM backend. + ## Quantizing HuggingFace Models with PTQ You can quantize HuggingFace models using the example scripts provided in the Model Optimizer repository. The primary script for LLM PTQ is typically found within the `examples/llm_ptq` directory. diff --git a/tests/kernels/quantization/test_flashinfer_mxfp8_trtllm.py b/tests/kernels/quantization/test_flashinfer_mxfp8_trtllm.py new file mode 100644 index 0000000000..315e35b782 --- /dev/null +++ b/tests/kernels/quantization/test_flashinfer_mxfp8_trtllm.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import torch.nn.functional as F + +from vllm.model_executor.kernels.linear import ( + FlashInferTrtllmMxfp8LinearKernel, + Mxfp8LinearLayerConfig, +) +from vllm.platforms import current_platform +from vllm.utils import flashinfer as vllm_flashinfer +from vllm.utils.flashinfer import has_flashinfer + +if not ( + current_platform.is_cuda() + and current_platform.is_device_capability_family(100) + and has_flashinfer() +): + pytest.skip( + reason="FlashInfer TRTLLM MXFP8 requires an SM100-family GPU", + allow_module_level=True, + ) + + +def _make_layer(weight: torch.Tensor) -> torch.nn.Module: + from flashinfer import SfLayout, mxfp8_quantize + + weight_mxfp8, weight_scale = mxfp8_quantize( + weight, + sf_swizzle_layout=SfLayout.layout_linear, + ) + layer = torch.nn.Module() + layer.weight = torch.nn.Parameter(weight_mxfp8, requires_grad=False) + layer.weight_scale = torch.nn.Parameter( + weight_scale.view(weight.shape[0], weight.shape[1] // 32), + requires_grad=False, + ) + return layer + + +@pytest.mark.parametrize("shape", [(1, 130, 256), (7, 256, 512), (128, 130, 768)]) +@torch.inference_mode() +def test_flashinfer_trtllm_mxfp8_linear_numerics( + shape: tuple[int, int, int], +) -> None: + torch.manual_seed(0) + m, n, k = shape + x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda") + weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda") + layer = _make_layer(weight) + kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig()) + kernel.process_weights_after_loading(layer) + + output = kernel.apply_weights(layer, x) + reference = torch.mm(x, weight.t()) + similarity = F.cosine_similarity( + output.float().flatten(), reference.float().flatten(), dim=0 + ) + + assert output.shape == (m, n) + assert output.is_contiguous() + assert similarity.item() > 0.98 + + +@torch.inference_mode() +def test_flashinfer_trtllm_mxfp8_custom_ops() -> None: + x = torch.randn((7, 512), dtype=torch.bfloat16, device="cuda") + weight = torch.randn((256, 512), dtype=torch.bfloat16, device="cuda") + layer = _make_layer(weight) + kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig()) + kernel.process_weights_after_loading(layer) + + torch.library.opcheck( + torch.ops.vllm.flashinfer_mxfp8_quantize_8x4.default, + (x,), + ) + x_mxfp8, x_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(x) + # SchemaCheckMode compares inputs with allclose, which CUDA does not + # implement for float8. The numerical tests above guard input mutation. + torch.library.opcheck( + torch.ops.vllm.mm_mxfp8.default, + ( + x_mxfp8, + layer.weight.t(), + x_scale, + layer.weight_scale, + torch.bfloat16, + "trtllm", + True, + ), + test_utils=( + "test_autograd_registration", + "test_faketensor", + "test_aot_dispatch_dynamic", + ), + ) + + +@torch.inference_mode() +def test_flashinfer_trtllm_mxfp8_linear_cuda_graph() -> None: + torch.manual_seed(0) + m, n, k = 7, 130, 512 + weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda") + layer = _make_layer(weight) + kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig()) + kernel.process_weights_after_loading(layer) + + static_x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda") + kernel.apply_weights(layer, static_x) + torch.accelerator.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + graph_output = kernel.apply_weights(layer, static_x) + + new_x = torch.randn_like(static_x) + static_x.copy_(new_x) + graph.replay() + eager_output = kernel.apply_weights(layer, new_x) + + torch.testing.assert_close(graph_output, eager_output, rtol=0, atol=0) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index b94c8f7d58..16d3098df3 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -115,6 +115,7 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import ( from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( FlashInferCutedslMxfp8LinearKernel, FlashInferCutlassMxfp8LinearKernel, + FlashInferTrtllmMxfp8LinearKernel, ) from vllm.model_executor.kernels.linear.mxfp8.humming import ( HummingMxfp8LinearKernel, @@ -267,6 +268,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { FlashInferCutedslMxfp8LinearKernel, }, "flashinfer_trtllm": { + FlashInferTrtllmMxfp8LinearKernel, FlashInferTrtllmNvFp4LinearKernel, }, "flashinfer_cudnn": { @@ -507,6 +509,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = { B12xMxfp8LinearKernel, EmulationMxfp8LinearKernel, HummingMxfp8LinearKernel, + FlashInferTrtllmMxfp8LinearKernel, ], PlatformEnum.ROCM: [ # Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and @@ -1216,6 +1219,7 @@ __all__ = [ "MarlinMxFp4LinearKernel", "FlashInferCutedslMxfp8LinearKernel", "FlashInferCutlassMxfp8LinearKernel", + "FlashInferTrtllmMxfp8LinearKernel", "MarlinMxfp8LinearKernel", "XPUMxFp8LinearKernel", "EmulationMxfp8LinearKernel", diff --git a/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py b/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py index 3150d93f99..6ea9138de9 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py +++ b/vllm/model_executor/kernels/linear/mxfp8/flashinfer.py @@ -177,3 +177,103 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel): output_shape = (*input_shape[:-1], N) return output.view(output_shape) + + +class FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearKernel): + """MXFP8 W8A8 GEMM via FlashInfer's TensorRT-LLM wrapper.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not ( + current_platform.is_cuda() + and current_platform.is_device_capability_family(100) + ): + return False, "requires SM100-family GPU" + if not has_flashinfer(): + return False, "requires FlashInfer" + return True, None + + @classmethod + def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]: + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a + + if hasattr(layer, "_mxfp8_trtllm_output_size") and layer.weight_scale.ndim == 1: + return + + weight = layer.weight.data # [N, K] + N, K = weight.shape + if K % 256 != 0: + raise ValueError( + f"FlashInfer TRTLLM MXFP8 requires K to be divisible by 256, got K={K}." + ) + + scale_k = K // MXFP8_BLOCK_SIZE + weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous() + padded_n = ((N + 127) // 128) * 128 + if padded_n != N: + padded_weight = weight.new_zeros((padded_n, K)) + padded_weight[:N] = weight + weight = padded_weight + + padded_scale = weight_scale.new_zeros((padded_n, scale_k)) + padded_scale[:N] = weight_scale + weight_scale = padded_scale + else: + weight = weight.contiguous() + + layer.weight = Parameter( + shuffle_matrix_a(weight, 128).reshape(padded_n, K), + requires_grad=False, + ) + layer.weight_scale = Parameter( + shuffle_matrix_sf_a( + weight_scale, + 128, + num_elts_per_sf=MXFP8_BLOCK_SIZE, + ).reshape(-1), + requires_grad=False, + ) + layer._mxfp8_trtllm_output_size = N + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + assert x.dtype == torch.bfloat16, ( + f"FlashInfer TRTLLM MXFP8 requires bfloat16 activations, got {x.dtype}." + ) + + weight = layer.weight # shuffled [padded N, K] + weight_scale = layer.weight_scale + _, K = weight.shape + output_size = layer._mxfp8_trtllm_output_size + input_shape = x.shape + input_2d = x.view(-1, K) + + input_mxfp8, input_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4( + input_2d + ) + output = vllm_flashinfer.mm_mxfp8( + input_mxfp8, + weight.t(), + input_scale, + weight_scale, + out_dtype=x.dtype, + backend="trtllm", + use_8x4_sf_layout=True, + ) + if output.shape[-1] != output_size: + output = output[:, :output_size].contiguous() + + if bias is not None: + output = output + bias + + output_shape = (*input_shape[:-1], output_size) + return output.view(output_shape) diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index 4f7d2eecd7..f77b669cc2 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -726,6 +726,36 @@ if has_flashinfer(): rounded_m, rounded_n, dtype=torch.uint8, device=a.device ) + @torch.library.custom_op( + "vllm::flashinfer_mxfp8_quantize_8x4", + mutates_args=[], + device_types="cuda", + ) + def flashinfer_mxfp8_quantize_8x4( + a: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + from flashinfer import SfLayout + from flashinfer import mxfp8_quantize as mxfp8_quantize_ + + return mxfp8_quantize_( + a, + backend="cuda", + sf_swizzle_layout=SfLayout.layout_8x4, + ) + + @torch.library.register_fake( + "vllm::flashinfer_mxfp8_quantize_8x4", + ) + def flashinfer_mxfp8_quantize_8x4_fake( + a: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + m, k = a.shape + scale_size = cdiv(m, 8) * 8 * cdiv(k // 32, 4) * 4 + return ( + torch.empty(m, k, dtype=torch.float8_e4m3fn, device=a.device), + torch.empty(scale_size, dtype=torch.uint8, device=a.device), + ) + @torch.library.custom_op( "vllm::mm_mxfp8", mutates_args=[], @@ -738,6 +768,7 @@ if has_flashinfer(): B_scale: torch.Tensor, out_dtype: torch.dtype, backend: str = "cutlass", + use_8x4_sf_layout: bool = False, ) -> torch.Tensor: from flashinfer import mm_mxfp8 as mm_mxfp8_ @@ -749,6 +780,7 @@ if has_flashinfer(): out=None, out_dtype=out_dtype, backend=backend, + use_8x4_sf_layout=use_8x4_sf_layout, ) @torch.library.register_fake( @@ -761,6 +793,7 @@ if has_flashinfer(): B_scale: torch.Tensor, out_dtype: torch.dtype, backend: str = "cutlass", + use_8x4_sf_layout: bool = False, ) -> torch.Tensor: # A is [m, k], B is [k, n] -> output [m, n] return torch.empty(A.shape[0], B.shape[1], dtype=out_dtype, device=A.device)