[Kernel] Add FlashInfer TRTLLM MXFP8 linear backend (#52204)

Signed-off-by: seonjinn <sna@nvidia.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Seonjin
2026-08-20 10:07:22 -04:00
committed by GitHub
co-authored by Misha Goin
parent de216b6e64
commit bd8865a299
5 changed files with 265 additions and 0 deletions
+5
View File
@@ -33,6 +33,11 @@ following `quantization.quant_algo` values:
[Engine Arguments](../../configuration/engine_args.md) page and shown by
`vllm serve --help=KernelConfig`.
!!! note
For models quantized to MXFP8 with BF16 activations on SM100-family GPUs,
use `--linear-backend flashinfer_trtllm` to select FlashInfer's TensorRT-LLM
GEMM backend.
## Quantizing HuggingFace Models with PTQ
You can quantize HuggingFace models using the example scripts provided in the Model Optimizer repository. The primary script for LLM PTQ is typically found within the `examples/llm_ptq` directory.
@@ -0,0 +1,123 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.kernels.linear import (
FlashInferTrtllmMxfp8LinearKernel,
Mxfp8LinearLayerConfig,
)
from vllm.platforms import current_platform
from vllm.utils import flashinfer as vllm_flashinfer
from vllm.utils.flashinfer import has_flashinfer
if not (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and has_flashinfer()
):
pytest.skip(
reason="FlashInfer TRTLLM MXFP8 requires an SM100-family GPU",
allow_module_level=True,
)
def _make_layer(weight: torch.Tensor) -> torch.nn.Module:
from flashinfer import SfLayout, mxfp8_quantize
weight_mxfp8, weight_scale = mxfp8_quantize(
weight,
sf_swizzle_layout=SfLayout.layout_linear,
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(weight_mxfp8, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(
weight_scale.view(weight.shape[0], weight.shape[1] // 32),
requires_grad=False,
)
return layer
@pytest.mark.parametrize("shape", [(1, 130, 256), (7, 256, 512), (128, 130, 768)])
@torch.inference_mode()
def test_flashinfer_trtllm_mxfp8_linear_numerics(
shape: tuple[int, int, int],
) -> None:
torch.manual_seed(0)
m, n, k = shape
x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda")
weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda")
layer = _make_layer(weight)
kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig())
kernel.process_weights_after_loading(layer)
output = kernel.apply_weights(layer, x)
reference = torch.mm(x, weight.t())
similarity = F.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
)
assert output.shape == (m, n)
assert output.is_contiguous()
assert similarity.item() > 0.98
@torch.inference_mode()
def test_flashinfer_trtllm_mxfp8_custom_ops() -> None:
x = torch.randn((7, 512), dtype=torch.bfloat16, device="cuda")
weight = torch.randn((256, 512), dtype=torch.bfloat16, device="cuda")
layer = _make_layer(weight)
kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig())
kernel.process_weights_after_loading(layer)
torch.library.opcheck(
torch.ops.vllm.flashinfer_mxfp8_quantize_8x4.default,
(x,),
)
x_mxfp8, x_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(x)
# SchemaCheckMode compares inputs with allclose, which CUDA does not
# implement for float8. The numerical tests above guard input mutation.
torch.library.opcheck(
torch.ops.vllm.mm_mxfp8.default,
(
x_mxfp8,
layer.weight.t(),
x_scale,
layer.weight_scale,
torch.bfloat16,
"trtllm",
True,
),
test_utils=(
"test_autograd_registration",
"test_faketensor",
"test_aot_dispatch_dynamic",
),
)
@torch.inference_mode()
def test_flashinfer_trtllm_mxfp8_linear_cuda_graph() -> None:
torch.manual_seed(0)
m, n, k = 7, 130, 512
weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda")
layer = _make_layer(weight)
kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig())
kernel.process_weights_after_loading(layer)
static_x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda")
kernel.apply_weights(layer, static_x)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_output = kernel.apply_weights(layer, static_x)
new_x = torch.randn_like(static_x)
static_x.copy_(new_x)
graph.replay()
eager_output = kernel.apply_weights(layer, new_x)
torch.testing.assert_close(graph_output, eager_output, rtol=0, atol=0)
@@ -115,6 +115,7 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import (
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
FlashInferCutedslMxfp8LinearKernel,
FlashInferCutlassMxfp8LinearKernel,
FlashInferTrtllmMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.humming import (
HummingMxfp8LinearKernel,
@@ -267,6 +268,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
FlashInferCutedslMxfp8LinearKernel,
},
"flashinfer_trtllm": {
FlashInferTrtllmMxfp8LinearKernel,
FlashInferTrtllmNvFp4LinearKernel,
},
"flashinfer_cudnn": {
@@ -507,6 +509,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
B12xMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
HummingMxfp8LinearKernel,
FlashInferTrtllmMxfp8LinearKernel,
],
PlatformEnum.ROCM: [
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
@@ -1216,6 +1219,7 @@ __all__ = [
"MarlinMxFp4LinearKernel",
"FlashInferCutedslMxfp8LinearKernel",
"FlashInferCutlassMxfp8LinearKernel",
"FlashInferTrtllmMxfp8LinearKernel",
"MarlinMxfp8LinearKernel",
"XPUMxFp8LinearKernel",
"EmulationMxfp8LinearKernel",
@@ -177,3 +177,103 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel):
output_shape = (*input_shape[:-1], N)
return output.view(output_shape)
class FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearKernel):
"""MXFP8 W8A8 GEMM via FlashInfer's TensorRT-LLM wrapper."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
):
return False, "requires SM100-family GPU"
if not has_flashinfer():
return False, "requires FlashInfer"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a
if hasattr(layer, "_mxfp8_trtllm_output_size") and layer.weight_scale.ndim == 1:
return
weight = layer.weight.data # [N, K]
N, K = weight.shape
if K % 256 != 0:
raise ValueError(
f"FlashInfer TRTLLM MXFP8 requires K to be divisible by 256, got K={K}."
)
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
padded_n = ((N + 127) // 128) * 128
if padded_n != N:
padded_weight = weight.new_zeros((padded_n, K))
padded_weight[:N] = weight
weight = padded_weight
padded_scale = weight_scale.new_zeros((padded_n, scale_k))
padded_scale[:N] = weight_scale
weight_scale = padded_scale
else:
weight = weight.contiguous()
layer.weight = Parameter(
shuffle_matrix_a(weight, 128).reshape(padded_n, K),
requires_grad=False,
)
layer.weight_scale = Parameter(
shuffle_matrix_sf_a(
weight_scale,
128,
num_elts_per_sf=MXFP8_BLOCK_SIZE,
).reshape(-1),
requires_grad=False,
)
layer._mxfp8_trtllm_output_size = N
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
assert x.dtype == torch.bfloat16, (
f"FlashInfer TRTLLM MXFP8 requires bfloat16 activations, got {x.dtype}."
)
weight = layer.weight # shuffled [padded N, K]
weight_scale = layer.weight_scale
_, K = weight.shape
output_size = layer._mxfp8_trtllm_output_size
input_shape = x.shape
input_2d = x.view(-1, K)
input_mxfp8, input_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(
input_2d
)
output = vllm_flashinfer.mm_mxfp8(
input_mxfp8,
weight.t(),
input_scale,
weight_scale,
out_dtype=x.dtype,
backend="trtllm",
use_8x4_sf_layout=True,
)
if output.shape[-1] != output_size:
output = output[:, :output_size].contiguous()
if bias is not None:
output = output + bias
output_shape = (*input_shape[:-1], output_size)
return output.view(output_shape)
+33
View File
@@ -726,6 +726,36 @@ if has_flashinfer():
rounded_m, rounded_n, dtype=torch.uint8, device=a.device
)
@torch.library.custom_op(
"vllm::flashinfer_mxfp8_quantize_8x4",
mutates_args=[],
device_types="cuda",
)
def flashinfer_mxfp8_quantize_8x4(
a: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
from flashinfer import SfLayout
from flashinfer import mxfp8_quantize as mxfp8_quantize_
return mxfp8_quantize_(
a,
backend="cuda",
sf_swizzle_layout=SfLayout.layout_8x4,
)
@torch.library.register_fake(
"vllm::flashinfer_mxfp8_quantize_8x4",
)
def flashinfer_mxfp8_quantize_8x4_fake(
a: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
m, k = a.shape
scale_size = cdiv(m, 8) * 8 * cdiv(k // 32, 4) * 4
return (
torch.empty(m, k, dtype=torch.float8_e4m3fn, device=a.device),
torch.empty(scale_size, dtype=torch.uint8, device=a.device),
)
@torch.library.custom_op(
"vllm::mm_mxfp8",
mutates_args=[],
@@ -738,6 +768,7 @@ if has_flashinfer():
B_scale: torch.Tensor,
out_dtype: torch.dtype,
backend: str = "cutlass",
use_8x4_sf_layout: bool = False,
) -> torch.Tensor:
from flashinfer import mm_mxfp8 as mm_mxfp8_
@@ -749,6 +780,7 @@ if has_flashinfer():
out=None,
out_dtype=out_dtype,
backend=backend,
use_8x4_sf_layout=use_8x4_sf_layout,
)
@torch.library.register_fake(
@@ -761,6 +793,7 @@ if has_flashinfer():
B_scale: torch.Tensor,
out_dtype: torch.dtype,
backend: str = "cutlass",
use_8x4_sf_layout: bool = False,
) -> torch.Tensor:
# A is [m, k], B is [k, n] -> output [m, n]
return torch.empty(A.shape[0], B.shape[1], dtype=out_dtype, device=A.device)