[Kernel] Add FlashInfer TRTLLM MXFP8 linear backend (#52204)

Signed-off-by: seonjinn <sna@nvidia.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Seonjin
2026-08-20 10:07:22 -04:00
committed by GitHub
co-authored by Misha Goin
parent de216b6e64
commit bd8865a299
5 changed files with 265 additions and 0 deletions
@@ -115,6 +115,7 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import (
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
FlashInferCutedslMxfp8LinearKernel,
FlashInferCutlassMxfp8LinearKernel,
FlashInferTrtllmMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.humming import (
HummingMxfp8LinearKernel,
@@ -267,6 +268,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
FlashInferCutedslMxfp8LinearKernel,
},
"flashinfer_trtllm": {
FlashInferTrtllmMxfp8LinearKernel,
FlashInferTrtllmNvFp4LinearKernel,
},
"flashinfer_cudnn": {
@@ -507,6 +509,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
B12xMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
HummingMxfp8LinearKernel,
FlashInferTrtllmMxfp8LinearKernel,
],
PlatformEnum.ROCM: [
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
@@ -1216,6 +1219,7 @@ __all__ = [
"MarlinMxFp4LinearKernel",
"FlashInferCutedslMxfp8LinearKernel",
"FlashInferCutlassMxfp8LinearKernel",
"FlashInferTrtllmMxfp8LinearKernel",
"MarlinMxfp8LinearKernel",
"XPUMxFp8LinearKernel",
"EmulationMxfp8LinearKernel",
@@ -177,3 +177,103 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel):
output_shape = (*input_shape[:-1], N)
return output.view(output_shape)
class FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearKernel):
"""MXFP8 W8A8 GEMM via FlashInfer's TensorRT-LLM wrapper."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
):
return False, "requires SM100-family GPU"
if not has_flashinfer():
return False, "requires FlashInfer"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a
if hasattr(layer, "_mxfp8_trtllm_output_size") and layer.weight_scale.ndim == 1:
return
weight = layer.weight.data # [N, K]
N, K = weight.shape
if K % 256 != 0:
raise ValueError(
f"FlashInfer TRTLLM MXFP8 requires K to be divisible by 256, got K={K}."
)
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
padded_n = ((N + 127) // 128) * 128
if padded_n != N:
padded_weight = weight.new_zeros((padded_n, K))
padded_weight[:N] = weight
weight = padded_weight
padded_scale = weight_scale.new_zeros((padded_n, scale_k))
padded_scale[:N] = weight_scale
weight_scale = padded_scale
else:
weight = weight.contiguous()
layer.weight = Parameter(
shuffle_matrix_a(weight, 128).reshape(padded_n, K),
requires_grad=False,
)
layer.weight_scale = Parameter(
shuffle_matrix_sf_a(
weight_scale,
128,
num_elts_per_sf=MXFP8_BLOCK_SIZE,
).reshape(-1),
requires_grad=False,
)
layer._mxfp8_trtllm_output_size = N
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
assert x.dtype == torch.bfloat16, (
f"FlashInfer TRTLLM MXFP8 requires bfloat16 activations, got {x.dtype}."
)
weight = layer.weight # shuffled [padded N, K]
weight_scale = layer.weight_scale
_, K = weight.shape
output_size = layer._mxfp8_trtllm_output_size
input_shape = x.shape
input_2d = x.view(-1, K)
input_mxfp8, input_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(
input_2d
)
output = vllm_flashinfer.mm_mxfp8(
input_mxfp8,
weight.t(),
input_scale,
weight_scale,
out_dtype=x.dtype,
backend="trtllm",
use_8x4_sf_layout=True,
)
if output.shape[-1] != output_size:
output = output[:, :output_size].contiguous()
if bias is not None:
output = output + bias
output_shape = (*input_shape[:-1], output_size)
return output.view(output_shape)