[Kernel] Add FlashInfer TRTLLM MXFP8 linear backend (#52204)

Signed-off-by: seonjinn <sna@nvidia.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Seonjin
2026-08-20 10:07:22 -04:00
committed by GitHub
co-authored by Misha Goin
parent de216b6e64
commit bd8865a299
5 changed files with 265 additions and 0 deletions
+33
View File
@@ -726,6 +726,36 @@ if has_flashinfer():
rounded_m, rounded_n, dtype=torch.uint8, device=a.device
)
@torch.library.custom_op(
"vllm::flashinfer_mxfp8_quantize_8x4",
mutates_args=[],
device_types="cuda",
)
def flashinfer_mxfp8_quantize_8x4(
a: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
from flashinfer import SfLayout
from flashinfer import mxfp8_quantize as mxfp8_quantize_
return mxfp8_quantize_(
a,
backend="cuda",
sf_swizzle_layout=SfLayout.layout_8x4,
)
@torch.library.register_fake(
"vllm::flashinfer_mxfp8_quantize_8x4",
)
def flashinfer_mxfp8_quantize_8x4_fake(
a: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
m, k = a.shape
scale_size = cdiv(m, 8) * 8 * cdiv(k // 32, 4) * 4
return (
torch.empty(m, k, dtype=torch.float8_e4m3fn, device=a.device),
torch.empty(scale_size, dtype=torch.uint8, device=a.device),
)
@torch.library.custom_op(
"vllm::mm_mxfp8",
mutates_args=[],
@@ -738,6 +768,7 @@ if has_flashinfer():
B_scale: torch.Tensor,
out_dtype: torch.dtype,
backend: str = "cutlass",
use_8x4_sf_layout: bool = False,
) -> torch.Tensor:
from flashinfer import mm_mxfp8 as mm_mxfp8_
@@ -749,6 +780,7 @@ if has_flashinfer():
out=None,
out_dtype=out_dtype,
backend=backend,
use_8x4_sf_layout=use_8x4_sf_layout,
)
@torch.library.register_fake(
@@ -761,6 +793,7 @@ if has_flashinfer():
B_scale: torch.Tensor,
out_dtype: torch.dtype,
backend: str = "cutlass",
use_8x4_sf_layout: bool = False,
) -> torch.Tensor:
# A is [m, k], B is [k, n] -> output [m, n]
return torch.empty(A.shape[0], B.shape[1], dtype=out_dtype, device=A.device)