[Kernel] Add FlashInfer TRTLLM MXFP8 linear backend (#52204)
Signed-off-by: seonjinn <sna@nvidia.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -726,6 +726,36 @@ if has_flashinfer():
|
||||
rounded_m, rounded_n, dtype=torch.uint8, device=a.device
|
||||
)
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::flashinfer_mxfp8_quantize_8x4",
|
||||
mutates_args=[],
|
||||
device_types="cuda",
|
||||
)
|
||||
def flashinfer_mxfp8_quantize_8x4(
|
||||
a: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from flashinfer import SfLayout
|
||||
from flashinfer import mxfp8_quantize as mxfp8_quantize_
|
||||
|
||||
return mxfp8_quantize_(
|
||||
a,
|
||||
backend="cuda",
|
||||
sf_swizzle_layout=SfLayout.layout_8x4,
|
||||
)
|
||||
|
||||
@torch.library.register_fake(
|
||||
"vllm::flashinfer_mxfp8_quantize_8x4",
|
||||
)
|
||||
def flashinfer_mxfp8_quantize_8x4_fake(
|
||||
a: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
m, k = a.shape
|
||||
scale_size = cdiv(m, 8) * 8 * cdiv(k // 32, 4) * 4
|
||||
return (
|
||||
torch.empty(m, k, dtype=torch.float8_e4m3fn, device=a.device),
|
||||
torch.empty(scale_size, dtype=torch.uint8, device=a.device),
|
||||
)
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::mm_mxfp8",
|
||||
mutates_args=[],
|
||||
@@ -738,6 +768,7 @@ if has_flashinfer():
|
||||
B_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str = "cutlass",
|
||||
use_8x4_sf_layout: bool = False,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import mm_mxfp8 as mm_mxfp8_
|
||||
|
||||
@@ -749,6 +780,7 @@ if has_flashinfer():
|
||||
out=None,
|
||||
out_dtype=out_dtype,
|
||||
backend=backend,
|
||||
use_8x4_sf_layout=use_8x4_sf_layout,
|
||||
)
|
||||
|
||||
@torch.library.register_fake(
|
||||
@@ -761,6 +793,7 @@ if has_flashinfer():
|
||||
B_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str = "cutlass",
|
||||
use_8x4_sf_layout: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# A is [m, k], B is [k, n] -> output [m, n]
|
||||
return torch.empty(A.shape[0], B.shape[1], dtype=out_dtype, device=A.device)
|
||||
|
||||
Reference in New Issue
Block a user