[Kernel] Add FlashInfer TRTLLM MXFP8 linear backend (#52204)
Signed-off-by: seonjinn <sna@nvidia.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -115,6 +115,7 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
FlashInferCutlassMxfp8LinearKernel,
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.humming import (
|
||||
HummingMxfp8LinearKernel,
|
||||
@@ -267,6 +268,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
},
|
||||
"flashinfer_trtllm": {
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
FlashInferTrtllmNvFp4LinearKernel,
|
||||
},
|
||||
"flashinfer_cudnn": {
|
||||
@@ -507,6 +509,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
B12xMxfp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
HummingMxfp8LinearKernel,
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
|
||||
@@ -1216,6 +1219,7 @@ __all__ = [
|
||||
"MarlinMxFp4LinearKernel",
|
||||
"FlashInferCutedslMxfp8LinearKernel",
|
||||
"FlashInferCutlassMxfp8LinearKernel",
|
||||
"FlashInferTrtllmMxfp8LinearKernel",
|
||||
"MarlinMxfp8LinearKernel",
|
||||
"XPUMxFp8LinearKernel",
|
||||
"EmulationMxfp8LinearKernel",
|
||||
|
||||
@@ -177,3 +177,103 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
|
||||
output_shape = (*input_shape[:-1], N)
|
||||
return output.view(output_shape)
|
||||
|
||||
|
||||
class FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM via FlashInfer's TensorRT-LLM wrapper."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return False, "requires SM100-family GPU"
|
||||
if not has_flashinfer():
|
||||
return False, "requires FlashInfer"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a
|
||||
|
||||
if hasattr(layer, "_mxfp8_trtllm_output_size") and layer.weight_scale.ndim == 1:
|
||||
return
|
||||
|
||||
weight = layer.weight.data # [N, K]
|
||||
N, K = weight.shape
|
||||
if K % 256 != 0:
|
||||
raise ValueError(
|
||||
f"FlashInfer TRTLLM MXFP8 requires K to be divisible by 256, got K={K}."
|
||||
)
|
||||
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
padded_n = ((N + 127) // 128) * 128
|
||||
if padded_n != N:
|
||||
padded_weight = weight.new_zeros((padded_n, K))
|
||||
padded_weight[:N] = weight
|
||||
weight = padded_weight
|
||||
|
||||
padded_scale = weight_scale.new_zeros((padded_n, scale_k))
|
||||
padded_scale[:N] = weight_scale
|
||||
weight_scale = padded_scale
|
||||
else:
|
||||
weight = weight.contiguous()
|
||||
|
||||
layer.weight = Parameter(
|
||||
shuffle_matrix_a(weight, 128).reshape(padded_n, K),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = Parameter(
|
||||
shuffle_matrix_sf_a(
|
||||
weight_scale,
|
||||
128,
|
||||
num_elts_per_sf=MXFP8_BLOCK_SIZE,
|
||||
).reshape(-1),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer._mxfp8_trtllm_output_size = N
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert x.dtype == torch.bfloat16, (
|
||||
f"FlashInfer TRTLLM MXFP8 requires bfloat16 activations, got {x.dtype}."
|
||||
)
|
||||
|
||||
weight = layer.weight # shuffled [padded N, K]
|
||||
weight_scale = layer.weight_scale
|
||||
_, K = weight.shape
|
||||
output_size = layer._mxfp8_trtllm_output_size
|
||||
input_shape = x.shape
|
||||
input_2d = x.view(-1, K)
|
||||
|
||||
input_mxfp8, input_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(
|
||||
input_2d
|
||||
)
|
||||
output = vllm_flashinfer.mm_mxfp8(
|
||||
input_mxfp8,
|
||||
weight.t(),
|
||||
input_scale,
|
||||
weight_scale,
|
||||
out_dtype=x.dtype,
|
||||
backend="trtllm",
|
||||
use_8x4_sf_layout=True,
|
||||
)
|
||||
if output.shape[-1] != output_size:
|
||||
output = output[:, :output_size].contiguous()
|
||||
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
|
||||
output_shape = (*input_shape[:-1], output_size)
|
||||
return output.view(output_shape)
|
||||
|
||||
Reference in New Issue
Block a user