[Kernel] Add FlashInfer TRTLLM MXFP8 linear backend (#52204)
Signed-off-by: seonjinn <sna@nvidia.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -33,6 +33,11 @@ following `quantization.quant_algo` values:
|
||||
[Engine Arguments](../../configuration/engine_args.md) page and shown by
|
||||
`vllm serve --help=KernelConfig`.
|
||||
|
||||
!!! note
|
||||
For models quantized to MXFP8 with BF16 activations on SM100-family GPUs,
|
||||
use `--linear-backend flashinfer_trtllm` to select FlashInfer's TensorRT-LLM
|
||||
GEMM backend.
|
||||
|
||||
## Quantizing HuggingFace Models with PTQ
|
||||
|
||||
You can quantize HuggingFace models using the example scripts provided in the Model Optimizer repository. The primary script for LLM PTQ is typically found within the `examples/llm_ptq` directory.
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import flashinfer as vllm_flashinfer
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
if not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and has_flashinfer()
|
||||
):
|
||||
pytest.skip(
|
||||
reason="FlashInfer TRTLLM MXFP8 requires an SM100-family GPU",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _make_layer(weight: torch.Tensor) -> torch.nn.Module:
|
||||
from flashinfer import SfLayout, mxfp8_quantize
|
||||
|
||||
weight_mxfp8, weight_scale = mxfp8_quantize(
|
||||
weight,
|
||||
sf_swizzle_layout=SfLayout.layout_linear,
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(weight_mxfp8, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
weight_scale.view(weight.shape[0], weight.shape[1] // 32),
|
||||
requires_grad=False,
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 130, 256), (7, 256, 512), (128, 130, 768)])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxfp8_linear_numerics(
|
||||
shape: tuple[int, int, int],
|
||||
) -> None:
|
||||
torch.manual_seed(0)
|
||||
m, n, k = shape
|
||||
x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda")
|
||||
layer = _make_layer(weight)
|
||||
kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
output = kernel.apply_weights(layer, x)
|
||||
reference = torch.mm(x, weight.t())
|
||||
similarity = F.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
)
|
||||
|
||||
assert output.shape == (m, n)
|
||||
assert output.is_contiguous()
|
||||
assert similarity.item() > 0.98
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxfp8_custom_ops() -> None:
|
||||
x = torch.randn((7, 512), dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn((256, 512), dtype=torch.bfloat16, device="cuda")
|
||||
layer = _make_layer(weight)
|
||||
kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.flashinfer_mxfp8_quantize_8x4.default,
|
||||
(x,),
|
||||
)
|
||||
x_mxfp8, x_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(x)
|
||||
# SchemaCheckMode compares inputs with allclose, which CUDA does not
|
||||
# implement for float8. The numerical tests above guard input mutation.
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.mm_mxfp8.default,
|
||||
(
|
||||
x_mxfp8,
|
||||
layer.weight.t(),
|
||||
x_scale,
|
||||
layer.weight_scale,
|
||||
torch.bfloat16,
|
||||
"trtllm",
|
||||
True,
|
||||
),
|
||||
test_utils=(
|
||||
"test_autograd_registration",
|
||||
"test_faketensor",
|
||||
"test_aot_dispatch_dynamic",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxfp8_linear_cuda_graph() -> None:
|
||||
torch.manual_seed(0)
|
||||
m, n, k = 7, 130, 512
|
||||
weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda")
|
||||
layer = _make_layer(weight)
|
||||
kernel = FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearLayerConfig())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
static_x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda")
|
||||
kernel.apply_weights(layer, static_x)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_output = kernel.apply_weights(layer, static_x)
|
||||
|
||||
new_x = torch.randn_like(static_x)
|
||||
static_x.copy_(new_x)
|
||||
graph.replay()
|
||||
eager_output = kernel.apply_weights(layer, new_x)
|
||||
|
||||
torch.testing.assert_close(graph_output, eager_output, rtol=0, atol=0)
|
||||
@@ -115,6 +115,7 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
FlashInferCutlassMxfp8LinearKernel,
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.humming import (
|
||||
HummingMxfp8LinearKernel,
|
||||
@@ -267,6 +268,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
},
|
||||
"flashinfer_trtllm": {
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
FlashInferTrtllmNvFp4LinearKernel,
|
||||
},
|
||||
"flashinfer_cudnn": {
|
||||
@@ -507,6 +509,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
B12xMxfp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
HummingMxfp8LinearKernel,
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
|
||||
@@ -1216,6 +1219,7 @@ __all__ = [
|
||||
"MarlinMxFp4LinearKernel",
|
||||
"FlashInferCutedslMxfp8LinearKernel",
|
||||
"FlashInferCutlassMxfp8LinearKernel",
|
||||
"FlashInferTrtllmMxfp8LinearKernel",
|
||||
"MarlinMxfp8LinearKernel",
|
||||
"XPUMxFp8LinearKernel",
|
||||
"EmulationMxfp8LinearKernel",
|
||||
|
||||
@@ -177,3 +177,103 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
|
||||
output_shape = (*input_shape[:-1], N)
|
||||
return output.view(output_shape)
|
||||
|
||||
|
||||
class FlashInferTrtllmMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM via FlashInfer's TensorRT-LLM wrapper."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return False, "requires SM100-family GPU"
|
||||
if not has_flashinfer():
|
||||
return False, "requires FlashInfer"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a
|
||||
|
||||
if hasattr(layer, "_mxfp8_trtllm_output_size") and layer.weight_scale.ndim == 1:
|
||||
return
|
||||
|
||||
weight = layer.weight.data # [N, K]
|
||||
N, K = weight.shape
|
||||
if K % 256 != 0:
|
||||
raise ValueError(
|
||||
f"FlashInfer TRTLLM MXFP8 requires K to be divisible by 256, got K={K}."
|
||||
)
|
||||
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
padded_n = ((N + 127) // 128) * 128
|
||||
if padded_n != N:
|
||||
padded_weight = weight.new_zeros((padded_n, K))
|
||||
padded_weight[:N] = weight
|
||||
weight = padded_weight
|
||||
|
||||
padded_scale = weight_scale.new_zeros((padded_n, scale_k))
|
||||
padded_scale[:N] = weight_scale
|
||||
weight_scale = padded_scale
|
||||
else:
|
||||
weight = weight.contiguous()
|
||||
|
||||
layer.weight = Parameter(
|
||||
shuffle_matrix_a(weight, 128).reshape(padded_n, K),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = Parameter(
|
||||
shuffle_matrix_sf_a(
|
||||
weight_scale,
|
||||
128,
|
||||
num_elts_per_sf=MXFP8_BLOCK_SIZE,
|
||||
).reshape(-1),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer._mxfp8_trtllm_output_size = N
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert x.dtype == torch.bfloat16, (
|
||||
f"FlashInfer TRTLLM MXFP8 requires bfloat16 activations, got {x.dtype}."
|
||||
)
|
||||
|
||||
weight = layer.weight # shuffled [padded N, K]
|
||||
weight_scale = layer.weight_scale
|
||||
_, K = weight.shape
|
||||
output_size = layer._mxfp8_trtllm_output_size
|
||||
input_shape = x.shape
|
||||
input_2d = x.view(-1, K)
|
||||
|
||||
input_mxfp8, input_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(
|
||||
input_2d
|
||||
)
|
||||
output = vllm_flashinfer.mm_mxfp8(
|
||||
input_mxfp8,
|
||||
weight.t(),
|
||||
input_scale,
|
||||
weight_scale,
|
||||
out_dtype=x.dtype,
|
||||
backend="trtllm",
|
||||
use_8x4_sf_layout=True,
|
||||
)
|
||||
if output.shape[-1] != output_size:
|
||||
output = output[:, :output_size].contiguous()
|
||||
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
|
||||
output_shape = (*input_shape[:-1], output_size)
|
||||
return output.view(output_shape)
|
||||
|
||||
@@ -726,6 +726,36 @@ if has_flashinfer():
|
||||
rounded_m, rounded_n, dtype=torch.uint8, device=a.device
|
||||
)
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::flashinfer_mxfp8_quantize_8x4",
|
||||
mutates_args=[],
|
||||
device_types="cuda",
|
||||
)
|
||||
def flashinfer_mxfp8_quantize_8x4(
|
||||
a: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from flashinfer import SfLayout
|
||||
from flashinfer import mxfp8_quantize as mxfp8_quantize_
|
||||
|
||||
return mxfp8_quantize_(
|
||||
a,
|
||||
backend="cuda",
|
||||
sf_swizzle_layout=SfLayout.layout_8x4,
|
||||
)
|
||||
|
||||
@torch.library.register_fake(
|
||||
"vllm::flashinfer_mxfp8_quantize_8x4",
|
||||
)
|
||||
def flashinfer_mxfp8_quantize_8x4_fake(
|
||||
a: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
m, k = a.shape
|
||||
scale_size = cdiv(m, 8) * 8 * cdiv(k // 32, 4) * 4
|
||||
return (
|
||||
torch.empty(m, k, dtype=torch.float8_e4m3fn, device=a.device),
|
||||
torch.empty(scale_size, dtype=torch.uint8, device=a.device),
|
||||
)
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::mm_mxfp8",
|
||||
mutates_args=[],
|
||||
@@ -738,6 +768,7 @@ if has_flashinfer():
|
||||
B_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str = "cutlass",
|
||||
use_8x4_sf_layout: bool = False,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import mm_mxfp8 as mm_mxfp8_
|
||||
|
||||
@@ -749,6 +780,7 @@ if has_flashinfer():
|
||||
out=None,
|
||||
out_dtype=out_dtype,
|
||||
backend=backend,
|
||||
use_8x4_sf_layout=use_8x4_sf_layout,
|
||||
)
|
||||
|
||||
@torch.library.register_fake(
|
||||
@@ -761,6 +793,7 @@ if has_flashinfer():
|
||||
B_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
backend: str = "cutlass",
|
||||
use_8x4_sf_layout: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# A is [m, k], B is [k, n] -> output [m, n]
|
||||
return torch.empty(A.shape[0], B.shape[1], dtype=out_dtype, device=A.device)
|
||||
|
||||
Reference in New Issue
Block a user