[Kernel] Add MXFP8 to Marlin GEMM/MoE and refactor Mxfp8LinearOp (#34664)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-04-01 09:41:42 -07:00
committed by GitHub
parent dc0428ebb8
commit db5d0719e1
15 changed files with 481 additions and 129 deletions
@@ -336,6 +336,202 @@ def pack_fp8_to_int32(
return int32_tensor.T.contiguous() if size_k_first else int32_tensor
def mxfp8_marlin_process_scales(marlin_scales: torch.Tensor) -> torch.Tensor:
"""Reorder scales for e8m0 kernel layout and convert to float8_e8m0fnu."""
# fit the layout of fp8 dequantization
marlin_scales = marlin_scales.view(-1, 4)[:, [0, 2, 1, 3]].view(
marlin_scales.size(0), -1
)
marlin_scales = marlin_scales.to(torch.float8_e8m0fnu)
return marlin_scales
def apply_mxfp8_marlin_linear(
input: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
workspace: torch.Tensor,
size_n: int,
size_k: int,
bias: torch.Tensor | None = None,
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
) -> torch.Tensor:
reshaped_x = input.reshape(-1, input.shape[-1])
out_shape = input.shape[:-1] + (size_n,)
use_atomic_add = should_use_atomic_add_reduce(
m=reshaped_x.size(0),
n=size_n,
k=size_k,
device=input.device,
dtype=input.dtype,
)
output = ops.marlin_gemm(
a=reshaped_x,
c=None,
b_q_weight=weight,
b_bias=bias,
b_scales=weight_scale,
a_scales=None,
global_scale=None,
b_zeros=None,
g_idx=None,
perm=None,
workspace=workspace,
b_q_type=scalar_types.float8_e4m3fn,
size_m=reshaped_x.size(0),
size_n=size_n,
size_k=size_k,
use_atomic_add=use_atomic_add,
use_fp32_reduce=use_fp32_reduce,
)
return output.reshape(out_shape)
def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
"""Repack MXFP8 weights and scales into Marlin kernel format.
Expects the layer to have:
- weight: [N, K] float8_e4m3fn
- weight_scale: [N, K//32] uint8 (e8m0 encoded)
- input_size_per_partition / output_size_per_partition
"""
part_size_n = layer.output_size_per_partition
part_size_k = layer.input_size_per_partition
group_size = 32 # MX standard block size
device = layer.weight.device
# WORKSPACE
layer.workspace = marlin_make_workspace_new(device)
# WEIGHT - repack FP8 weights to Marlin format
perm = torch.empty(0, dtype=torch.int, device=device)
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
qweight = qweight.T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=part_size_k,
size_n=part_size_n,
num_bits=8,
)
replace_parameter(layer, "weight", marlin_qweight)
# WEIGHT SCALES
# Convert uint8 scales -> e8m0fnu -> param_dtype for permutation
# Scales are [N, K//32], need [K//32, N] for marlin_permute_scales
param_dtype = torch.get_default_dtype()
scales = layer.weight_scale.data[:part_size_n, : part_size_k // group_size]
scales = scales.contiguous()
scales = scales.view(torch.float8_e8m0fnu).to(param_dtype)
scales = scales.T.contiguous()
# Permute scales to Marlin layout
marlin_scales = marlin_permute_scales(
s=scales,
size_k=part_size_k,
size_n=part_size_n,
group_size=group_size,
)
# Reorder for e8m0 kernel layout and convert back to e8m0fnu
marlin_scales = mxfp8_marlin_process_scales(marlin_scales)
replace_parameter(layer, "weight_scale", marlin_scales)
# BIAS
if hasattr(layer, "bias") and layer.bias is not None:
assert layer.bias.shape == (part_size_n,)
bias = marlin_permute_bias(layer.bias)
replace_parameter(layer, "bias", bias)
def prepare_mxfp8_moe_layer_for_marlin(
layer: torch.nn.Module,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Repack MXFP8 MoE weights and scales into Marlin kernel format.
Args:
layer: MoE layer (used to read params_dtype and attach workspace).
w13: [E, 2*N, K] float8_e4m3fn weights.
w2: [E, K, N] float8_e4m3fn weights.
w13_scale: [E, 2*N, K//32] uint8 e8m0 scales.
w2_scale: [E, K, N//32] uint8 e8m0 scales.
Returns:
(w13, w2, w13_scale, w2_scale) in Marlin format.
"""
group_size = 32
e = w13.shape[0]
w13_n = w13.shape[1]
k = w13.shape[2]
n = w2.shape[2]
device = w13.device
param_dtype = torch.get_default_dtype()
perm = torch.empty(0, dtype=torch.int, device=device)
layer.workspace = marlin_make_workspace_new(device, 4)
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
if "w13" in name:
size_n, size_k = w13_n, k
else:
size_n, size_k = k, n
assert weight.shape == (e, size_n, size_k)
tensor_list = []
for i in range(e):
qweight = pack_fp8_to_int32(weight[i], size_k_first=False)
qweight = qweight.T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=8,
)
tensor_list.append(marlin_qweight)
return torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
w13 = repack_weight(w13, "w13")
w2 = repack_weight(w2, "w2")
def permute_scales(scales: torch.Tensor, name: str) -> torch.Tensor:
if "w13" in name:
size_n, size_k = w13_n, k
else:
size_n, size_k = k, n
tensor_list = []
for i in range(e):
s = scales[i][:size_n, : size_k // group_size].contiguous()
s = s.view(torch.float8_e8m0fnu).to(param_dtype)
s = s.T.contiguous()
marlin_s = marlin_permute_scales(
s=s,
size_k=size_k,
size_n=size_n,
group_size=group_size,
)
marlin_s = mxfp8_marlin_process_scales(marlin_s)
tensor_list.append(marlin_s)
return torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
w13_scale = permute_scales(w13_scale, "w13")
w2_scale = permute_scales(w2_scale, "w2")
return w13, w2, w13_scale, w2_scale
def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
if is_a_8bit:
@@ -4,6 +4,7 @@
from enum import Enum
import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.utils import flashinfer as vllm_flashinfer
@@ -15,6 +16,7 @@ logger = init_logger(__name__)
class Mxfp8LinearBackend(Enum):
EMULATION = "emulation"
FLASHINFER_CUTLASS = "flashinfer-cutlass"
MARLIN = "marlin"
# MXFP8 constants
@@ -23,6 +25,28 @@ MXFP8_SCALE_DTYPE = torch.uint8
MXFP8_BLOCK_SIZE = 32
def select_mxfp8_linear_backend() -> Mxfp8LinearBackend:
"""Select the best MXFP8 linear backend for the current device.
- SM100+ (Blackwell): FLASHINFER_CUTLASS (native MXFP8 W8A8 GEMM)
- SM80+ (Ampere/Ada): MARLIN (MXFP8 W8A16 GEMM)
- Otherwise: EMULATION (dequant to BF16 fallback)
"""
from vllm.platforms import current_platform
if current_platform.has_device_capability(100):
return Mxfp8LinearBackend.FLASHINFER_CUTLASS
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
is_fp8_marlin_supported,
)
if is_fp8_marlin_supported():
return Mxfp8LinearBackend.MARLIN
return Mxfp8LinearBackend.EMULATION
def swizzle_mxfp8_scale(sf: torch.Tensor, M: int, K: int) -> torch.Tensor:
"""Swizzle MXFP8 scales from row-major 2D to F8_128x4 layout."""
scaling_vector_size = MXFP8_BLOCK_SIZE # 32 for MXFP8
@@ -47,17 +71,71 @@ def swizzle_mxfp8_scale(sf: torch.Tensor, M: int, K: int) -> torch.Tensor:
return sf_swizzled.contiguous().view(-1)
def _mxfp8_e4m3_quantize_torch(
x: torch.Tensor,
is_sf_swizzled_layout: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Naive MXFP8 quantization.
For each block of 32 elements along the last dimension, compute a
shared e8m0 scale (the biased exponent of the block-wise amax)
and quantize each element to float8_e4m3fn.
Returns (quantized_values [same shape, fp8], scales uint8).
Scale shape depends on is_sf_swizzled_layout:
False -> [..., K//32] (row-major 2D)
True -> [flat swizzled 1D]
"""
assert x.shape[-1] % MXFP8_BLOCK_SIZE == 0
orig_shape = x.shape
num_blocks = x.shape[-1] // MXFP8_BLOCK_SIZE
x_fp32 = x.to(torch.float32)
x_blocked = x_fp32.view(*orig_shape[:-1], num_blocks, MXFP8_BLOCK_SIZE)
amax = x_blocked.abs().amax(dim=-1)
amax = amax.clamp(min=torch.finfo(torch.float32).tiny)
scale_biased = torch.floor(torch.log2(amax)) + 127.0
scale_biased = scale_biased.clamp(0, 254)
scales_uint8 = scale_biased.to(torch.uint8)
descale = torch.exp2(scale_biased - 127.0)
x_scaled = x_blocked / descale.unsqueeze(-1)
x_fp8 = x_scaled.view(orig_shape).to(MXFP8_VALUE_DTYPE)
if x.ndim == 2:
M, K = x.shape
scales_uint8 = scales_uint8.view(M, -1)
if is_sf_swizzled_layout:
scales_uint8 = swizzle_mxfp8_scale(scales_uint8, M=M, K=K)
elif x.ndim == 3:
B, M, K = x.shape
scales_uint8 = scales_uint8.view(B, M, -1)
if is_sf_swizzled_layout:
swizzled = []
for i in range(B):
swizzled.append(swizzle_mxfp8_scale(scales_uint8[i], M=M, K=K))
scales_uint8 = torch.cat(swizzled)
return x_fp8, scales_uint8
def _mxfp8_e4m3_quantize_impl(
x: torch.Tensor, is_sf_swizzled_layout: bool = False
) -> tuple[torch.Tensor, torch.Tensor]:
from flashinfer import mxfp8_quantize as flashinfer_mxfp8_quantize
from vllm.platforms import current_platform
x_q, x_scales = flashinfer_mxfp8_quantize(
x, is_sf_swizzled_layout=is_sf_swizzled_layout
)
if x_scales.ndim == 1 and x.ndim == 2 and not is_sf_swizzled_layout:
x_scales = x_scales.view(x.size(0), -1)
return x_q, x_scales
if current_platform.has_device_capability(100):
from flashinfer import mxfp8_quantize as flashinfer_mxfp8_quantize
x_q, x_scales = flashinfer_mxfp8_quantize(
x, is_sf_swizzled_layout=is_sf_swizzled_layout
)
if x_scales.ndim == 1 and x.ndim == 2 and not is_sf_swizzled_layout:
x_scales = x_scales.view(x.size(0), -1)
return x_q, x_scales
return _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout)
def mxfp8_e4m3_quantize(
@@ -128,11 +206,51 @@ direct_register_custom_op(
class Mxfp8LinearOp:
def __init__(self, backend: Mxfp8LinearBackend):
if backend not in Mxfp8LinearBackend:
raise ValueError(f"Unsupported backend: {backend}")
def __init__(self):
self.backend = select_mxfp8_linear_backend()
logger.info_once("Using %s backend for MXFP8 GEMM", self.backend)
self.backend = backend
def process_weights(self, layer: torch.nn.Module) -> None:
"""Process MXFP8 weights after loading into backend-specific format."""
if self.backend == Mxfp8LinearBackend.MARLIN:
self._process_weights_marlin(layer)
elif self.backend == Mxfp8LinearBackend.FLASHINFER_CUTLASS:
self._process_weights_flashinfer_cutlass(layer)
else:
self._process_weights_emulation(layer)
def _process_weights_emulation(self, layer: torch.nn.Module) -> None:
"""Keep scales as 2D uint8 for dequant-to-BF16 emulation."""
weight = layer.weight.data # [N, K]
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
def _process_weights_flashinfer_cutlass(self, layer: torch.nn.Module) -> None:
"""Swizzle scales to F8_128x4 layout for flashinfer CUTLASS."""
weight = layer.weight.data # [N, K]
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale_2d = layer.weight_scale.data[:N, :scale_k].contiguous()
weight_scale_swizzled = swizzle_mxfp8_scale(weight_scale_2d, M=N, K=K)
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
layer.weight_scale = Parameter(
weight_scale_swizzled.contiguous(), requires_grad=False
)
def _process_weights_marlin(self, layer: torch.nn.Module) -> None:
"""Repack MXFP8 weights and scales into Marlin kernel format."""
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_mxfp8_layer_for_marlin,
)
prepare_mxfp8_layer_for_marlin(layer)
def _apply_emulation(
self,
@@ -142,7 +260,6 @@ class Mxfp8LinearOp:
out_dtype: torch.dtype,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
# Validate weight_scale dtype and shape (must be 2D for TORCH backend)
if weight_scale.dtype != MXFP8_SCALE_DTYPE:
raise ValueError(
f"TORCH backend requires {MXFP8_SCALE_DTYPE} weight_scale dtype, "
@@ -219,6 +336,32 @@ class Mxfp8LinearOp:
output_shape = (*input_shape[:-1], N)
return output.view(output_shape)
def _apply_marlin(
self,
input: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
bias: torch.Tensor | None = None,
*,
workspace: torch.Tensor,
size_n: int,
size_k: int,
) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
apply_mxfp8_marlin_linear,
)
return apply_mxfp8_marlin_linear(
input=input,
weight=weight,
weight_scale=weight_scale,
workspace=workspace,
size_n=size_n,
size_k=size_k,
bias=bias,
)
def apply(
self,
input: torch.Tensor,
@@ -226,10 +369,27 @@ class Mxfp8LinearOp:
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
bias: torch.Tensor | None = None,
*,
workspace: torch.Tensor | None = None,
size_n: int = 0,
size_k: int = 0,
) -> torch.Tensor:
if self.backend == Mxfp8LinearBackend.EMULATION:
return self._apply_emulation(input, weight, weight_scale, out_dtype, bias)
if self.backend == Mxfp8LinearBackend.MARLIN:
assert workspace is not None
return self._apply_marlin(
input,
weight,
weight_scale,
out_dtype,
bias,
workspace=workspace,
size_n=size_n,
size_k=size_k,
)
assert self.backend == Mxfp8LinearBackend.FLASHINFER_CUTLASS
return self._apply_flashinfer_cutlass(
input, weight, weight_scale, out_dtype, bias