[Kernel] Add MXFP8 to Marlin GEMM/MoE and refactor Mxfp8LinearOp (#34664)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -336,6 +336,202 @@ def pack_fp8_to_int32(
|
||||
return int32_tensor.T.contiguous() if size_k_first else int32_tensor
|
||||
|
||||
|
||||
def mxfp8_marlin_process_scales(marlin_scales: torch.Tensor) -> torch.Tensor:
|
||||
"""Reorder scales for e8m0 kernel layout and convert to float8_e8m0fnu."""
|
||||
# fit the layout of fp8 dequantization
|
||||
marlin_scales = marlin_scales.view(-1, 4)[:, [0, 2, 1, 3]].view(
|
||||
marlin_scales.size(0), -1
|
||||
)
|
||||
marlin_scales = marlin_scales.to(torch.float8_e8m0fnu)
|
||||
return marlin_scales
|
||||
|
||||
|
||||
def apply_mxfp8_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: torch.Tensor | None = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = input.reshape(-1, input.shape[-1])
|
||||
out_shape = input.shape[:-1] + (size_n,)
|
||||
|
||||
use_atomic_add = should_use_atomic_add_reduce(
|
||||
m=reshaped_x.size(0),
|
||||
n=size_n,
|
||||
k=size_k,
|
||||
device=input.device,
|
||||
dtype=input.dtype,
|
||||
)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
a=reshaped_x,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
a_scales=None,
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
)
|
||||
|
||||
return output.reshape(out_shape)
|
||||
|
||||
|
||||
def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
"""Repack MXFP8 weights and scales into Marlin kernel format.
|
||||
|
||||
Expects the layer to have:
|
||||
- weight: [N, K] float8_e4m3fn
|
||||
- weight_scale: [N, K//32] uint8 (e8m0 encoded)
|
||||
- input_size_per_partition / output_size_per_partition
|
||||
"""
|
||||
part_size_n = layer.output_size_per_partition
|
||||
part_size_k = layer.input_size_per_partition
|
||||
group_size = 32 # MX standard block size
|
||||
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device)
|
||||
|
||||
# WEIGHT - repack FP8 weights to Marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
|
||||
qweight = qweight.T.contiguous()
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n,
|
||||
num_bits=8,
|
||||
)
|
||||
replace_parameter(layer, "weight", marlin_qweight)
|
||||
|
||||
# WEIGHT SCALES
|
||||
# Convert uint8 scales -> e8m0fnu -> param_dtype for permutation
|
||||
# Scales are [N, K//32], need [K//32, N] for marlin_permute_scales
|
||||
param_dtype = torch.get_default_dtype()
|
||||
scales = layer.weight_scale.data[:part_size_n, : part_size_k // group_size]
|
||||
scales = scales.contiguous()
|
||||
scales = scales.view(torch.float8_e8m0fnu).to(param_dtype)
|
||||
scales = scales.T.contiguous()
|
||||
|
||||
# Permute scales to Marlin layout
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scales,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n,
|
||||
group_size=group_size,
|
||||
)
|
||||
|
||||
# Reorder for e8m0 kernel layout and convert back to e8m0fnu
|
||||
marlin_scales = mxfp8_marlin_process_scales(marlin_scales)
|
||||
replace_parameter(layer, "weight_scale", marlin_scales)
|
||||
|
||||
# BIAS
|
||||
if hasattr(layer, "bias") and layer.bias is not None:
|
||||
assert layer.bias.shape == (part_size_n,)
|
||||
bias = marlin_permute_bias(layer.bias)
|
||||
replace_parameter(layer, "bias", bias)
|
||||
|
||||
|
||||
def prepare_mxfp8_moe_layer_for_marlin(
|
||||
layer: torch.nn.Module,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Repack MXFP8 MoE weights and scales into Marlin kernel format.
|
||||
|
||||
Args:
|
||||
layer: MoE layer (used to read params_dtype and attach workspace).
|
||||
w13: [E, 2*N, K] float8_e4m3fn weights.
|
||||
w2: [E, K, N] float8_e4m3fn weights.
|
||||
w13_scale: [E, 2*N, K//32] uint8 e8m0 scales.
|
||||
w2_scale: [E, K, N//32] uint8 e8m0 scales.
|
||||
|
||||
Returns:
|
||||
(w13, w2, w13_scale, w2_scale) in Marlin format.
|
||||
"""
|
||||
group_size = 32
|
||||
e = w13.shape[0]
|
||||
w13_n = w13.shape[1]
|
||||
k = w13.shape[2]
|
||||
n = w2.shape[2]
|
||||
|
||||
device = w13.device
|
||||
param_dtype = torch.get_default_dtype()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
if "w13" in name:
|
||||
size_n, size_k = w13_n, k
|
||||
else:
|
||||
size_n, size_k = k, n
|
||||
|
||||
assert weight.shape == (e, size_n, size_k)
|
||||
|
||||
tensor_list = []
|
||||
for i in range(e):
|
||||
qweight = pack_fp8_to_int32(weight[i], size_k_first=False)
|
||||
qweight = qweight.T.contiguous()
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=8,
|
||||
)
|
||||
tensor_list.append(marlin_qweight)
|
||||
return torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
|
||||
w13 = repack_weight(w13, "w13")
|
||||
w2 = repack_weight(w2, "w2")
|
||||
|
||||
def permute_scales(scales: torch.Tensor, name: str) -> torch.Tensor:
|
||||
if "w13" in name:
|
||||
size_n, size_k = w13_n, k
|
||||
else:
|
||||
size_n, size_k = k, n
|
||||
|
||||
tensor_list = []
|
||||
for i in range(e):
|
||||
s = scales[i][:size_n, : size_k // group_size].contiguous()
|
||||
s = s.view(torch.float8_e8m0fnu).to(param_dtype)
|
||||
s = s.T.contiguous()
|
||||
marlin_s = marlin_permute_scales(
|
||||
s=s,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
group_size=group_size,
|
||||
)
|
||||
marlin_s = mxfp8_marlin_process_scales(marlin_s)
|
||||
tensor_list.append(marlin_s)
|
||||
return torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
|
||||
w13_scale = permute_scales(w13_scale, "w13")
|
||||
w2_scale = permute_scales(w2_scale, "w2")
|
||||
|
||||
return w13, w2, w13_scale, w2_scale
|
||||
|
||||
|
||||
def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
if is_a_8bit:
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
from enum import Enum
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import flashinfer as vllm_flashinfer
|
||||
@@ -15,6 +16,7 @@ logger = init_logger(__name__)
|
||||
class Mxfp8LinearBackend(Enum):
|
||||
EMULATION = "emulation"
|
||||
FLASHINFER_CUTLASS = "flashinfer-cutlass"
|
||||
MARLIN = "marlin"
|
||||
|
||||
|
||||
# MXFP8 constants
|
||||
@@ -23,6 +25,28 @@ MXFP8_SCALE_DTYPE = torch.uint8
|
||||
MXFP8_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
def select_mxfp8_linear_backend() -> Mxfp8LinearBackend:
|
||||
"""Select the best MXFP8 linear backend for the current device.
|
||||
|
||||
- SM100+ (Blackwell): FLASHINFER_CUTLASS (native MXFP8 W8A8 GEMM)
|
||||
- SM80+ (Ampere/Ada): MARLIN (MXFP8 W8A16 GEMM)
|
||||
- Otherwise: EMULATION (dequant to BF16 fallback)
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.has_device_capability(100):
|
||||
return Mxfp8LinearBackend.FLASHINFER_CUTLASS
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
is_fp8_marlin_supported,
|
||||
)
|
||||
|
||||
if is_fp8_marlin_supported():
|
||||
return Mxfp8LinearBackend.MARLIN
|
||||
|
||||
return Mxfp8LinearBackend.EMULATION
|
||||
|
||||
|
||||
def swizzle_mxfp8_scale(sf: torch.Tensor, M: int, K: int) -> torch.Tensor:
|
||||
"""Swizzle MXFP8 scales from row-major 2D to F8_128x4 layout."""
|
||||
scaling_vector_size = MXFP8_BLOCK_SIZE # 32 for MXFP8
|
||||
@@ -47,17 +71,71 @@ def swizzle_mxfp8_scale(sf: torch.Tensor, M: int, K: int) -> torch.Tensor:
|
||||
return sf_swizzled.contiguous().view(-1)
|
||||
|
||||
|
||||
def _mxfp8_e4m3_quantize_torch(
|
||||
x: torch.Tensor,
|
||||
is_sf_swizzled_layout: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Naive MXFP8 quantization.
|
||||
For each block of 32 elements along the last dimension, compute a
|
||||
shared e8m0 scale (the biased exponent of the block-wise amax)
|
||||
and quantize each element to float8_e4m3fn.
|
||||
|
||||
Returns (quantized_values [same shape, fp8], scales uint8).
|
||||
Scale shape depends on is_sf_swizzled_layout:
|
||||
False -> [..., K//32] (row-major 2D)
|
||||
True -> [flat swizzled 1D]
|
||||
"""
|
||||
assert x.shape[-1] % MXFP8_BLOCK_SIZE == 0
|
||||
orig_shape = x.shape
|
||||
num_blocks = x.shape[-1] // MXFP8_BLOCK_SIZE
|
||||
|
||||
x_fp32 = x.to(torch.float32)
|
||||
x_blocked = x_fp32.view(*orig_shape[:-1], num_blocks, MXFP8_BLOCK_SIZE)
|
||||
|
||||
amax = x_blocked.abs().amax(dim=-1)
|
||||
amax = amax.clamp(min=torch.finfo(torch.float32).tiny)
|
||||
scale_biased = torch.floor(torch.log2(amax)) + 127.0
|
||||
scale_biased = scale_biased.clamp(0, 254)
|
||||
scales_uint8 = scale_biased.to(torch.uint8)
|
||||
|
||||
descale = torch.exp2(scale_biased - 127.0)
|
||||
x_scaled = x_blocked / descale.unsqueeze(-1)
|
||||
|
||||
x_fp8 = x_scaled.view(orig_shape).to(MXFP8_VALUE_DTYPE)
|
||||
|
||||
if x.ndim == 2:
|
||||
M, K = x.shape
|
||||
scales_uint8 = scales_uint8.view(M, -1)
|
||||
if is_sf_swizzled_layout:
|
||||
scales_uint8 = swizzle_mxfp8_scale(scales_uint8, M=M, K=K)
|
||||
elif x.ndim == 3:
|
||||
B, M, K = x.shape
|
||||
scales_uint8 = scales_uint8.view(B, M, -1)
|
||||
if is_sf_swizzled_layout:
|
||||
swizzled = []
|
||||
for i in range(B):
|
||||
swizzled.append(swizzle_mxfp8_scale(scales_uint8[i], M=M, K=K))
|
||||
scales_uint8 = torch.cat(swizzled)
|
||||
|
||||
return x_fp8, scales_uint8
|
||||
|
||||
|
||||
def _mxfp8_e4m3_quantize_impl(
|
||||
x: torch.Tensor, is_sf_swizzled_layout: bool = False
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from flashinfer import mxfp8_quantize as flashinfer_mxfp8_quantize
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
x_q, x_scales = flashinfer_mxfp8_quantize(
|
||||
x, is_sf_swizzled_layout=is_sf_swizzled_layout
|
||||
)
|
||||
if x_scales.ndim == 1 and x.ndim == 2 and not is_sf_swizzled_layout:
|
||||
x_scales = x_scales.view(x.size(0), -1)
|
||||
return x_q, x_scales
|
||||
if current_platform.has_device_capability(100):
|
||||
from flashinfer import mxfp8_quantize as flashinfer_mxfp8_quantize
|
||||
|
||||
x_q, x_scales = flashinfer_mxfp8_quantize(
|
||||
x, is_sf_swizzled_layout=is_sf_swizzled_layout
|
||||
)
|
||||
if x_scales.ndim == 1 and x.ndim == 2 and not is_sf_swizzled_layout:
|
||||
x_scales = x_scales.view(x.size(0), -1)
|
||||
return x_q, x_scales
|
||||
|
||||
return _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout)
|
||||
|
||||
|
||||
def mxfp8_e4m3_quantize(
|
||||
@@ -128,11 +206,51 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class Mxfp8LinearOp:
|
||||
def __init__(self, backend: Mxfp8LinearBackend):
|
||||
if backend not in Mxfp8LinearBackend:
|
||||
raise ValueError(f"Unsupported backend: {backend}")
|
||||
def __init__(self):
|
||||
self.backend = select_mxfp8_linear_backend()
|
||||
logger.info_once("Using %s backend for MXFP8 GEMM", self.backend)
|
||||
|
||||
self.backend = backend
|
||||
def process_weights(self, layer: torch.nn.Module) -> None:
|
||||
"""Process MXFP8 weights after loading into backend-specific format."""
|
||||
if self.backend == Mxfp8LinearBackend.MARLIN:
|
||||
self._process_weights_marlin(layer)
|
||||
elif self.backend == Mxfp8LinearBackend.FLASHINFER_CUTLASS:
|
||||
self._process_weights_flashinfer_cutlass(layer)
|
||||
else:
|
||||
self._process_weights_emulation(layer)
|
||||
|
||||
def _process_weights_emulation(self, layer: torch.nn.Module) -> None:
|
||||
"""Keep scales as 2D uint8 for dequant-to-BF16 emulation."""
|
||||
weight = layer.weight.data # [N, K]
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
|
||||
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
def _process_weights_flashinfer_cutlass(self, layer: torch.nn.Module) -> None:
|
||||
"""Swizzle scales to F8_128x4 layout for flashinfer CUTLASS."""
|
||||
weight = layer.weight.data # [N, K]
|
||||
N, K = weight.shape
|
||||
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale_2d = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
weight_scale_swizzled = swizzle_mxfp8_scale(weight_scale_2d, M=N, K=K)
|
||||
|
||||
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(
|
||||
weight_scale_swizzled.contiguous(), requires_grad=False
|
||||
)
|
||||
|
||||
def _process_weights_marlin(self, layer: torch.nn.Module) -> None:
|
||||
"""Repack MXFP8 weights and scales into Marlin kernel format."""
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_mxfp8_layer_for_marlin,
|
||||
)
|
||||
|
||||
prepare_mxfp8_layer_for_marlin(layer)
|
||||
|
||||
def _apply_emulation(
|
||||
self,
|
||||
@@ -142,7 +260,6 @@ class Mxfp8LinearOp:
|
||||
out_dtype: torch.dtype,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Validate weight_scale dtype and shape (must be 2D for TORCH backend)
|
||||
if weight_scale.dtype != MXFP8_SCALE_DTYPE:
|
||||
raise ValueError(
|
||||
f"TORCH backend requires {MXFP8_SCALE_DTYPE} weight_scale dtype, "
|
||||
@@ -219,6 +336,32 @@ class Mxfp8LinearOp:
|
||||
output_shape = (*input_shape[:-1], N)
|
||||
return output.view(output_shape)
|
||||
|
||||
def _apply_marlin(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
bias: torch.Tensor | None = None,
|
||||
*,
|
||||
workspace: torch.Tensor,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
apply_mxfp8_marlin_linear,
|
||||
)
|
||||
|
||||
return apply_mxfp8_marlin_linear(
|
||||
input=input,
|
||||
weight=weight,
|
||||
weight_scale=weight_scale,
|
||||
workspace=workspace,
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
bias=bias,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
@@ -226,10 +369,27 @@ class Mxfp8LinearOp:
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
bias: torch.Tensor | None = None,
|
||||
*,
|
||||
workspace: torch.Tensor | None = None,
|
||||
size_n: int = 0,
|
||||
size_k: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if self.backend == Mxfp8LinearBackend.EMULATION:
|
||||
return self._apply_emulation(input, weight, weight_scale, out_dtype, bias)
|
||||
|
||||
if self.backend == Mxfp8LinearBackend.MARLIN:
|
||||
assert workspace is not None
|
||||
return self._apply_marlin(
|
||||
input,
|
||||
weight,
|
||||
weight_scale,
|
||||
out_dtype,
|
||||
bias,
|
||||
workspace=workspace,
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
)
|
||||
|
||||
assert self.backend == Mxfp8LinearBackend.FLASHINFER_CUTLASS
|
||||
return self._apply_flashinfer_cutlass(
|
||||
input, weight, weight_scale, out_dtype, bias
|
||||
|
||||
Reference in New Issue
Block a user