[Kernel] Extend Marlin thread-tile padding to MoE (WNA16 + FP8/MXFP8) (#45703)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-23 11:45:10 -07:00
committed by GitHub
co-authored by Claude
parent d8e422ccda
commit 37a682d392
7 changed files with 573 additions and 29 deletions
@@ -330,12 +330,43 @@ def check_marlin_supports_layer(
)[0]
def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bool:
def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1) -> int:
"""Smallest MoE intermediate size satisfying the Marlin MoE thread tiles.
The kernel needs gate-up ``2 * intermediate % 128 == 0`` and down
``intermediate % 64 == 0``, i.e. ``intermediate % 64 == 0``. A misaligned
size is zero-padded to the next valid tile at weight prep, kept a multiple
of ``group_size`` so the group count stays integral. The padded region never
reaches the MoE output: w13's padded output channels are zeroed by the
zero-padded scales, so the padded inputs to w2 are zero.
"""
group = group_size if group_size > 0 else 1
padded = round_up(intermediate_size, math.lcm(64, group))
if padded != intermediate_size:
logger.warning_once(
"Marlin requires thread-tile padding for the MoE intermediate size "
"of some layers in this model. Padded experts pad/slice activations "
"on every forward; performance may be degraded."
)
return padded
def check_moe_marlin_supports_layer(
layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False
) -> bool:
"""Whether the fused MoE Marlin kernel supports ``layer``.
Callers without act-order may pass ``allow_tile_padding=True``: a
tile-misaligned intermediate size is then zero-padded to a valid thread
tile at weight prep (see marlin_moe_padded_intermediate), so only a group
straddling the padded boundary stays unsupported. hidden_size is the MoE
I/O extent and is never padded. Act-order keeps the strict shape.
"""
if current_platform.is_rocm():
return False
hidden_size = layer.hidden_size
# Note: The layer has not performed rounding on intermediate_size's at this
# point. Use the unpadded size which won't change.
# The layer has not rounded intermediate_size yet; use the stable unpadded
# size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64.
intermediate_size_per_partition = (
layer.moe_config.intermediate_size_per_partition_unpadded
)
@@ -343,13 +374,15 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
# apply_router_weight_on_input is not supported for moe marlin
supports_router_weight = not layer.apply_router_weight_on_input
# gate-up: (n, k) = (intermediate_size_per_partition * 2, hidden_size)
# down: (n, k) = (hidden_size, intermediate_size_per_partition)
# moe marlin requires n % 128 == 0 and k % 64 == 0
supports_shape = (
hidden_size % 128 == 0
and intermediate_size_per_partition % max(64, group_size) == 0
)
if allow_tile_padding:
supports_shape = hidden_size % 128 == 0 and (
group_size <= 0 or intermediate_size_per_partition % group_size == 0
)
else:
supports_shape = (
hidden_size % 128 == 0
and intermediate_size_per_partition % max(64, group_size) == 0
)
supports_group_size = group_size in [-1, 32, 64, 128]
return supports_shape and supports_group_size and supports_router_weight
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
USE_FP32_REDUCE_DEFAULT,
get_marlin_input_dtype,
marlin_make_workspace_new,
marlin_moe_padded_intermediate,
marlin_pad_dim,
marlin_pad_qweight,
marlin_pad_scales,
@@ -216,6 +217,25 @@ def prepare_fp8_layer_for_marlin(
replace_parameter(layer, "bias", bias)
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
if padded_n == n:
return x
e = x.size(0)
rest = x.shape[2:]
x = x.view(e, 2, n, *rest)
x = torch.nn.functional.pad(x, (0, 0) * len(rest) + (0, padded_n - n))
return x.reshape(e, 2 * padded_n, *rest)
def _moe_pad_last(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad the last dim of a ``(E, ..., n)`` tensor to padded_n."""
if padded_n == n:
return x
return torch.nn.functional.pad(x, (0, padded_n - n))
def prepare_fp8_moe_layer_for_marlin(
layer: torch.nn.Module,
w13_weight: torch.Tensor,
@@ -246,6 +266,15 @@ def prepare_fp8_moe_layer_for_marlin(
n = layer.intermediate_size_per_partition
w13_n = w13_weight.size(1)
weight_block_size = getattr(layer, "weight_block_size", None)
group_size = -1 if weight_block_size is None else weight_block_size[1]
# Pad a tile-misaligned intermediate size to a valid Marlin thread tile.
# FP8 zero decodes to 0.0, so padded weights drop out; the converted scales
# are padded to match below (the padded values are irrelevant).
padded_n = marlin_moe_padded_intermediate(n, group_size)
if padded_n != n:
w13_weight = _moe_pad_shard_rows(w13_weight, n, padded_n)
w2_weight = _moe_pad_last(w2_weight, n, padded_n)
# WORKSPACE
device = layer.w13_weight.device
@@ -258,13 +287,7 @@ def prepare_fp8_moe_layer_for_marlin(
# Repack weights to marlin format
def repack_weight(name: str, weight: torch.Tensor) -> torch.Tensor:
tensor_list = []
if "w13" in name:
size_n, size_k = w13_n, k
else:
size_n, size_k = k, n
assert weight.shape == (e, size_n, size_k)
size_n, size_k = weight.size(1), weight.size(2)
for i in range(e):
qweight = pack_fp8_to_int32(weight[i], size_k_first=False)
qweight = qweight.T.contiguous()
@@ -280,9 +303,7 @@ def prepare_fp8_moe_layer_for_marlin(
w2_weight = repack_weight("w2", w2_weight)
# WEIGHT SCALES
# Permute scales
group_size = -1 if weight_block_size is None else weight_block_size[1]
# Permute scales (convert at the original size, then pad to the tile).
def permute_scales(scales: torch.Tensor, name: str) -> torch.Tensor:
scales = scales.to(layer.orig_dtype)
tensor_list = []
@@ -320,6 +341,20 @@ def prepare_fp8_moe_layer_for_marlin(
# size_n may not divisible by block_size[0]
scales = scales[..., :size_n].contiguous()
# Pad the converted (E, G, size_n) scales to the padded thread tile.
if padded_n != n:
if "w13" in name:
g = scales.size(1)
scales = scales.view(e, g, 2, n)
scales = torch.nn.functional.pad(scales, (0, padded_n - n))
scales = scales.reshape(e, g, 2 * padded_n)
size_n = 2 * padded_n
else:
if group_size > 0:
pad_groups = (padded_n - n) // group_size
scales = torch.nn.functional.pad(scales, (0, 0, 0, pad_groups))
size_k = padded_n
for i in range(e):
marlin_scales = marlin_permute_scales(
s=scales[i], size_k=size_k, size_n=size_n, group_size=group_size
@@ -497,10 +532,19 @@ def prepare_mxfp8_moe_layer_for_marlin(
"""
group_size = 32
e = w13.shape[0]
w13_n = w13.shape[1]
k = w13.shape[2]
n = w2.shape[2]
# Pad a tile-misaligned intermediate size to a valid Marlin thread tile.
padded_n = marlin_moe_padded_intermediate(n, group_size)
if padded_n != n:
w13 = _moe_pad_shard_rows(w13, n, padded_n)
w13_scale = _moe_pad_shard_rows(w13_scale, n, padded_n)
w2 = _moe_pad_last(w2, n, padded_n)
w2_scale = _moe_pad_last(w2_scale, n // group_size, padded_n // group_size)
n = padded_n
w13_n = w13.shape[1]
device = w13.device
param_dtype = torch.get_default_dtype()
perm = torch.empty(0, dtype=torch.int, device=device)