[Kernel] Extend Marlin thread-tile padding to MoE (WNA16 + FP8/MXFP8) (#45703)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -330,12 +330,43 @@ def check_marlin_supports_layer(
|
||||
)[0]
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bool:
|
||||
def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1) -> int:
|
||||
"""Smallest MoE intermediate size satisfying the Marlin MoE thread tiles.
|
||||
|
||||
The kernel needs gate-up ``2 * intermediate % 128 == 0`` and down
|
||||
``intermediate % 64 == 0``, i.e. ``intermediate % 64 == 0``. A misaligned
|
||||
size is zero-padded to the next valid tile at weight prep, kept a multiple
|
||||
of ``group_size`` so the group count stays integral. The padded region never
|
||||
reaches the MoE output: w13's padded output channels are zeroed by the
|
||||
zero-padded scales, so the padded inputs to w2 are zero.
|
||||
"""
|
||||
group = group_size if group_size > 0 else 1
|
||||
padded = round_up(intermediate_size, math.lcm(64, group))
|
||||
if padded != intermediate_size:
|
||||
logger.warning_once(
|
||||
"Marlin requires thread-tile padding for the MoE intermediate size "
|
||||
"of some layers in this model. Padded experts pad/slice activations "
|
||||
"on every forward; performance may be degraded."
|
||||
)
|
||||
return padded
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(
|
||||
layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False
|
||||
) -> bool:
|
||||
"""Whether the fused MoE Marlin kernel supports ``layer``.
|
||||
|
||||
Callers without act-order may pass ``allow_tile_padding=True``: a
|
||||
tile-misaligned intermediate size is then zero-padded to a valid thread
|
||||
tile at weight prep (see marlin_moe_padded_intermediate), so only a group
|
||||
straddling the padded boundary stays unsupported. hidden_size is the MoE
|
||||
I/O extent and is never padded. Act-order keeps the strict shape.
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
return False
|
||||
hidden_size = layer.hidden_size
|
||||
# Note: The layer has not performed rounding on intermediate_size's at this
|
||||
# point. Use the unpadded size which won't change.
|
||||
# The layer has not rounded intermediate_size yet; use the stable unpadded
|
||||
# size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64.
|
||||
intermediate_size_per_partition = (
|
||||
layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
)
|
||||
@@ -343,13 +374,15 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
|
||||
# apply_router_weight_on_input is not supported for moe marlin
|
||||
supports_router_weight = not layer.apply_router_weight_on_input
|
||||
|
||||
# gate-up: (n, k) = (intermediate_size_per_partition * 2, hidden_size)
|
||||
# down: (n, k) = (hidden_size, intermediate_size_per_partition)
|
||||
# moe marlin requires n % 128 == 0 and k % 64 == 0
|
||||
supports_shape = (
|
||||
hidden_size % 128 == 0
|
||||
and intermediate_size_per_partition % max(64, group_size) == 0
|
||||
)
|
||||
if allow_tile_padding:
|
||||
supports_shape = hidden_size % 128 == 0 and (
|
||||
group_size <= 0 or intermediate_size_per_partition % group_size == 0
|
||||
)
|
||||
else:
|
||||
supports_shape = (
|
||||
hidden_size % 128 == 0
|
||||
and intermediate_size_per_partition % max(64, group_size) == 0
|
||||
)
|
||||
supports_group_size = group_size in [-1, 32, 64, 128]
|
||||
return supports_shape and supports_group_size and supports_router_weight
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_dim,
|
||||
marlin_pad_qweight,
|
||||
marlin_pad_scales,
|
||||
@@ -216,6 +217,25 @@ def prepare_fp8_layer_for_marlin(
|
||||
replace_parameter(layer, "bias", bias)
|
||||
|
||||
|
||||
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
|
||||
if padded_n == n:
|
||||
return x
|
||||
e = x.size(0)
|
||||
rest = x.shape[2:]
|
||||
x = x.view(e, 2, n, *rest)
|
||||
x = torch.nn.functional.pad(x, (0, 0) * len(rest) + (0, padded_n - n))
|
||||
return x.reshape(e, 2 * padded_n, *rest)
|
||||
|
||||
|
||||
def _moe_pad_last(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad the last dim of a ``(E, ..., n)`` tensor to padded_n."""
|
||||
if padded_n == n:
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, padded_n - n))
|
||||
|
||||
|
||||
def prepare_fp8_moe_layer_for_marlin(
|
||||
layer: torch.nn.Module,
|
||||
w13_weight: torch.Tensor,
|
||||
@@ -246,6 +266,15 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
n = layer.intermediate_size_per_partition
|
||||
w13_n = w13_weight.size(1)
|
||||
weight_block_size = getattr(layer, "weight_block_size", None)
|
||||
group_size = -1 if weight_block_size is None else weight_block_size[1]
|
||||
|
||||
# Pad a tile-misaligned intermediate size to a valid Marlin thread tile.
|
||||
# FP8 zero decodes to 0.0, so padded weights drop out; the converted scales
|
||||
# are padded to match below (the padded values are irrelevant).
|
||||
padded_n = marlin_moe_padded_intermediate(n, group_size)
|
||||
if padded_n != n:
|
||||
w13_weight = _moe_pad_shard_rows(w13_weight, n, padded_n)
|
||||
w2_weight = _moe_pad_last(w2_weight, n, padded_n)
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
@@ -258,13 +287,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
# Repack weights to marlin format
|
||||
def repack_weight(name: str, weight: torch.Tensor) -> torch.Tensor:
|
||||
tensor_list = []
|
||||
if "w13" in name:
|
||||
size_n, size_k = w13_n, k
|
||||
else:
|
||||
size_n, size_k = k, n
|
||||
|
||||
assert weight.shape == (e, size_n, size_k)
|
||||
|
||||
size_n, size_k = weight.size(1), weight.size(2)
|
||||
for i in range(e):
|
||||
qweight = pack_fp8_to_int32(weight[i], size_k_first=False)
|
||||
qweight = qweight.T.contiguous()
|
||||
@@ -280,9 +303,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
w2_weight = repack_weight("w2", w2_weight)
|
||||
|
||||
# WEIGHT SCALES
|
||||
# Permute scales
|
||||
group_size = -1 if weight_block_size is None else weight_block_size[1]
|
||||
|
||||
# Permute scales (convert at the original size, then pad to the tile).
|
||||
def permute_scales(scales: torch.Tensor, name: str) -> torch.Tensor:
|
||||
scales = scales.to(layer.orig_dtype)
|
||||
tensor_list = []
|
||||
@@ -320,6 +341,20 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
# size_n may not divisible by block_size[0]
|
||||
scales = scales[..., :size_n].contiguous()
|
||||
|
||||
# Pad the converted (E, G, size_n) scales to the padded thread tile.
|
||||
if padded_n != n:
|
||||
if "w13" in name:
|
||||
g = scales.size(1)
|
||||
scales = scales.view(e, g, 2, n)
|
||||
scales = torch.nn.functional.pad(scales, (0, padded_n - n))
|
||||
scales = scales.reshape(e, g, 2 * padded_n)
|
||||
size_n = 2 * padded_n
|
||||
else:
|
||||
if group_size > 0:
|
||||
pad_groups = (padded_n - n) // group_size
|
||||
scales = torch.nn.functional.pad(scales, (0, 0, 0, pad_groups))
|
||||
size_k = padded_n
|
||||
|
||||
for i in range(e):
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scales[i], size_k=size_k, size_n=size_n, group_size=group_size
|
||||
@@ -497,10 +532,19 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
"""
|
||||
group_size = 32
|
||||
e = w13.shape[0]
|
||||
w13_n = w13.shape[1]
|
||||
k = w13.shape[2]
|
||||
n = w2.shape[2]
|
||||
|
||||
# Pad a tile-misaligned intermediate size to a valid Marlin thread tile.
|
||||
padded_n = marlin_moe_padded_intermediate(n, group_size)
|
||||
if padded_n != n:
|
||||
w13 = _moe_pad_shard_rows(w13, n, padded_n)
|
||||
w13_scale = _moe_pad_shard_rows(w13_scale, n, padded_n)
|
||||
w2 = _moe_pad_last(w2, n, padded_n)
|
||||
w2_scale = _moe_pad_last(w2_scale, n // group_size, padded_n // group_size)
|
||||
n = padded_n
|
||||
w13_n = w13.shape[1]
|
||||
|
||||
device = w13.device
|
||||
param_dtype = torch.get_default_dtype()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
Reference in New Issue
Block a user