[Kernel] Extend Marlin thread-tile padding to MoE (WNA16 + FP8/MXFP8) (#45703)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-23 11:45:10 -07:00
committed by GitHub
co-authored by Claude
parent d8e422ccda
commit 37a682d392
7 changed files with 573 additions and 29 deletions
@@ -5,6 +5,8 @@
Run `pytest tests/kernels/quantization/test_marlin_tile_padding.py`.
"""
from types import SimpleNamespace
import pytest
import torch
@@ -14,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
apply_gptq_marlin_linear,
marlin_make_empty_g_idx,
marlin_make_workspace_new,
marlin_moe_padded_intermediate,
marlin_pad_qweight,
marlin_pad_scales,
marlin_padded_nk,
@@ -115,6 +118,74 @@ def test_marlin_pad_helpers_shapes():
assert padded.shape == (1, padded_n)
# Rank-local MoE intermediate sizes. group<=0 / 32 with a non-multiple-of-64
# size is where tile padding triggers; 64/128 are already tile-aligned.
MOE_INTERMEDIATE_SIZES = [64, 96, 100, 176, 192, 256, 2816]
@pytest.mark.parametrize("intermediate", MOE_INTERMEDIATE_SIZES)
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
def test_marlin_moe_padded_intermediate(intermediate, group_size):
# The MoE gate only admits shapes where the group does not straddle the
# boundary, i.e. group divides the intermediate size.
if group_size > 0 and intermediate % group_size != 0:
pytest.skip("group straddles the boundary; rejected by the MoE gate")
padded = marlin_moe_padded_intermediate(intermediate, group_size)
assert padded >= intermediate
# Valid MoE thread tile: gate-up n = 2*intermediate % 128, down k % 64.
assert (2 * padded) % 128 == 0
assert padded % 64 == 0
if group_size > 0:
assert padded % group_size == 0
# Minimal: no smaller valid intermediate exists.
for cand in range(intermediate, padded):
if (
(2 * cand) % 128 == 0
and cand % 64 == 0
and (group_size <= 0 or cand % group_size == 0)
):
pytest.fail(f"{cand} beats {padded}")
# Already-tile-aligned sizes pass through unchanged (zero hot-path cost).
if intermediate % 64 == 0:
assert padded == intermediate
def test_marlin_moe_pad_helpers_shapes():
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
_pad_rows,
_pad_w13_bias,
_pad_w13_shard_cols,
)
E, rows, N, padded_N = 2, 8, 96, 128
# w13 stores the two gate/up shards along the last dim; padding each shard
# must preserve the loaded values and zero the padded columns.
w13 = torch.arange(E * rows * 2 * N).reshape(E, rows, 2 * N).float()
padded = _pad_w13_shard_cols(w13, N, padded_N)
assert padded.shape == (E, rows, 2 * padded_N)
shards = padded.view(E, rows, 2, padded_N)
orig = w13.view(E, rows, 2, N)
assert torch.equal(shards[..., :N], orig)
assert shards[..., N:].abs().sum() == 0
# w2 stores the intermediate dim in the rows.
w2 = torch.ones(E, N // 32, 16)
padded = _pad_rows(w2, padded_N // 32)
assert padded.shape == (E, padded_N // 32, 16)
assert padded[:, N // 32 :, :].abs().sum() == 0
bias = torch.arange(E * 2 * N).reshape(E, 2 * N).float()
padded = _pad_w13_bias(bias, N, padded_N)
assert padded.shape == (E, 2 * padded_N)
bias_shards = padded.view(E, 2, padded_N)
assert torch.equal(bias_shards[..., :N], bias.view(E, 2, N))
assert bias_shards[..., N:].abs().sum() == 0
def _gpu_marlin_unsupported() -> bool:
return not (
current_platform.is_cuda() and current_platform.has_device_capability(80)
@@ -468,3 +539,321 @@ def test_check_marlin_supports_layer_allow_tile_padding():
# A group straddling the TP shard cannot be fixed by padding
layer = _FakeLinear(4608, 4672, input_size=18688)
assert not check_marlin_supports_layer(layer, 128, allow_tile_padding=True)
@pytest.mark.skipif(
_gpu_marlin_unsupported(),
reason="Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("group_size", [-1, 32])
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
"""Pad a tile-misaligned MoE intermediate the way the WNA16 Marlin MoE prep
does, run the real repack + fused_marlin_moe, and check against the
dequantized reference. Symmetric int4's quantized zero decodes to -8, so the
padded region only stays out of the output via the zero-padded scales.
"""
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
fused_marlin_moe,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
_pad_rows,
_pad_w13_shard_cols,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_moe_padded_intermediate,
marlin_moe_permute_scales,
)
n, k, e = shape
topk, m = 2, 33
padded_n = marlin_moe_padded_intermediate(n, group_size)
assert padded_n != n, "test should exercise padding"
dtype = torch.float16
device = torch.device("cuda")
quant_type = scalar_types.uint4b8
bits = quant_type.size_bits
pack = 32 // bits
a = torch.randn((m, k), device=device, dtype=dtype) / 10
w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
def quant(w, size_k, size_n):
# w is (size_n, size_k); gptq expects (size_k, size_n).
ref, q_w, s, _, _ = gptq_quantize_weights(
w.T, quant_type, group_size, act_order=False
)
return ref, gptq_pack(q_w, bits, size_k, size_n), s
w13_qw, w13_s, w13_ref = [], [], []
w2_qw, w2_s, w2_ref = [], [], []
for i in range(e):
ref, qw, s = quant(w1[i], k, 2 * n)
w13_ref.append(ref.T) # (2n, k)
w13_qw.append(qw)
w13_s.append(s)
ref, qw, s = quant(w2[i], n, k)
w2_ref.append(ref.T) # (k, n)
w2_qw.append(qw)
w2_s.append(s)
w13_qweight = torch.stack(w13_qw)
w2_qweight = torch.stack(w2_qw)
w13_scales = torch.stack(w13_s)
w2_scales = torch.stack(w2_s)
w1_ref = torch.stack(w13_ref) # (e, 2n, k)
w2_ref = torch.stack(w2_ref) # (e, k, n)
# Pad the intermediate via the production helpers.
w13_qweight = _pad_w13_shard_cols(w13_qweight, n, padded_n)
w2_qweight = _pad_rows(w2_qweight, padded_n // pack)
w13_scales = _pad_w13_shard_cols(w13_scales, n, padded_n)
if group_size > 0:
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
sort_idx = torch.empty((e, 0), dtype=torch.int32, device=device)
marlin_w13 = ops.gptq_marlin_moe_repack(
w13_qweight, sort_idx, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
)
marlin_w2 = ops.gptq_marlin_moe_repack(
w2_qweight, sort_idx, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
)
group_or_pack = group_size if group_size != -1 else pack
marlin_w13_s = marlin_moe_permute_scales(
s=w13_scales, size_k=n, size_n=w13_scales.shape[2], group_size=group_size
)
marlin_w2_s = marlin_moe_permute_scales(
s=w2_scales,
size_k=w2_scales.shape[1] * group_or_pack,
size_n=w2_scales.shape[2],
group_size=group_size,
)
score = torch.randn((m, e), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
marlin_out = fused_marlin_moe(
a,
marlin_w13,
marlin_w2,
None,
None,
marlin_w13_s,
marlin_w2_s,
topk_weights,
topk_ids,
quant_type_id=quant_type.id,
global_num_experts=e,
is_k_full=True,
)
with set_current_vllm_config(VllmConfig()):
ref = torch_experts(
a,
w1_ref,
w2_ref,
topk_weight=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
)
torch.testing.assert_close(marlin_out, ref, atol=5e-2, rtol=0)
def test_check_moe_marlin_supports_layer_padding():
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_layer,
)
def make_layer(hidden, intermediate):
layer = SimpleNamespace()
layer.hidden_size = hidden
layer.apply_router_weight_on_input = False
layer.moe_config = SimpleNamespace(
intermediate_size_per_partition_unpadded=intermediate
)
return layer
# group=32 with intermediate % 64 != 0: rejected strictly, accepted w/ padding
layer = make_layer(4096, 96)
assert not check_moe_marlin_supports_layer(layer, 32)
assert check_moe_marlin_supports_layer(layer, 32, allow_tile_padding=True)
# channelwise misaligned intermediate is paddable
assert check_moe_marlin_supports_layer(layer, -1, allow_tile_padding=True)
# A group straddling the boundary cannot be fixed by padding
layer = make_layer(4096, 176)
assert not check_moe_marlin_supports_layer(layer, 128, allow_tile_padding=True)
# hidden_size is the MoE I/O extent and is never padded
layer = make_layer(4090, 128)
assert not check_moe_marlin_supports_layer(layer, 64, allow_tile_padding=True)
@pytest.mark.skipif(
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
reason="FP8 Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("quant", ["channel", "tensor"])
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
def test_fp8_marlin_moe_padded_round_trip(shape, quant):
"""FP8 weight-only MoE: pad a tile-misaligned intermediate and check the
real prepare + fused_marlin_moe against the dequantized reference."""
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
fused_marlin_moe,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_moe_intermediate_size,
marlin_moe_padded_intermediate,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_fp8_moe_layer_for_marlin,
)
n, k, e = shape
topk, m = 2, 33
fp8 = torch.float8_e4m3fn
dtype = torch.bfloat16
device = torch.device("cuda")
padded_n = marlin_moe_padded_intermediate(n, -1)
assert padded_n != n
def q(w): # (out, in) -> fp8 weight, scale, dequant reference
dim = None if quant == "tensor" else 1
s = (w.abs().amax(dim, keepdim=dim is not None) / 448.0).clamp(min=1e-8)
wq = (w / s).clamp(-448, 448).to(fp8)
ref = wq.to(dtype) * s.to(dtype)
s = s.reshape(1) if quant == "tensor" else s.squeeze(1)
return wq, s, ref
a = torch.randn((m, k), device=device, dtype=dtype) / 10
w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
w13_q, w13_s, w1_ref = zip(*(q(w1[i]) for i in range(e)))
w2_q, w2_s, w2_ref = zip(*(q(w2[i]) for i in range(e)))
w13_weight, w2_weight = torch.stack(w13_q), torch.stack(w2_q)
layer = SimpleNamespace(
num_experts=e,
hidden_size=k,
intermediate_size_per_partition=n,
orig_dtype=dtype,
w13_weight=w13_weight,
)
pw13, pw2, ps13, ps2 = prepare_fp8_moe_layer_for_marlin(
layer, w13_weight, w2_weight, torch.stack(w13_s), torch.stack(w2_s)
)
assert marlin_moe_intermediate_size(pw13, pw2) == padded_n
score = torch.randn((m, e), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
out = fused_marlin_moe(
a,
pw13,
pw2,
None,
None,
ps13,
ps2,
topk_weights,
topk_ids,
quant_type_id=scalar_types.float8_e4m3fn.id,
global_num_experts=e,
is_k_full=True,
workspace=layer.workspace,
)
with set_current_vllm_config(VllmConfig()):
ref = torch_experts(
a,
torch.stack(w1_ref),
torch.stack(w2_ref),
topk_weight=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
)
torch.testing.assert_close(out, ref, atol=8e-2, rtol=0)
@pytest.mark.skipif(
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
reason="FP8 Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
def test_mxfp8_marlin_moe_padded_round_trip(shape):
"""MXFP8 weight-only MoE round-trip at a tile-misaligned intermediate, with
unit e8m0 scales so the reference is the exact fp8 dequant."""
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
fused_marlin_moe,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_moe_intermediate_size,
marlin_moe_padded_intermediate,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_mxfp8_moe_layer_for_marlin,
)
n, k, e = shape
topk, m, gs, e8m0_one = 2, 33, 32, 127
fp8 = torch.float8_e4m3fn
dtype = torch.bfloat16
device = torch.device("cuda")
padded_n = marlin_moe_padded_intermediate(n, gs)
assert padded_n != n
a = torch.randn((m, k), device=device, dtype=dtype) / 10
w13_weight = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
w2_weight = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
w13_weight = w13_weight.clamp(-448, 448).to(fp8)
w2_weight = w2_weight.clamp(-448, 448).to(fp8)
w13_scale = torch.full(
(e, 2 * n, k // gs), e8m0_one, dtype=torch.uint8, device=device
)
w2_scale = torch.full((e, k, n // gs), e8m0_one, dtype=torch.uint8, device=device)
layer = SimpleNamespace(
num_experts=e, hidden_size=k, intermediate_size_per_partition=n
)
with set_current_vllm_config(VllmConfig()):
pw13, pw2, ps13, ps2 = prepare_mxfp8_moe_layer_for_marlin(
layer, w13_weight, w2_weight, w13_scale, w2_scale
)
assert marlin_moe_intermediate_size(pw13, pw2) == padded_n
score = torch.randn((m, e), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
out = fused_marlin_moe(
a,
pw13,
pw2,
None,
None,
ps13,
ps2,
topk_weights,
topk_ids,
quant_type_id=scalar_types.float8_e4m3fn.id,
global_num_experts=e,
is_k_full=True,
workspace=layer.workspace,
)
with set_current_vllm_config(VllmConfig()):
ref = torch_experts(
a,
w13_weight.to(dtype),
w2_weight.to(dtype),
topk_weight=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
)
torch.testing.assert_close(out, ref, atol=8e-2, rtol=0)
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_act_int8_process_scales,
marlin_moe_padded_intermediate,
marlin_moe_permute_scales,
marlin_permute_bias,
moe_awq_to_marlin_zero_points,
@@ -361,6 +362,34 @@ def _process_weights_flashinfer(
)
def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor:
"""Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)``
tensor along its last dim, from ``unit`` to ``padded_unit`` columns."""
if padded_unit == unit:
return x
e, rows, _ = x.shape
x = x.view(e, rows, 2, unit)
x = torch.nn.functional.pad(x, (0, padded_unit - unit))
return x.reshape(e, rows, 2 * padded_unit).contiguous()
def _pad_rows(x: torch.Tensor, padded_rows: int) -> torch.Tensor:
"""Zero-pad a ``(E, rows, cols)`` tensor to ``padded_rows`` rows."""
if padded_rows == x.size(1):
return x
return torch.nn.functional.pad(x, (0, 0, 0, padded_rows - x.size(1)))
def _pad_w13_bias(bias: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad each gate/up shard of a ``(E, 2 * n)`` bias to ``padded_n``."""
if padded_n == n:
return bias
e = bias.size(0)
bias = bias.view(e, 2, n)
bias = torch.nn.functional.pad(bias, (0, padded_n - n))
return bias.reshape(e, 2 * padded_n).contiguous()
def _process_weights_marlin(
layer: torch.nn.Module,
input_dtype: torch.dtype | None,
@@ -431,6 +460,29 @@ def _process_weights_marlin(
marlin_w13_scales = w13_scales
marlin_w2_scales = w2_scales
# --- Pad the intermediate size to a valid Marlin thread tile ---
# GPTQ packs along K: w13's N is in the (shard) columns, w2's N in the rows.
# Act-order keeps the strict shape and is never padded.
N = layer.intermediate_size_per_partition
padded_N = marlin_moe_padded_intermediate(N, group_size)
if padded_N != N:
assert actorder != "group", (
"Marlin MoE thread-tile padding is unsupported with act-order"
)
marlin_w13_qweight = _pad_w13_shard_cols(marlin_w13_qweight, N, padded_N)
marlin_w2_qweight = _pad_rows(marlin_w2_qweight, padded_N // pack_factor)
marlin_w13_scales = _pad_w13_shard_cols(marlin_w13_scales, N, padded_N)
if group_size > 0:
marlin_w2_scales = _pad_rows(marlin_w2_scales, padded_N // group_size)
if w13_qzeros is not None:
w13_qzeros = _pad_w13_shard_cols(
w13_qzeros, N // pack_factor, padded_N // pack_factor
)
if w2_qzeros is not None and group_size > 0:
w2_qzeros = _pad_rows(w2_qzeros, padded_N // group_size)
if w13_bias is not None:
w13_bias = _pad_w13_bias(w13_bias, N, padded_N)
# --- Process act_order (g_idx) ---
if actorder == "group":
num_experts = w13_g_idx.shape[0]
@@ -608,6 +660,25 @@ def _process_awq_weights_marlin(
w13_scales = w13_scales.data * 512
w2_scales = w2_scales.data * 512
# --- Pad the intermediate size to a valid Marlin thread tile ---
# AWQ packs along N: w13's N is in the (shard) columns, w2's N in the rows.
N = layer.intermediate_size_per_partition
padded_N = marlin_moe_padded_intermediate(N, group_size)
if padded_N != N:
w13_qweight = _pad_w13_shard_cols(
w13_qweight, N // pack_factor, padded_N // pack_factor
)
w2_qweight = _pad_rows(w2_qweight, padded_N)
w13_scales = _pad_w13_shard_cols(w13_scales, N, padded_N)
w13_qzeros = _pad_w13_shard_cols(
w13_qzeros, N // pack_factor, padded_N // pack_factor
)
if group_size > 0:
w2_scales = _pad_rows(w2_scales, padded_N // group_size)
w2_qzeros = _pad_rows(w2_qzeros, padded_N // group_size)
if w13_bias is not None:
w13_bias = _pad_w13_bias(w13_bias, N, padded_N)
w13_g_idx_sort_indices = torch.nn.Parameter(
torch.empty((num_experts, 0), dtype=torch.int32, device=device),
requires_grad=False,
@@ -339,7 +339,9 @@ class AutoAWQConfig(QuantizationConfig):
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if not check_moe_marlin_supports_layer(layer, self.group_size):
if not check_moe_marlin_supports_layer(
layer, self.group_size, allow_tile_padding=True
):
logger.warning_once(
f"Layer '{prefix}' is not supported by AutoAWQMoEMarlin. "
"Falling back to Moe WNA16 kernels."
@@ -243,7 +243,9 @@ class AutoGPTQConfig(QuantizationConfig):
if isinstance(layer, RoutedExperts):
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
if not check_moe_marlin_supports_layer(layer, self.group_size):
if not check_moe_marlin_supports_layer(
layer, self.group_size, allow_tile_padding=not self.desc_act
):
logger.warning_once(
f"Layer '{prefix}' is not supported by GPTQMoeMarlin. "
"Falling back to Moe WNA16 kernels."
@@ -96,15 +96,18 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
)
# Prefer to use the MarlinMoE kernel when it is supported.
is_actorder = (
weight_quant.strategy == QuantizationStrategy.GROUP
and weight_quant.actorder
in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC)
)
if (
not check_moe_marlin_supports_layer(layer, group_size)
not check_moe_marlin_supports_layer(
layer, group_size, allow_tile_padding=not is_actorder
)
or current_platform.is_rocm()
):
if (
weight_quant.strategy == QuantizationStrategy.GROUP
and weight_quant.actorder
in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC)
):
if is_actorder:
raise ValueError(
"WNA16MoE is not supported with actorder=group/dynamic."
)
@@ -330,12 +330,43 @@ def check_marlin_supports_layer(
)[0]
def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bool:
def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1) -> int:
"""Smallest MoE intermediate size satisfying the Marlin MoE thread tiles.
The kernel needs gate-up ``2 * intermediate % 128 == 0`` and down
``intermediate % 64 == 0``, i.e. ``intermediate % 64 == 0``. A misaligned
size is zero-padded to the next valid tile at weight prep, kept a multiple
of ``group_size`` so the group count stays integral. The padded region never
reaches the MoE output: w13's padded output channels are zeroed by the
zero-padded scales, so the padded inputs to w2 are zero.
"""
group = group_size if group_size > 0 else 1
padded = round_up(intermediate_size, math.lcm(64, group))
if padded != intermediate_size:
logger.warning_once(
"Marlin requires thread-tile padding for the MoE intermediate size "
"of some layers in this model. Padded experts pad/slice activations "
"on every forward; performance may be degraded."
)
return padded
def check_moe_marlin_supports_layer(
layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False
) -> bool:
"""Whether the fused MoE Marlin kernel supports ``layer``.
Callers without act-order may pass ``allow_tile_padding=True``: a
tile-misaligned intermediate size is then zero-padded to a valid thread
tile at weight prep (see marlin_moe_padded_intermediate), so only a group
straddling the padded boundary stays unsupported. hidden_size is the MoE
I/O extent and is never padded. Act-order keeps the strict shape.
"""
if current_platform.is_rocm():
return False
hidden_size = layer.hidden_size
# Note: The layer has not performed rounding on intermediate_size's at this
# point. Use the unpadded size which won't change.
# The layer has not rounded intermediate_size yet; use the stable unpadded
# size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64.
intermediate_size_per_partition = (
layer.moe_config.intermediate_size_per_partition_unpadded
)
@@ -343,13 +374,15 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
# apply_router_weight_on_input is not supported for moe marlin
supports_router_weight = not layer.apply_router_weight_on_input
# gate-up: (n, k) = (intermediate_size_per_partition * 2, hidden_size)
# down: (n, k) = (hidden_size, intermediate_size_per_partition)
# moe marlin requires n % 128 == 0 and k % 64 == 0
supports_shape = (
hidden_size % 128 == 0
and intermediate_size_per_partition % max(64, group_size) == 0
)
if allow_tile_padding:
supports_shape = hidden_size % 128 == 0 and (
group_size <= 0 or intermediate_size_per_partition % group_size == 0
)
else:
supports_shape = (
hidden_size % 128 == 0
and intermediate_size_per_partition % max(64, group_size) == 0
)
supports_group_size = group_size in [-1, 32, 64, 128]
return supports_shape and supports_group_size and supports_router_weight
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
USE_FP32_REDUCE_DEFAULT,
get_marlin_input_dtype,
marlin_make_workspace_new,
marlin_moe_padded_intermediate,
marlin_pad_dim,
marlin_pad_qweight,
marlin_pad_scales,
@@ -216,6 +217,25 @@ def prepare_fp8_layer_for_marlin(
replace_parameter(layer, "bias", bias)
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
if padded_n == n:
return x
e = x.size(0)
rest = x.shape[2:]
x = x.view(e, 2, n, *rest)
x = torch.nn.functional.pad(x, (0, 0) * len(rest) + (0, padded_n - n))
return x.reshape(e, 2 * padded_n, *rest)
def _moe_pad_last(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad the last dim of a ``(E, ..., n)`` tensor to padded_n."""
if padded_n == n:
return x
return torch.nn.functional.pad(x, (0, padded_n - n))
def prepare_fp8_moe_layer_for_marlin(
layer: torch.nn.Module,
w13_weight: torch.Tensor,
@@ -246,6 +266,15 @@ def prepare_fp8_moe_layer_for_marlin(
n = layer.intermediate_size_per_partition
w13_n = w13_weight.size(1)
weight_block_size = getattr(layer, "weight_block_size", None)
group_size = -1 if weight_block_size is None else weight_block_size[1]
# Pad a tile-misaligned intermediate size to a valid Marlin thread tile.
# FP8 zero decodes to 0.0, so padded weights drop out; the converted scales
# are padded to match below (the padded values are irrelevant).
padded_n = marlin_moe_padded_intermediate(n, group_size)
if padded_n != n:
w13_weight = _moe_pad_shard_rows(w13_weight, n, padded_n)
w2_weight = _moe_pad_last(w2_weight, n, padded_n)
# WORKSPACE
device = layer.w13_weight.device
@@ -258,13 +287,7 @@ def prepare_fp8_moe_layer_for_marlin(
# Repack weights to marlin format
def repack_weight(name: str, weight: torch.Tensor) -> torch.Tensor:
tensor_list = []
if "w13" in name:
size_n, size_k = w13_n, k
else:
size_n, size_k = k, n
assert weight.shape == (e, size_n, size_k)
size_n, size_k = weight.size(1), weight.size(2)
for i in range(e):
qweight = pack_fp8_to_int32(weight[i], size_k_first=False)
qweight = qweight.T.contiguous()
@@ -280,9 +303,7 @@ def prepare_fp8_moe_layer_for_marlin(
w2_weight = repack_weight("w2", w2_weight)
# WEIGHT SCALES
# Permute scales
group_size = -1 if weight_block_size is None else weight_block_size[1]
# Permute scales (convert at the original size, then pad to the tile).
def permute_scales(scales: torch.Tensor, name: str) -> torch.Tensor:
scales = scales.to(layer.orig_dtype)
tensor_list = []
@@ -320,6 +341,20 @@ def prepare_fp8_moe_layer_for_marlin(
# size_n may not divisible by block_size[0]
scales = scales[..., :size_n].contiguous()
# Pad the converted (E, G, size_n) scales to the padded thread tile.
if padded_n != n:
if "w13" in name:
g = scales.size(1)
scales = scales.view(e, g, 2, n)
scales = torch.nn.functional.pad(scales, (0, padded_n - n))
scales = scales.reshape(e, g, 2 * padded_n)
size_n = 2 * padded_n
else:
if group_size > 0:
pad_groups = (padded_n - n) // group_size
scales = torch.nn.functional.pad(scales, (0, 0, 0, pad_groups))
size_k = padded_n
for i in range(e):
marlin_scales = marlin_permute_scales(
s=scales[i], size_k=size_k, size_n=size_n, group_size=group_size
@@ -497,10 +532,19 @@ def prepare_mxfp8_moe_layer_for_marlin(
"""
group_size = 32
e = w13.shape[0]
w13_n = w13.shape[1]
k = w13.shape[2]
n = w2.shape[2]
# Pad a tile-misaligned intermediate size to a valid Marlin thread tile.
padded_n = marlin_moe_padded_intermediate(n, group_size)
if padded_n != n:
w13 = _moe_pad_shard_rows(w13, n, padded_n)
w13_scale = _moe_pad_shard_rows(w13_scale, n, padded_n)
w2 = _moe_pad_last(w2, n, padded_n)
w2_scale = _moe_pad_last(w2_scale, n // group_size, padded_n // group_size)
n = padded_n
w13_n = w13.shape[1]
device = w13.device
param_dtype = torch.get_default_dtype()
perm = torch.empty(0, dtype=torch.int, device=device)