[Kernel] Extend Marlin thread-tile padding to MoE (WNA16 + FP8/MXFP8) (#45703)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,8 @@
|
||||
Run `pytest tests/kernels/quantization/test_marlin_tile_padding.py`.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -14,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
apply_gptq_marlin_linear,
|
||||
marlin_make_empty_g_idx,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_qweight,
|
||||
marlin_pad_scales,
|
||||
marlin_padded_nk,
|
||||
@@ -115,6 +118,74 @@ def test_marlin_pad_helpers_shapes():
|
||||
assert padded.shape == (1, padded_n)
|
||||
|
||||
|
||||
# Rank-local MoE intermediate sizes. group<=0 / 32 with a non-multiple-of-64
|
||||
# size is where tile padding triggers; 64/128 are already tile-aligned.
|
||||
MOE_INTERMEDIATE_SIZES = [64, 96, 100, 176, 192, 256, 2816]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate", MOE_INTERMEDIATE_SIZES)
|
||||
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
|
||||
def test_marlin_moe_padded_intermediate(intermediate, group_size):
|
||||
# The MoE gate only admits shapes where the group does not straddle the
|
||||
# boundary, i.e. group divides the intermediate size.
|
||||
if group_size > 0 and intermediate % group_size != 0:
|
||||
pytest.skip("group straddles the boundary; rejected by the MoE gate")
|
||||
|
||||
padded = marlin_moe_padded_intermediate(intermediate, group_size)
|
||||
assert padded >= intermediate
|
||||
# Valid MoE thread tile: gate-up n = 2*intermediate % 128, down k % 64.
|
||||
assert (2 * padded) % 128 == 0
|
||||
assert padded % 64 == 0
|
||||
if group_size > 0:
|
||||
assert padded % group_size == 0
|
||||
|
||||
# Minimal: no smaller valid intermediate exists.
|
||||
for cand in range(intermediate, padded):
|
||||
if (
|
||||
(2 * cand) % 128 == 0
|
||||
and cand % 64 == 0
|
||||
and (group_size <= 0 or cand % group_size == 0)
|
||||
):
|
||||
pytest.fail(f"{cand} beats {padded}")
|
||||
|
||||
# Already-tile-aligned sizes pass through unchanged (zero hot-path cost).
|
||||
if intermediate % 64 == 0:
|
||||
assert padded == intermediate
|
||||
|
||||
|
||||
def test_marlin_moe_pad_helpers_shapes():
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
_pad_rows,
|
||||
_pad_w13_bias,
|
||||
_pad_w13_shard_cols,
|
||||
)
|
||||
|
||||
E, rows, N, padded_N = 2, 8, 96, 128
|
||||
|
||||
# w13 stores the two gate/up shards along the last dim; padding each shard
|
||||
# must preserve the loaded values and zero the padded columns.
|
||||
w13 = torch.arange(E * rows * 2 * N).reshape(E, rows, 2 * N).float()
|
||||
padded = _pad_w13_shard_cols(w13, N, padded_N)
|
||||
assert padded.shape == (E, rows, 2 * padded_N)
|
||||
shards = padded.view(E, rows, 2, padded_N)
|
||||
orig = w13.view(E, rows, 2, N)
|
||||
assert torch.equal(shards[..., :N], orig)
|
||||
assert shards[..., N:].abs().sum() == 0
|
||||
|
||||
# w2 stores the intermediate dim in the rows.
|
||||
w2 = torch.ones(E, N // 32, 16)
|
||||
padded = _pad_rows(w2, padded_N // 32)
|
||||
assert padded.shape == (E, padded_N // 32, 16)
|
||||
assert padded[:, N // 32 :, :].abs().sum() == 0
|
||||
|
||||
bias = torch.arange(E * 2 * N).reshape(E, 2 * N).float()
|
||||
padded = _pad_w13_bias(bias, N, padded_N)
|
||||
assert padded.shape == (E, 2 * padded_N)
|
||||
bias_shards = padded.view(E, 2, padded_N)
|
||||
assert torch.equal(bias_shards[..., :N], bias.view(E, 2, N))
|
||||
assert bias_shards[..., N:].abs().sum() == 0
|
||||
|
||||
|
||||
def _gpu_marlin_unsupported() -> bool:
|
||||
return not (
|
||||
current_platform.is_cuda() and current_platform.has_device_capability(80)
|
||||
@@ -468,3 +539,321 @@ def test_check_marlin_supports_layer_allow_tile_padding():
|
||||
# A group straddling the TP shard cannot be fixed by padding
|
||||
layer = _FakeLinear(4608, 4672, input_size=18688)
|
||||
assert not check_marlin_supports_layer(layer, 128, allow_tile_padding=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported(),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("group_size", [-1, 32])
|
||||
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
|
||||
def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
"""Pad a tile-misaligned MoE intermediate the way the WNA16 Marlin MoE prep
|
||||
does, run the real repack + fused_marlin_moe, and check against the
|
||||
dequantized reference. Symmetric int4's quantized zero decodes to -8, so the
|
||||
padded region only stays out of the output via the zero-padded scales.
|
||||
"""
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
fused_marlin_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
_pad_rows,
|
||||
_pad_w13_shard_cols,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_moe_permute_scales,
|
||||
)
|
||||
|
||||
n, k, e = shape
|
||||
topk, m = 2, 33
|
||||
padded_n = marlin_moe_padded_intermediate(n, group_size)
|
||||
assert padded_n != n, "test should exercise padding"
|
||||
|
||||
dtype = torch.float16
|
||||
device = torch.device("cuda")
|
||||
quant_type = scalar_types.uint4b8
|
||||
bits = quant_type.size_bits
|
||||
pack = 32 // bits
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
|
||||
|
||||
def quant(w, size_k, size_n):
|
||||
# w is (size_n, size_k); gptq expects (size_k, size_n).
|
||||
ref, q_w, s, _, _ = gptq_quantize_weights(
|
||||
w.T, quant_type, group_size, act_order=False
|
||||
)
|
||||
return ref, gptq_pack(q_w, bits, size_k, size_n), s
|
||||
|
||||
w13_qw, w13_s, w13_ref = [], [], []
|
||||
w2_qw, w2_s, w2_ref = [], [], []
|
||||
for i in range(e):
|
||||
ref, qw, s = quant(w1[i], k, 2 * n)
|
||||
w13_ref.append(ref.T) # (2n, k)
|
||||
w13_qw.append(qw)
|
||||
w13_s.append(s)
|
||||
ref, qw, s = quant(w2[i], n, k)
|
||||
w2_ref.append(ref.T) # (k, n)
|
||||
w2_qw.append(qw)
|
||||
w2_s.append(s)
|
||||
|
||||
w13_qweight = torch.stack(w13_qw)
|
||||
w2_qweight = torch.stack(w2_qw)
|
||||
w13_scales = torch.stack(w13_s)
|
||||
w2_scales = torch.stack(w2_s)
|
||||
w1_ref = torch.stack(w13_ref) # (e, 2n, k)
|
||||
w2_ref = torch.stack(w2_ref) # (e, k, n)
|
||||
|
||||
# Pad the intermediate via the production helpers.
|
||||
w13_qweight = _pad_w13_shard_cols(w13_qweight, n, padded_n)
|
||||
w2_qweight = _pad_rows(w2_qweight, padded_n // pack)
|
||||
w13_scales = _pad_w13_shard_cols(w13_scales, n, padded_n)
|
||||
if group_size > 0:
|
||||
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
|
||||
|
||||
sort_idx = torch.empty((e, 0), dtype=torch.int32, device=device)
|
||||
marlin_w13 = ops.gptq_marlin_moe_repack(
|
||||
w13_qweight, sort_idx, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
|
||||
)
|
||||
marlin_w2 = ops.gptq_marlin_moe_repack(
|
||||
w2_qweight, sort_idx, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
|
||||
)
|
||||
group_or_pack = group_size if group_size != -1 else pack
|
||||
marlin_w13_s = marlin_moe_permute_scales(
|
||||
s=w13_scales, size_k=n, size_n=w13_scales.shape[2], group_size=group_size
|
||||
)
|
||||
marlin_w2_s = marlin_moe_permute_scales(
|
||||
s=w2_scales,
|
||||
size_k=w2_scales.shape[1] * group_or_pack,
|
||||
size_n=w2_scales.shape[2],
|
||||
group_size=group_size,
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device=device, dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
|
||||
marlin_out = fused_marlin_moe(
|
||||
a,
|
||||
marlin_w13,
|
||||
marlin_w2,
|
||||
None,
|
||||
None,
|
||||
marlin_w13_s,
|
||||
marlin_w2_s,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_type_id=quant_type.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
a,
|
||||
w1_ref,
|
||||
w2_ref,
|
||||
topk_weight=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(marlin_out, ref, atol=5e-2, rtol=0)
|
||||
|
||||
|
||||
def test_check_moe_marlin_supports_layer_padding():
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
def make_layer(hidden, intermediate):
|
||||
layer = SimpleNamespace()
|
||||
layer.hidden_size = hidden
|
||||
layer.apply_router_weight_on_input = False
|
||||
layer.moe_config = SimpleNamespace(
|
||||
intermediate_size_per_partition_unpadded=intermediate
|
||||
)
|
||||
return layer
|
||||
|
||||
# group=32 with intermediate % 64 != 0: rejected strictly, accepted w/ padding
|
||||
layer = make_layer(4096, 96)
|
||||
assert not check_moe_marlin_supports_layer(layer, 32)
|
||||
assert check_moe_marlin_supports_layer(layer, 32, allow_tile_padding=True)
|
||||
# channelwise misaligned intermediate is paddable
|
||||
assert check_moe_marlin_supports_layer(layer, -1, allow_tile_padding=True)
|
||||
|
||||
# A group straddling the boundary cannot be fixed by padding
|
||||
layer = make_layer(4096, 176)
|
||||
assert not check_moe_marlin_supports_layer(layer, 128, allow_tile_padding=True)
|
||||
|
||||
# hidden_size is the MoE I/O extent and is never padded
|
||||
layer = make_layer(4090, 128)
|
||||
assert not check_moe_marlin_supports_layer(layer, 64, allow_tile_padding=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
|
||||
reason="FP8 Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("quant", ["channel", "tensor"])
|
||||
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
|
||||
def test_fp8_marlin_moe_padded_round_trip(shape, quant):
|
||||
"""FP8 weight-only MoE: pad a tile-misaligned intermediate and check the
|
||||
real prepare + fused_marlin_moe against the dequantized reference."""
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
fused_marlin_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_moe_intermediate_size,
|
||||
marlin_moe_padded_intermediate,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_fp8_moe_layer_for_marlin,
|
||||
)
|
||||
|
||||
n, k, e = shape
|
||||
topk, m = 2, 33
|
||||
fp8 = torch.float8_e4m3fn
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device("cuda")
|
||||
padded_n = marlin_moe_padded_intermediate(n, -1)
|
||||
assert padded_n != n
|
||||
|
||||
def q(w): # (out, in) -> fp8 weight, scale, dequant reference
|
||||
dim = None if quant == "tensor" else 1
|
||||
s = (w.abs().amax(dim, keepdim=dim is not None) / 448.0).clamp(min=1e-8)
|
||||
wq = (w / s).clamp(-448, 448).to(fp8)
|
||||
ref = wq.to(dtype) * s.to(dtype)
|
||||
s = s.reshape(1) if quant == "tensor" else s.squeeze(1)
|
||||
return wq, s, ref
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
|
||||
w13_q, w13_s, w1_ref = zip(*(q(w1[i]) for i in range(e)))
|
||||
w2_q, w2_s, w2_ref = zip(*(q(w2[i]) for i in range(e)))
|
||||
|
||||
w13_weight, w2_weight = torch.stack(w13_q), torch.stack(w2_q)
|
||||
layer = SimpleNamespace(
|
||||
num_experts=e,
|
||||
hidden_size=k,
|
||||
intermediate_size_per_partition=n,
|
||||
orig_dtype=dtype,
|
||||
w13_weight=w13_weight,
|
||||
)
|
||||
pw13, pw2, ps13, ps2 = prepare_fp8_moe_layer_for_marlin(
|
||||
layer, w13_weight, w2_weight, torch.stack(w13_s), torch.stack(w2_s)
|
||||
)
|
||||
assert marlin_moe_intermediate_size(pw13, pw2) == padded_n
|
||||
|
||||
score = torch.randn((m, e), device=device, dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
out = fused_marlin_moe(
|
||||
a,
|
||||
pw13,
|
||||
pw2,
|
||||
None,
|
||||
None,
|
||||
ps13,
|
||||
ps2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
workspace=layer.workspace,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
a,
|
||||
torch.stack(w1_ref),
|
||||
torch.stack(w2_ref),
|
||||
topk_weight=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
)
|
||||
torch.testing.assert_close(out, ref, atol=8e-2, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
|
||||
reason="FP8 Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
|
||||
def test_mxfp8_marlin_moe_padded_round_trip(shape):
|
||||
"""MXFP8 weight-only MoE round-trip at a tile-misaligned intermediate, with
|
||||
unit e8m0 scales so the reference is the exact fp8 dequant."""
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
fused_marlin_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_moe_intermediate_size,
|
||||
marlin_moe_padded_intermediate,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_mxfp8_moe_layer_for_marlin,
|
||||
)
|
||||
|
||||
n, k, e = shape
|
||||
topk, m, gs, e8m0_one = 2, 33, 32, 127
|
||||
fp8 = torch.float8_e4m3fn
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device("cuda")
|
||||
padded_n = marlin_moe_padded_intermediate(n, gs)
|
||||
assert padded_n != n
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 10
|
||||
w13_weight = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
|
||||
w2_weight = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
|
||||
w13_weight = w13_weight.clamp(-448, 448).to(fp8)
|
||||
w2_weight = w2_weight.clamp(-448, 448).to(fp8)
|
||||
w13_scale = torch.full(
|
||||
(e, 2 * n, k // gs), e8m0_one, dtype=torch.uint8, device=device
|
||||
)
|
||||
w2_scale = torch.full((e, k, n // gs), e8m0_one, dtype=torch.uint8, device=device)
|
||||
|
||||
layer = SimpleNamespace(
|
||||
num_experts=e, hidden_size=k, intermediate_size_per_partition=n
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
pw13, pw2, ps13, ps2 = prepare_mxfp8_moe_layer_for_marlin(
|
||||
layer, w13_weight, w2_weight, w13_scale, w2_scale
|
||||
)
|
||||
assert marlin_moe_intermediate_size(pw13, pw2) == padded_n
|
||||
|
||||
score = torch.randn((m, e), device=device, dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
out = fused_marlin_moe(
|
||||
a,
|
||||
pw13,
|
||||
pw2,
|
||||
None,
|
||||
None,
|
||||
ps13,
|
||||
ps2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
workspace=layer.workspace,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
a,
|
||||
w13_weight.to(dtype),
|
||||
w2_weight.to(dtype),
|
||||
topk_weight=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
)
|
||||
torch.testing.assert_close(out, ref, atol=8e-2, rtol=0)
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_act_int8_process_scales,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_moe_permute_scales,
|
||||
marlin_permute_bias,
|
||||
moe_awq_to_marlin_zero_points,
|
||||
@@ -361,6 +362,34 @@ def _process_weights_flashinfer(
|
||||
)
|
||||
|
||||
|
||||
def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor:
|
||||
"""Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)``
|
||||
tensor along its last dim, from ``unit`` to ``padded_unit`` columns."""
|
||||
if padded_unit == unit:
|
||||
return x
|
||||
e, rows, _ = x.shape
|
||||
x = x.view(e, rows, 2, unit)
|
||||
x = torch.nn.functional.pad(x, (0, padded_unit - unit))
|
||||
return x.reshape(e, rows, 2 * padded_unit).contiguous()
|
||||
|
||||
|
||||
def _pad_rows(x: torch.Tensor, padded_rows: int) -> torch.Tensor:
|
||||
"""Zero-pad a ``(E, rows, cols)`` tensor to ``padded_rows`` rows."""
|
||||
if padded_rows == x.size(1):
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, 0, 0, padded_rows - x.size(1)))
|
||||
|
||||
|
||||
def _pad_w13_bias(bias: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a ``(E, 2 * n)`` bias to ``padded_n``."""
|
||||
if padded_n == n:
|
||||
return bias
|
||||
e = bias.size(0)
|
||||
bias = bias.view(e, 2, n)
|
||||
bias = torch.nn.functional.pad(bias, (0, padded_n - n))
|
||||
return bias.reshape(e, 2 * padded_n).contiguous()
|
||||
|
||||
|
||||
def _process_weights_marlin(
|
||||
layer: torch.nn.Module,
|
||||
input_dtype: torch.dtype | None,
|
||||
@@ -431,6 +460,29 @@ def _process_weights_marlin(
|
||||
marlin_w13_scales = w13_scales
|
||||
marlin_w2_scales = w2_scales
|
||||
|
||||
# --- Pad the intermediate size to a valid Marlin thread tile ---
|
||||
# GPTQ packs along K: w13's N is in the (shard) columns, w2's N in the rows.
|
||||
# Act-order keeps the strict shape and is never padded.
|
||||
N = layer.intermediate_size_per_partition
|
||||
padded_N = marlin_moe_padded_intermediate(N, group_size)
|
||||
if padded_N != N:
|
||||
assert actorder != "group", (
|
||||
"Marlin MoE thread-tile padding is unsupported with act-order"
|
||||
)
|
||||
marlin_w13_qweight = _pad_w13_shard_cols(marlin_w13_qweight, N, padded_N)
|
||||
marlin_w2_qweight = _pad_rows(marlin_w2_qweight, padded_N // pack_factor)
|
||||
marlin_w13_scales = _pad_w13_shard_cols(marlin_w13_scales, N, padded_N)
|
||||
if group_size > 0:
|
||||
marlin_w2_scales = _pad_rows(marlin_w2_scales, padded_N // group_size)
|
||||
if w13_qzeros is not None:
|
||||
w13_qzeros = _pad_w13_shard_cols(
|
||||
w13_qzeros, N // pack_factor, padded_N // pack_factor
|
||||
)
|
||||
if w2_qzeros is not None and group_size > 0:
|
||||
w2_qzeros = _pad_rows(w2_qzeros, padded_N // group_size)
|
||||
if w13_bias is not None:
|
||||
w13_bias = _pad_w13_bias(w13_bias, N, padded_N)
|
||||
|
||||
# --- Process act_order (g_idx) ---
|
||||
if actorder == "group":
|
||||
num_experts = w13_g_idx.shape[0]
|
||||
@@ -608,6 +660,25 @@ def _process_awq_weights_marlin(
|
||||
w13_scales = w13_scales.data * 512
|
||||
w2_scales = w2_scales.data * 512
|
||||
|
||||
# --- Pad the intermediate size to a valid Marlin thread tile ---
|
||||
# AWQ packs along N: w13's N is in the (shard) columns, w2's N in the rows.
|
||||
N = layer.intermediate_size_per_partition
|
||||
padded_N = marlin_moe_padded_intermediate(N, group_size)
|
||||
if padded_N != N:
|
||||
w13_qweight = _pad_w13_shard_cols(
|
||||
w13_qweight, N // pack_factor, padded_N // pack_factor
|
||||
)
|
||||
w2_qweight = _pad_rows(w2_qweight, padded_N)
|
||||
w13_scales = _pad_w13_shard_cols(w13_scales, N, padded_N)
|
||||
w13_qzeros = _pad_w13_shard_cols(
|
||||
w13_qzeros, N // pack_factor, padded_N // pack_factor
|
||||
)
|
||||
if group_size > 0:
|
||||
w2_scales = _pad_rows(w2_scales, padded_N // group_size)
|
||||
w2_qzeros = _pad_rows(w2_qzeros, padded_N // group_size)
|
||||
if w13_bias is not None:
|
||||
w13_bias = _pad_w13_bias(w13_bias, N, padded_N)
|
||||
|
||||
w13_g_idx_sort_indices = torch.nn.Parameter(
|
||||
torch.empty((num_experts, 0), dtype=torch.int32, device=device),
|
||||
requires_grad=False,
|
||||
|
||||
@@ -339,7 +339,9 @@ class AutoAWQConfig(QuantizationConfig):
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
|
||||
if not check_moe_marlin_supports_layer(layer, self.group_size):
|
||||
if not check_moe_marlin_supports_layer(
|
||||
layer, self.group_size, allow_tile_padding=True
|
||||
):
|
||||
logger.warning_once(
|
||||
f"Layer '{prefix}' is not supported by AutoAWQMoEMarlin. "
|
||||
"Falling back to Moe WNA16 kernels."
|
||||
|
||||
@@ -243,7 +243,9 @@ class AutoGPTQConfig(QuantizationConfig):
|
||||
if isinstance(layer, RoutedExperts):
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
if not check_moe_marlin_supports_layer(layer, self.group_size):
|
||||
if not check_moe_marlin_supports_layer(
|
||||
layer, self.group_size, allow_tile_padding=not self.desc_act
|
||||
):
|
||||
logger.warning_once(
|
||||
f"Layer '{prefix}' is not supported by GPTQMoeMarlin. "
|
||||
"Falling back to Moe WNA16 kernels."
|
||||
|
||||
+9
-6
@@ -96,15 +96,18 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
# Prefer to use the MarlinMoE kernel when it is supported.
|
||||
is_actorder = (
|
||||
weight_quant.strategy == QuantizationStrategy.GROUP
|
||||
and weight_quant.actorder
|
||||
in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC)
|
||||
)
|
||||
if (
|
||||
not check_moe_marlin_supports_layer(layer, group_size)
|
||||
not check_moe_marlin_supports_layer(
|
||||
layer, group_size, allow_tile_padding=not is_actorder
|
||||
)
|
||||
or current_platform.is_rocm()
|
||||
):
|
||||
if (
|
||||
weight_quant.strategy == QuantizationStrategy.GROUP
|
||||
and weight_quant.actorder
|
||||
in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC)
|
||||
):
|
||||
if is_actorder:
|
||||
raise ValueError(
|
||||
"WNA16MoE is not supported with actorder=group/dynamic."
|
||||
)
|
||||
|
||||
@@ -330,12 +330,43 @@ def check_marlin_supports_layer(
|
||||
)[0]
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bool:
|
||||
def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1) -> int:
|
||||
"""Smallest MoE intermediate size satisfying the Marlin MoE thread tiles.
|
||||
|
||||
The kernel needs gate-up ``2 * intermediate % 128 == 0`` and down
|
||||
``intermediate % 64 == 0``, i.e. ``intermediate % 64 == 0``. A misaligned
|
||||
size is zero-padded to the next valid tile at weight prep, kept a multiple
|
||||
of ``group_size`` so the group count stays integral. The padded region never
|
||||
reaches the MoE output: w13's padded output channels are zeroed by the
|
||||
zero-padded scales, so the padded inputs to w2 are zero.
|
||||
"""
|
||||
group = group_size if group_size > 0 else 1
|
||||
padded = round_up(intermediate_size, math.lcm(64, group))
|
||||
if padded != intermediate_size:
|
||||
logger.warning_once(
|
||||
"Marlin requires thread-tile padding for the MoE intermediate size "
|
||||
"of some layers in this model. Padded experts pad/slice activations "
|
||||
"on every forward; performance may be degraded."
|
||||
)
|
||||
return padded
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(
|
||||
layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False
|
||||
) -> bool:
|
||||
"""Whether the fused MoE Marlin kernel supports ``layer``.
|
||||
|
||||
Callers without act-order may pass ``allow_tile_padding=True``: a
|
||||
tile-misaligned intermediate size is then zero-padded to a valid thread
|
||||
tile at weight prep (see marlin_moe_padded_intermediate), so only a group
|
||||
straddling the padded boundary stays unsupported. hidden_size is the MoE
|
||||
I/O extent and is never padded. Act-order keeps the strict shape.
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
return False
|
||||
hidden_size = layer.hidden_size
|
||||
# Note: The layer has not performed rounding on intermediate_size's at this
|
||||
# point. Use the unpadded size which won't change.
|
||||
# The layer has not rounded intermediate_size yet; use the stable unpadded
|
||||
# size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64.
|
||||
intermediate_size_per_partition = (
|
||||
layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
)
|
||||
@@ -343,13 +374,15 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
|
||||
# apply_router_weight_on_input is not supported for moe marlin
|
||||
supports_router_weight = not layer.apply_router_weight_on_input
|
||||
|
||||
# gate-up: (n, k) = (intermediate_size_per_partition * 2, hidden_size)
|
||||
# down: (n, k) = (hidden_size, intermediate_size_per_partition)
|
||||
# moe marlin requires n % 128 == 0 and k % 64 == 0
|
||||
supports_shape = (
|
||||
hidden_size % 128 == 0
|
||||
and intermediate_size_per_partition % max(64, group_size) == 0
|
||||
)
|
||||
if allow_tile_padding:
|
||||
supports_shape = hidden_size % 128 == 0 and (
|
||||
group_size <= 0 or intermediate_size_per_partition % group_size == 0
|
||||
)
|
||||
else:
|
||||
supports_shape = (
|
||||
hidden_size % 128 == 0
|
||||
and intermediate_size_per_partition % max(64, group_size) == 0
|
||||
)
|
||||
supports_group_size = group_size in [-1, 32, 64, 128]
|
||||
return supports_shape and supports_group_size and supports_router_weight
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_dim,
|
||||
marlin_pad_qweight,
|
||||
marlin_pad_scales,
|
||||
@@ -216,6 +217,25 @@ def prepare_fp8_layer_for_marlin(
|
||||
replace_parameter(layer, "bias", bias)
|
||||
|
||||
|
||||
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
|
||||
if padded_n == n:
|
||||
return x
|
||||
e = x.size(0)
|
||||
rest = x.shape[2:]
|
||||
x = x.view(e, 2, n, *rest)
|
||||
x = torch.nn.functional.pad(x, (0, 0) * len(rest) + (0, padded_n - n))
|
||||
return x.reshape(e, 2 * padded_n, *rest)
|
||||
|
||||
|
||||
def _moe_pad_last(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad the last dim of a ``(E, ..., n)`` tensor to padded_n."""
|
||||
if padded_n == n:
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, padded_n - n))
|
||||
|
||||
|
||||
def prepare_fp8_moe_layer_for_marlin(
|
||||
layer: torch.nn.Module,
|
||||
w13_weight: torch.Tensor,
|
||||
@@ -246,6 +266,15 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
n = layer.intermediate_size_per_partition
|
||||
w13_n = w13_weight.size(1)
|
||||
weight_block_size = getattr(layer, "weight_block_size", None)
|
||||
group_size = -1 if weight_block_size is None else weight_block_size[1]
|
||||
|
||||
# Pad a tile-misaligned intermediate size to a valid Marlin thread tile.
|
||||
# FP8 zero decodes to 0.0, so padded weights drop out; the converted scales
|
||||
# are padded to match below (the padded values are irrelevant).
|
||||
padded_n = marlin_moe_padded_intermediate(n, group_size)
|
||||
if padded_n != n:
|
||||
w13_weight = _moe_pad_shard_rows(w13_weight, n, padded_n)
|
||||
w2_weight = _moe_pad_last(w2_weight, n, padded_n)
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
@@ -258,13 +287,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
# Repack weights to marlin format
|
||||
def repack_weight(name: str, weight: torch.Tensor) -> torch.Tensor:
|
||||
tensor_list = []
|
||||
if "w13" in name:
|
||||
size_n, size_k = w13_n, k
|
||||
else:
|
||||
size_n, size_k = k, n
|
||||
|
||||
assert weight.shape == (e, size_n, size_k)
|
||||
|
||||
size_n, size_k = weight.size(1), weight.size(2)
|
||||
for i in range(e):
|
||||
qweight = pack_fp8_to_int32(weight[i], size_k_first=False)
|
||||
qweight = qweight.T.contiguous()
|
||||
@@ -280,9 +303,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
w2_weight = repack_weight("w2", w2_weight)
|
||||
|
||||
# WEIGHT SCALES
|
||||
# Permute scales
|
||||
group_size = -1 if weight_block_size is None else weight_block_size[1]
|
||||
|
||||
# Permute scales (convert at the original size, then pad to the tile).
|
||||
def permute_scales(scales: torch.Tensor, name: str) -> torch.Tensor:
|
||||
scales = scales.to(layer.orig_dtype)
|
||||
tensor_list = []
|
||||
@@ -320,6 +341,20 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
# size_n may not divisible by block_size[0]
|
||||
scales = scales[..., :size_n].contiguous()
|
||||
|
||||
# Pad the converted (E, G, size_n) scales to the padded thread tile.
|
||||
if padded_n != n:
|
||||
if "w13" in name:
|
||||
g = scales.size(1)
|
||||
scales = scales.view(e, g, 2, n)
|
||||
scales = torch.nn.functional.pad(scales, (0, padded_n - n))
|
||||
scales = scales.reshape(e, g, 2 * padded_n)
|
||||
size_n = 2 * padded_n
|
||||
else:
|
||||
if group_size > 0:
|
||||
pad_groups = (padded_n - n) // group_size
|
||||
scales = torch.nn.functional.pad(scales, (0, 0, 0, pad_groups))
|
||||
size_k = padded_n
|
||||
|
||||
for i in range(e):
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scales[i], size_k=size_k, size_n=size_n, group_size=group_size
|
||||
@@ -497,10 +532,19 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
"""
|
||||
group_size = 32
|
||||
e = w13.shape[0]
|
||||
w13_n = w13.shape[1]
|
||||
k = w13.shape[2]
|
||||
n = w2.shape[2]
|
||||
|
||||
# Pad a tile-misaligned intermediate size to a valid Marlin thread tile.
|
||||
padded_n = marlin_moe_padded_intermediate(n, group_size)
|
||||
if padded_n != n:
|
||||
w13 = _moe_pad_shard_rows(w13, n, padded_n)
|
||||
w13_scale = _moe_pad_shard_rows(w13_scale, n, padded_n)
|
||||
w2 = _moe_pad_last(w2, n, padded_n)
|
||||
w2_scale = _moe_pad_last(w2_scale, n // group_size, padded_n // group_size)
|
||||
n = padded_n
|
||||
w13_n = w13.shape[1]
|
||||
|
||||
device = w13.device
|
||||
param_dtype = torch.get_default_dtype()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
Reference in New Issue
Block a user