From 37a682d392330da665690ee0a77c9d0a875f315f Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Tue, 23 Jun 2026 12:45:10 -0600 Subject: [PATCH] [Kernel] Extend Marlin thread-tile padding to MoE (WNA16 + FP8/MXFP8) (#45703) Signed-off-by: mgoin Co-authored-by: Claude --- .../quantization/test_marlin_tile_padding.py | 389 ++++++++++++++++++ .../layers/fused_moe/oracle/int_wna16.py | 71 ++++ .../layers/quantization/auto_awq.py | 4 +- .../layers/quantization/auto_gptq.py | 4 +- .../compressed_tensors_moe.py | 15 +- .../layers/quantization/utils/marlin_utils.py | 53 ++- .../quantization/utils/marlin_utils_fp8.py | 66 ++- 7 files changed, 573 insertions(+), 29 deletions(-) diff --git a/tests/kernels/quantization/test_marlin_tile_padding.py b/tests/kernels/quantization/test_marlin_tile_padding.py index 62b18d88ac..649e12c66b 100644 --- a/tests/kernels/quantization/test_marlin_tile_padding.py +++ b/tests/kernels/quantization/test_marlin_tile_padding.py @@ -5,6 +5,8 @@ Run `pytest tests/kernels/quantization/test_marlin_tile_padding.py`. """ +from types import SimpleNamespace + import pytest import torch @@ -14,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import ( apply_gptq_marlin_linear, marlin_make_empty_g_idx, marlin_make_workspace_new, + marlin_moe_padded_intermediate, marlin_pad_qweight, marlin_pad_scales, marlin_padded_nk, @@ -115,6 +118,74 @@ def test_marlin_pad_helpers_shapes(): assert padded.shape == (1, padded_n) +# Rank-local MoE intermediate sizes. group<=0 / 32 with a non-multiple-of-64 +# size is where tile padding triggers; 64/128 are already tile-aligned. +MOE_INTERMEDIATE_SIZES = [64, 96, 100, 176, 192, 256, 2816] + + +@pytest.mark.parametrize("intermediate", MOE_INTERMEDIATE_SIZES) +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128]) +def test_marlin_moe_padded_intermediate(intermediate, group_size): + # The MoE gate only admits shapes where the group does not straddle the + # boundary, i.e. group divides the intermediate size. + if group_size > 0 and intermediate % group_size != 0: + pytest.skip("group straddles the boundary; rejected by the MoE gate") + + padded = marlin_moe_padded_intermediate(intermediate, group_size) + assert padded >= intermediate + # Valid MoE thread tile: gate-up n = 2*intermediate % 128, down k % 64. + assert (2 * padded) % 128 == 0 + assert padded % 64 == 0 + if group_size > 0: + assert padded % group_size == 0 + + # Minimal: no smaller valid intermediate exists. + for cand in range(intermediate, padded): + if ( + (2 * cand) % 128 == 0 + and cand % 64 == 0 + and (group_size <= 0 or cand % group_size == 0) + ): + pytest.fail(f"{cand} beats {padded}") + + # Already-tile-aligned sizes pass through unchanged (zero hot-path cost). + if intermediate % 64 == 0: + assert padded == intermediate + + +def test_marlin_moe_pad_helpers_shapes(): + from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( + _pad_rows, + _pad_w13_bias, + _pad_w13_shard_cols, + ) + + E, rows, N, padded_N = 2, 8, 96, 128 + + # w13 stores the two gate/up shards along the last dim; padding each shard + # must preserve the loaded values and zero the padded columns. + w13 = torch.arange(E * rows * 2 * N).reshape(E, rows, 2 * N).float() + padded = _pad_w13_shard_cols(w13, N, padded_N) + assert padded.shape == (E, rows, 2 * padded_N) + shards = padded.view(E, rows, 2, padded_N) + orig = w13.view(E, rows, 2, N) + assert torch.equal(shards[..., :N], orig) + assert shards[..., N:].abs().sum() == 0 + + # w2 stores the intermediate dim in the rows. + w2 = torch.ones(E, N // 32, 16) + padded = _pad_rows(w2, padded_N // 32) + assert padded.shape == (E, padded_N // 32, 16) + assert padded[:, N // 32 :, :].abs().sum() == 0 + + bias = torch.arange(E * 2 * N).reshape(E, 2 * N).float() + padded = _pad_w13_bias(bias, N, padded_N) + assert padded.shape == (E, 2 * padded_N) + bias_shards = padded.view(E, 2, padded_N) + assert torch.equal(bias_shards[..., :N], bias.view(E, 2, N)) + assert bias_shards[..., N:].abs().sum() == 0 + + def _gpu_marlin_unsupported() -> bool: return not ( current_platform.is_cuda() and current_platform.has_device_capability(80) @@ -468,3 +539,321 @@ def test_check_marlin_supports_layer_allow_tile_padding(): # A group straddling the TP shard cannot be fixed by padding layer = _FakeLinear(4608, 4672, input_size=18688) assert not check_marlin_supports_layer(layer, 128, allow_tile_padding=True) + + +@pytest.mark.skipif( + _gpu_marlin_unsupported(), + reason="Marlin is not supported on this GPU type.", +) +@pytest.mark.parametrize("group_size", [-1, 32]) +@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)]) +def test_gptq_marlin_moe_padded_round_trip(shape, group_size): + """Pad a tile-misaligned MoE intermediate the way the WNA16 Marlin MoE prep + does, run the real repack + fused_marlin_moe, and check against the + dequantized reference. Symmetric int4's quantized zero decodes to -8, so the + padded region only stays out of the output via the zero-padded scales. + """ + from tests.kernels.utils import torch_experts + from vllm.config import VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe import fused_topk + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( + fused_marlin_moe, + ) + from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( + _pad_rows, + _pad_w13_shard_cols, + ) + from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + marlin_moe_padded_intermediate, + marlin_moe_permute_scales, + ) + + n, k, e = shape + topk, m = 2, 33 + padded_n = marlin_moe_padded_intermediate(n, group_size) + assert padded_n != n, "test should exercise padding" + + dtype = torch.float16 + device = torch.device("cuda") + quant_type = scalar_types.uint4b8 + bits = quant_type.size_bits + pack = 32 // bits + + a = torch.randn((m, k), device=device, dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5 + w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5 + + def quant(w, size_k, size_n): + # w is (size_n, size_k); gptq expects (size_k, size_n). + ref, q_w, s, _, _ = gptq_quantize_weights( + w.T, quant_type, group_size, act_order=False + ) + return ref, gptq_pack(q_w, bits, size_k, size_n), s + + w13_qw, w13_s, w13_ref = [], [], [] + w2_qw, w2_s, w2_ref = [], [], [] + for i in range(e): + ref, qw, s = quant(w1[i], k, 2 * n) + w13_ref.append(ref.T) # (2n, k) + w13_qw.append(qw) + w13_s.append(s) + ref, qw, s = quant(w2[i], n, k) + w2_ref.append(ref.T) # (k, n) + w2_qw.append(qw) + w2_s.append(s) + + w13_qweight = torch.stack(w13_qw) + w2_qweight = torch.stack(w2_qw) + w13_scales = torch.stack(w13_s) + w2_scales = torch.stack(w2_s) + w1_ref = torch.stack(w13_ref) # (e, 2n, k) + w2_ref = torch.stack(w2_ref) # (e, k, n) + + # Pad the intermediate via the production helpers. + w13_qweight = _pad_w13_shard_cols(w13_qweight, n, padded_n) + w2_qweight = _pad_rows(w2_qweight, padded_n // pack) + w13_scales = _pad_w13_shard_cols(w13_scales, n, padded_n) + if group_size > 0: + w2_scales = _pad_rows(w2_scales, padded_n // group_size) + + sort_idx = torch.empty((e, 0), dtype=torch.int32, device=device) + marlin_w13 = ops.gptq_marlin_moe_repack( + w13_qweight, sort_idx, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits + ) + marlin_w2 = ops.gptq_marlin_moe_repack( + w2_qweight, sort_idx, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits + ) + group_or_pack = group_size if group_size != -1 else pack + marlin_w13_s = marlin_moe_permute_scales( + s=w13_scales, size_k=n, size_n=w13_scales.shape[2], group_size=group_size + ) + marlin_w2_s = marlin_moe_permute_scales( + s=w2_scales, + size_k=w2_scales.shape[1] * group_or_pack, + size_n=w2_scales.shape[2], + group_size=group_size, + ) + + score = torch.randn((m, e), device=device, dtype=dtype) + topk_weights, topk_ids, _ = fused_topk(a, score, topk, False) + + marlin_out = fused_marlin_moe( + a, + marlin_w13, + marlin_w2, + None, + None, + marlin_w13_s, + marlin_w2_s, + topk_weights, + topk_ids, + quant_type_id=quant_type.id, + global_num_experts=e, + is_k_full=True, + ) + with set_current_vllm_config(VllmConfig()): + ref = torch_experts( + a, + w1_ref, + w2_ref, + topk_weight=topk_weights, + topk_ids=topk_ids, + global_num_experts=e, + ) + + torch.testing.assert_close(marlin_out, ref, atol=5e-2, rtol=0) + + +def test_check_moe_marlin_supports_layer_padding(): + from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + check_moe_marlin_supports_layer, + ) + + def make_layer(hidden, intermediate): + layer = SimpleNamespace() + layer.hidden_size = hidden + layer.apply_router_weight_on_input = False + layer.moe_config = SimpleNamespace( + intermediate_size_per_partition_unpadded=intermediate + ) + return layer + + # group=32 with intermediate % 64 != 0: rejected strictly, accepted w/ padding + layer = make_layer(4096, 96) + assert not check_moe_marlin_supports_layer(layer, 32) + assert check_moe_marlin_supports_layer(layer, 32, allow_tile_padding=True) + # channelwise misaligned intermediate is paddable + assert check_moe_marlin_supports_layer(layer, -1, allow_tile_padding=True) + + # A group straddling the boundary cannot be fixed by padding + layer = make_layer(4096, 176) + assert not check_moe_marlin_supports_layer(layer, 128, allow_tile_padding=True) + + # hidden_size is the MoE I/O extent and is never padded + layer = make_layer(4090, 128) + assert not check_moe_marlin_supports_layer(layer, 64, allow_tile_padding=True) + + +@pytest.mark.skipif( + _gpu_marlin_unsupported() or not is_fp8_marlin_supported(), + reason="FP8 Marlin is not supported on this GPU type.", +) +@pytest.mark.parametrize("quant", ["channel", "tensor"]) +@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)]) +def test_fp8_marlin_moe_padded_round_trip(shape, quant): + """FP8 weight-only MoE: pad a tile-misaligned intermediate and check the + real prepare + fused_marlin_moe against the dequantized reference.""" + from tests.kernels.utils import torch_experts + from vllm.config import VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe import fused_topk + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( + fused_marlin_moe, + ) + from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + marlin_moe_intermediate_size, + marlin_moe_padded_intermediate, + ) + from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( + prepare_fp8_moe_layer_for_marlin, + ) + + n, k, e = shape + topk, m = 2, 33 + fp8 = torch.float8_e4m3fn + dtype = torch.bfloat16 + device = torch.device("cuda") + padded_n = marlin_moe_padded_intermediate(n, -1) + assert padded_n != n + + def q(w): # (out, in) -> fp8 weight, scale, dequant reference + dim = None if quant == "tensor" else 1 + s = (w.abs().amax(dim, keepdim=dim is not None) / 448.0).clamp(min=1e-8) + wq = (w / s).clamp(-448, 448).to(fp8) + ref = wq.to(dtype) * s.to(dtype) + s = s.reshape(1) if quant == "tensor" else s.squeeze(1) + return wq, s, ref + + a = torch.randn((m, k), device=device, dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5 + w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5 + w13_q, w13_s, w1_ref = zip(*(q(w1[i]) for i in range(e))) + w2_q, w2_s, w2_ref = zip(*(q(w2[i]) for i in range(e))) + + w13_weight, w2_weight = torch.stack(w13_q), torch.stack(w2_q) + layer = SimpleNamespace( + num_experts=e, + hidden_size=k, + intermediate_size_per_partition=n, + orig_dtype=dtype, + w13_weight=w13_weight, + ) + pw13, pw2, ps13, ps2 = prepare_fp8_moe_layer_for_marlin( + layer, w13_weight, w2_weight, torch.stack(w13_s), torch.stack(w2_s) + ) + assert marlin_moe_intermediate_size(pw13, pw2) == padded_n + + score = torch.randn((m, e), device=device, dtype=dtype) + topk_weights, topk_ids, _ = fused_topk(a, score, topk, False) + out = fused_marlin_moe( + a, + pw13, + pw2, + None, + None, + ps13, + ps2, + topk_weights, + topk_ids, + quant_type_id=scalar_types.float8_e4m3fn.id, + global_num_experts=e, + is_k_full=True, + workspace=layer.workspace, + ) + with set_current_vllm_config(VllmConfig()): + ref = torch_experts( + a, + torch.stack(w1_ref), + torch.stack(w2_ref), + topk_weight=topk_weights, + topk_ids=topk_ids, + global_num_experts=e, + ) + torch.testing.assert_close(out, ref, atol=8e-2, rtol=0) + + +@pytest.mark.skipif( + _gpu_marlin_unsupported() or not is_fp8_marlin_supported(), + reason="FP8 Marlin is not supported on this GPU type.", +) +@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)]) +def test_mxfp8_marlin_moe_padded_round_trip(shape): + """MXFP8 weight-only MoE round-trip at a tile-misaligned intermediate, with + unit e8m0 scales so the reference is the exact fp8 dequant.""" + from tests.kernels.utils import torch_experts + from vllm.config import VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe import fused_topk + from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( + fused_marlin_moe, + ) + from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + marlin_moe_intermediate_size, + marlin_moe_padded_intermediate, + ) + from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( + prepare_mxfp8_moe_layer_for_marlin, + ) + + n, k, e = shape + topk, m, gs, e8m0_one = 2, 33, 32, 127 + fp8 = torch.float8_e4m3fn + dtype = torch.bfloat16 + device = torch.device("cuda") + padded_n = marlin_moe_padded_intermediate(n, gs) + assert padded_n != n + + a = torch.randn((m, k), device=device, dtype=dtype) / 10 + w13_weight = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5 + w2_weight = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5 + w13_weight = w13_weight.clamp(-448, 448).to(fp8) + w2_weight = w2_weight.clamp(-448, 448).to(fp8) + w13_scale = torch.full( + (e, 2 * n, k // gs), e8m0_one, dtype=torch.uint8, device=device + ) + w2_scale = torch.full((e, k, n // gs), e8m0_one, dtype=torch.uint8, device=device) + + layer = SimpleNamespace( + num_experts=e, hidden_size=k, intermediate_size_per_partition=n + ) + with set_current_vllm_config(VllmConfig()): + pw13, pw2, ps13, ps2 = prepare_mxfp8_moe_layer_for_marlin( + layer, w13_weight, w2_weight, w13_scale, w2_scale + ) + assert marlin_moe_intermediate_size(pw13, pw2) == padded_n + + score = torch.randn((m, e), device=device, dtype=dtype) + topk_weights, topk_ids, _ = fused_topk(a, score, topk, False) + out = fused_marlin_moe( + a, + pw13, + pw2, + None, + None, + ps13, + ps2, + topk_weights, + topk_ids, + quant_type_id=scalar_types.float8_e4m3fn.id, + global_num_experts=e, + is_k_full=True, + workspace=layer.workspace, + ) + with set_current_vllm_config(VllmConfig()): + ref = torch_experts( + a, + w13_weight.to(dtype), + w2_weight.to(dtype), + topk_weight=topk_weights, + topk_ids=topk_ids, + global_num_experts=e, + ) + torch.testing.assert_close(out, ref, atol=8e-2, rtol=0) diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index cbd12b3e60..0cf1382d40 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import ( from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.quantization.utils.marlin_utils import ( marlin_act_int8_process_scales, + marlin_moe_padded_intermediate, marlin_moe_permute_scales, marlin_permute_bias, moe_awq_to_marlin_zero_points, @@ -361,6 +362,34 @@ def _process_weights_flashinfer( ) +def _pad_w13_shard_cols(x: torch.Tensor, unit: int, padded_unit: int) -> torch.Tensor: + """Zero-pad each of the two gate/up shards of a ``(E, rows, 2 * unit)`` + tensor along its last dim, from ``unit`` to ``padded_unit`` columns.""" + if padded_unit == unit: + return x + e, rows, _ = x.shape + x = x.view(e, rows, 2, unit) + x = torch.nn.functional.pad(x, (0, padded_unit - unit)) + return x.reshape(e, rows, 2 * padded_unit).contiguous() + + +def _pad_rows(x: torch.Tensor, padded_rows: int) -> torch.Tensor: + """Zero-pad a ``(E, rows, cols)`` tensor to ``padded_rows`` rows.""" + if padded_rows == x.size(1): + return x + return torch.nn.functional.pad(x, (0, 0, 0, padded_rows - x.size(1))) + + +def _pad_w13_bias(bias: torch.Tensor, n: int, padded_n: int) -> torch.Tensor: + """Zero-pad each gate/up shard of a ``(E, 2 * n)`` bias to ``padded_n``.""" + if padded_n == n: + return bias + e = bias.size(0) + bias = bias.view(e, 2, n) + bias = torch.nn.functional.pad(bias, (0, padded_n - n)) + return bias.reshape(e, 2 * padded_n).contiguous() + + def _process_weights_marlin( layer: torch.nn.Module, input_dtype: torch.dtype | None, @@ -431,6 +460,29 @@ def _process_weights_marlin( marlin_w13_scales = w13_scales marlin_w2_scales = w2_scales + # --- Pad the intermediate size to a valid Marlin thread tile --- + # GPTQ packs along K: w13's N is in the (shard) columns, w2's N in the rows. + # Act-order keeps the strict shape and is never padded. + N = layer.intermediate_size_per_partition + padded_N = marlin_moe_padded_intermediate(N, group_size) + if padded_N != N: + assert actorder != "group", ( + "Marlin MoE thread-tile padding is unsupported with act-order" + ) + marlin_w13_qweight = _pad_w13_shard_cols(marlin_w13_qweight, N, padded_N) + marlin_w2_qweight = _pad_rows(marlin_w2_qweight, padded_N // pack_factor) + marlin_w13_scales = _pad_w13_shard_cols(marlin_w13_scales, N, padded_N) + if group_size > 0: + marlin_w2_scales = _pad_rows(marlin_w2_scales, padded_N // group_size) + if w13_qzeros is not None: + w13_qzeros = _pad_w13_shard_cols( + w13_qzeros, N // pack_factor, padded_N // pack_factor + ) + if w2_qzeros is not None and group_size > 0: + w2_qzeros = _pad_rows(w2_qzeros, padded_N // group_size) + if w13_bias is not None: + w13_bias = _pad_w13_bias(w13_bias, N, padded_N) + # --- Process act_order (g_idx) --- if actorder == "group": num_experts = w13_g_idx.shape[0] @@ -608,6 +660,25 @@ def _process_awq_weights_marlin( w13_scales = w13_scales.data * 512 w2_scales = w2_scales.data * 512 + # --- Pad the intermediate size to a valid Marlin thread tile --- + # AWQ packs along N: w13's N is in the (shard) columns, w2's N in the rows. + N = layer.intermediate_size_per_partition + padded_N = marlin_moe_padded_intermediate(N, group_size) + if padded_N != N: + w13_qweight = _pad_w13_shard_cols( + w13_qweight, N // pack_factor, padded_N // pack_factor + ) + w2_qweight = _pad_rows(w2_qweight, padded_N) + w13_scales = _pad_w13_shard_cols(w13_scales, N, padded_N) + w13_qzeros = _pad_w13_shard_cols( + w13_qzeros, N // pack_factor, padded_N // pack_factor + ) + if group_size > 0: + w2_scales = _pad_rows(w2_scales, padded_N // group_size) + w2_qzeros = _pad_rows(w2_qzeros, padded_N // group_size) + if w13_bias is not None: + w13_bias = _pad_w13_bias(w13_bias, N, padded_N) + w13_g_idx_sort_indices = torch.nn.Parameter( torch.empty((num_experts, 0), dtype=torch.int32, device=device), requires_grad=False, diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index a524c8c193..cebfad7e59 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -339,7 +339,9 @@ class AutoAWQConfig(QuantizationConfig): ): return UnquantizedFusedMoEMethod(layer.moe_config) - if not check_moe_marlin_supports_layer(layer, self.group_size): + if not check_moe_marlin_supports_layer( + layer, self.group_size, allow_tile_padding=True + ): logger.warning_once( f"Layer '{prefix}' is not supported by AutoAWQMoEMarlin. " "Falling back to Moe WNA16 kernels." diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py index f7fe7f6e9e..aca76162f9 100644 --- a/vllm/model_executor/layers/quantization/auto_gptq.py +++ b/vllm/model_executor/layers/quantization/auto_gptq.py @@ -243,7 +243,9 @@ class AutoGPTQConfig(QuantizationConfig): if isinstance(layer, RoutedExperts): from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config - if not check_moe_marlin_supports_layer(layer, self.group_size): + if not check_moe_marlin_supports_layer( + layer, self.group_size, allow_tile_padding=not self.desc_act + ): logger.warning_once( f"Layer '{prefix}' is not supported by GPTQMoeMarlin. " "Falling back to Moe WNA16 kernels." diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py index 2e45e0f298..0022148523 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe.py @@ -96,15 +96,18 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase): ) # Prefer to use the MarlinMoE kernel when it is supported. + is_actorder = ( + weight_quant.strategy == QuantizationStrategy.GROUP + and weight_quant.actorder + in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC) + ) if ( - not check_moe_marlin_supports_layer(layer, group_size) + not check_moe_marlin_supports_layer( + layer, group_size, allow_tile_padding=not is_actorder + ) or current_platform.is_rocm() ): - if ( - weight_quant.strategy == QuantizationStrategy.GROUP - and weight_quant.actorder - in (ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC) - ): + if is_actorder: raise ValueError( "WNA16MoE is not supported with actorder=group/dynamic." ) diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils.py b/vllm/model_executor/layers/quantization/utils/marlin_utils.py index 1aba32621f..cd6fae8cf2 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils.py @@ -330,12 +330,43 @@ def check_marlin_supports_layer( )[0] -def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bool: +def marlin_moe_padded_intermediate(intermediate_size: int, group_size: int = -1) -> int: + """Smallest MoE intermediate size satisfying the Marlin MoE thread tiles. + + The kernel needs gate-up ``2 * intermediate % 128 == 0`` and down + ``intermediate % 64 == 0``, i.e. ``intermediate % 64 == 0``. A misaligned + size is zero-padded to the next valid tile at weight prep, kept a multiple + of ``group_size`` so the group count stays integral. The padded region never + reaches the MoE output: w13's padded output channels are zeroed by the + zero-padded scales, so the padded inputs to w2 are zero. + """ + group = group_size if group_size > 0 else 1 + padded = round_up(intermediate_size, math.lcm(64, group)) + if padded != intermediate_size: + logger.warning_once( + "Marlin requires thread-tile padding for the MoE intermediate size " + "of some layers in this model. Padded experts pad/slice activations " + "on every forward; performance may be degraded." + ) + return padded + + +def check_moe_marlin_supports_layer( + layer: RoutedExperts, group_size: int, allow_tile_padding: bool = False +) -> bool: + """Whether the fused MoE Marlin kernel supports ``layer``. + + Callers without act-order may pass ``allow_tile_padding=True``: a + tile-misaligned intermediate size is then zero-padded to a valid thread + tile at weight prep (see marlin_moe_padded_intermediate), so only a group + straddling the padded boundary stays unsupported. hidden_size is the MoE + I/O extent and is never padded. Act-order keeps the strict shape. + """ if current_platform.is_rocm(): return False hidden_size = layer.hidden_size - # Note: The layer has not performed rounding on intermediate_size's at this - # point. Use the unpadded size which won't change. + # The layer has not rounded intermediate_size yet; use the stable unpadded + # size. gate-up needs n=2*intermediate % 128, down needs k=intermediate % 64. intermediate_size_per_partition = ( layer.moe_config.intermediate_size_per_partition_unpadded ) @@ -343,13 +374,15 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo # apply_router_weight_on_input is not supported for moe marlin supports_router_weight = not layer.apply_router_weight_on_input - # gate-up: (n, k) = (intermediate_size_per_partition * 2, hidden_size) - # down: (n, k) = (hidden_size, intermediate_size_per_partition) - # moe marlin requires n % 128 == 0 and k % 64 == 0 - supports_shape = ( - hidden_size % 128 == 0 - and intermediate_size_per_partition % max(64, group_size) == 0 - ) + if allow_tile_padding: + supports_shape = hidden_size % 128 == 0 and ( + group_size <= 0 or intermediate_size_per_partition % group_size == 0 + ) + else: + supports_shape = ( + hidden_size % 128 == 0 + and intermediate_size_per_partition % max(64, group_size) == 0 + ) supports_group_size = group_size in [-1, 32, 64, 128] return supports_shape and supports_group_size and supports_router_weight diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py index 02f1423279..739f76659c 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py @@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import ( USE_FP32_REDUCE_DEFAULT, get_marlin_input_dtype, marlin_make_workspace_new, + marlin_moe_padded_intermediate, marlin_pad_dim, marlin_pad_qweight, marlin_pad_scales, @@ -216,6 +217,25 @@ def prepare_fp8_layer_for_marlin( replace_parameter(layer, "bias", bias) +def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor: + """Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n + rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.""" + if padded_n == n: + return x + e = x.size(0) + rest = x.shape[2:] + x = x.view(e, 2, n, *rest) + x = torch.nn.functional.pad(x, (0, 0) * len(rest) + (0, padded_n - n)) + return x.reshape(e, 2 * padded_n, *rest) + + +def _moe_pad_last(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor: + """Zero-pad the last dim of a ``(E, ..., n)`` tensor to padded_n.""" + if padded_n == n: + return x + return torch.nn.functional.pad(x, (0, padded_n - n)) + + def prepare_fp8_moe_layer_for_marlin( layer: torch.nn.Module, w13_weight: torch.Tensor, @@ -246,6 +266,15 @@ def prepare_fp8_moe_layer_for_marlin( n = layer.intermediate_size_per_partition w13_n = w13_weight.size(1) weight_block_size = getattr(layer, "weight_block_size", None) + group_size = -1 if weight_block_size is None else weight_block_size[1] + + # Pad a tile-misaligned intermediate size to a valid Marlin thread tile. + # FP8 zero decodes to 0.0, so padded weights drop out; the converted scales + # are padded to match below (the padded values are irrelevant). + padded_n = marlin_moe_padded_intermediate(n, group_size) + if padded_n != n: + w13_weight = _moe_pad_shard_rows(w13_weight, n, padded_n) + w2_weight = _moe_pad_last(w2_weight, n, padded_n) # WORKSPACE device = layer.w13_weight.device @@ -258,13 +287,7 @@ def prepare_fp8_moe_layer_for_marlin( # Repack weights to marlin format def repack_weight(name: str, weight: torch.Tensor) -> torch.Tensor: tensor_list = [] - if "w13" in name: - size_n, size_k = w13_n, k - else: - size_n, size_k = k, n - - assert weight.shape == (e, size_n, size_k) - + size_n, size_k = weight.size(1), weight.size(2) for i in range(e): qweight = pack_fp8_to_int32(weight[i], size_k_first=False) qweight = qweight.T.contiguous() @@ -280,9 +303,7 @@ def prepare_fp8_moe_layer_for_marlin( w2_weight = repack_weight("w2", w2_weight) # WEIGHT SCALES - # Permute scales - group_size = -1 if weight_block_size is None else weight_block_size[1] - + # Permute scales (convert at the original size, then pad to the tile). def permute_scales(scales: torch.Tensor, name: str) -> torch.Tensor: scales = scales.to(layer.orig_dtype) tensor_list = [] @@ -320,6 +341,20 @@ def prepare_fp8_moe_layer_for_marlin( # size_n may not divisible by block_size[0] scales = scales[..., :size_n].contiguous() + # Pad the converted (E, G, size_n) scales to the padded thread tile. + if padded_n != n: + if "w13" in name: + g = scales.size(1) + scales = scales.view(e, g, 2, n) + scales = torch.nn.functional.pad(scales, (0, padded_n - n)) + scales = scales.reshape(e, g, 2 * padded_n) + size_n = 2 * padded_n + else: + if group_size > 0: + pad_groups = (padded_n - n) // group_size + scales = torch.nn.functional.pad(scales, (0, 0, 0, pad_groups)) + size_k = padded_n + for i in range(e): marlin_scales = marlin_permute_scales( s=scales[i], size_k=size_k, size_n=size_n, group_size=group_size @@ -497,10 +532,19 @@ def prepare_mxfp8_moe_layer_for_marlin( """ group_size = 32 e = w13.shape[0] - w13_n = w13.shape[1] k = w13.shape[2] n = w2.shape[2] + # Pad a tile-misaligned intermediate size to a valid Marlin thread tile. + padded_n = marlin_moe_padded_intermediate(n, group_size) + if padded_n != n: + w13 = _moe_pad_shard_rows(w13, n, padded_n) + w13_scale = _moe_pad_shard_rows(w13_scale, n, padded_n) + w2 = _moe_pad_last(w2, n, padded_n) + w2_scale = _moe_pad_last(w2_scale, n // group_size, padded_n // group_size) + n = padded_n + w13_n = w13.shape[1] + device = w13.device param_dtype = torch.get_default_dtype() perm = torch.empty(0, dtype=torch.int, device=device)