[Kernel] Add swap AB optimization to fused_moe_kernel (#36559)

Signed-off-by: Xin Yang <xyangx@amazon.com>
This commit is contained in:
Xin Yang
2026-06-25 01:44:30 +00:00
committed by GitHub
parent 9e88e969c0
commit 1273a8f05a
2 changed files with 59 additions and 14 deletions
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
from vllm.model_executor.layers.fused_moe.utils import (
enable_swap_ab,
moe_kernel_quantize_input,
)
from vllm.platforms import current_platform
@@ -343,6 +344,7 @@ def fused_moe_kernel(
use_int8_w8a16: tl.constexpr,
per_channel_quant: tl.constexpr,
HAS_BIAS: tl.constexpr,
SWAP_AB: tl.constexpr,
):
"""
Implements the fused computation for a Mixture of Experts (MOE) using
@@ -432,15 +434,25 @@ def fused_moe_kernel(
offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
offs_k = tl.arange(0, BLOCK_SIZE_K)
a_ptrs = a_ptr + (
offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak
)
if SWAP_AB:
a_ptrs = a_ptr + (
offs_k[:, None] * stride_ak + offs_token[None, :] // top_k * stride_am
)
b_ptrs = (
b_ptr
+ off_experts * stride_be
+ (offs_bn[:, None] * stride_bn + offs_k[None, :] * stride_bk)
)
else:
a_ptrs = a_ptr + (
offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak
)
b_ptrs = (
b_ptr
+ off_experts * stride_be
+ (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
)
b_ptrs = (
b_ptr
+ off_experts * stride_be
+ (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
)
if use_int8_w8a16:
b_scale_ptrs = (
b_scale_ptr + off_experts * stride_bse + offs_bn[None, :] * stride_bsn
@@ -477,16 +489,25 @@ def fused_moe_kernel(
# We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block
# of fp32 values for higher accuracy.
# `accumulator` will be converted back to fp16 after the loop.
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
if SWAP_AB:
accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_M), dtype=tl.float32)
else:
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
# Load the next block of A and B, generate a mask by checking the
# K dimension.
if SWAP_AB:
a_mask = (offs_k[:, None] < K - k * BLOCK_SIZE_K) & token_mask[None, :]
b_mask = offs_k[None, :] < K - k * BLOCK_SIZE_K
else:
a_mask = token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K)
b_mask = offs_k[:, None] < K - k * BLOCK_SIZE_K
a = tl.load(
a_ptrs,
mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K),
mask=a_mask,
other=0.0,
)
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0)
b = tl.load(b_ptrs, mask=b_mask, other=0.0)
# We accumulate along the K dimension.
if use_int8_w8a16:
accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
@@ -498,12 +519,17 @@ def fused_moe_kernel(
a_scale_ptrs + offs_ks * stride_ask, mask=token_mask, other=0.0
)
b_scale = tl.load(b_scale_ptrs + offs_ks * stride_bsk)
accumulator += tl.dot(a, b) * a_scale[:, None] * b_scale[None, :]
if SWAP_AB:
accumulator += tl.dot(b, a) * b_scale[:, None] * a_scale[None, :]
else:
accumulator += tl.dot(a, b) * a_scale[:, None] * b_scale[None, :]
else:
if use_fp8_w8a8:
# acc used to enable fp8_fast_accum
accumulator = tl.dot(a, b, acc=accumulator)
if SWAP_AB:
accumulator = tl.dot(b, a, acc=accumulator)
else:
accumulator = tl.dot(a, b, acc=accumulator)
else:
accumulator += tl.dot(a, b)
else:
@@ -512,6 +538,9 @@ def fused_moe_kernel(
a_ptrs += BLOCK_SIZE_K * stride_ak
b_ptrs += BLOCK_SIZE_K * stride_bk
if SWAP_AB:
accumulator = tl.trans(accumulator, (1, 0))
# Dequantization for supported quantization schemes:
# - int8_w8a16
# - fp8_w8a8
@@ -729,6 +758,11 @@ def invoke_fused_moe_triton_kernel(
assert topk_weights is None or topk_weights.stride(1) == 1
assert sorted_token_ids is None or sorted_token_ids.stride(0) == 1
if use_fp8_w8a8:
SWAP_AB = enable_swap_ab(config["BLOCK_SIZE_M"], config["BLOCK_SIZE_N"])
else:
SWAP_AB = False
if use_fp8_w8a8 or use_int8_w8a8:
assert B_scale is not None
assert block_shape is None or triton.cdiv(
@@ -810,6 +844,7 @@ def invoke_fused_moe_triton_kernel(
naive_block_assignment=(sorted_token_ids is None),
HAS_BIAS=HAS_BIAS,
BLOCK_SIZE_K=BLOCK_SIZE_K,
SWAP_AB=SWAP_AB,
**config,
)
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from math import prod
import torch
@@ -446,3 +447,12 @@ def swiglu_limit_func(
up = torch.clamp(up, min=-swiglu_limit, max=swiglu_limit)
output.copy_(F.silu(gate) * up)
@functools.lru_cache
def enable_swap_ab(BLOCK_SIZE_M: int, BLOCK_SIZE_N: int) -> bool:
return (
current_platform.is_device_capability(90)
and BLOCK_SIZE_M < 64
and BLOCK_SIZE_N >= 64
)