From 1273a8f05a1a3fa90c0edfdf1c6c87e70054e680 Mon Sep 17 00:00:00 2001 From: Xin Yang <105740670+xyang16@users.noreply.github.com> Date: Wed, 24 Jun 2026 18:44:30 -0700 Subject: [PATCH] [Kernel] Add swap AB optimization to fused_moe_kernel (#36559) Signed-off-by: Xin Yang --- .../layers/fused_moe/fused_moe.py | 63 ++++++++++++++----- vllm/model_executor/layers/fused_moe/utils.py | 10 +++ 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 49957c8f5e..a77148de4c 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( moe_align_block_size, ) from vllm.model_executor.layers.fused_moe.utils import ( + enable_swap_ab, moe_kernel_quantize_input, ) from vllm.platforms import current_platform @@ -343,6 +344,7 @@ def fused_moe_kernel( use_int8_w8a16: tl.constexpr, per_channel_quant: tl.constexpr, HAS_BIAS: tl.constexpr, + SWAP_AB: tl.constexpr, ): """ Implements the fused computation for a Mixture of Experts (MOE) using @@ -432,15 +434,25 @@ def fused_moe_kernel( offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N offs_k = tl.arange(0, BLOCK_SIZE_K) - a_ptrs = a_ptr + ( - offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak - ) + if SWAP_AB: + a_ptrs = a_ptr + ( + offs_k[:, None] * stride_ak + offs_token[None, :] // top_k * stride_am + ) + b_ptrs = ( + b_ptr + + off_experts * stride_be + + (offs_bn[:, None] * stride_bn + offs_k[None, :] * stride_bk) + ) + else: + a_ptrs = a_ptr + ( + offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak + ) + b_ptrs = ( + b_ptr + + off_experts * stride_be + + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + ) - b_ptrs = ( - b_ptr - + off_experts * stride_be - + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) - ) if use_int8_w8a16: b_scale_ptrs = ( b_scale_ptr + off_experts * stride_bse + offs_bn[None, :] * stride_bsn @@ -477,16 +489,25 @@ def fused_moe_kernel( # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block # of fp32 values for higher accuracy. # `accumulator` will be converted back to fp16 after the loop. - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + if SWAP_AB: + accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_M), dtype=tl.float32) + else: + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): # Load the next block of A and B, generate a mask by checking the # K dimension. + if SWAP_AB: + a_mask = (offs_k[:, None] < K - k * BLOCK_SIZE_K) & token_mask[None, :] + b_mask = offs_k[None, :] < K - k * BLOCK_SIZE_K + else: + a_mask = token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K) + b_mask = offs_k[:, None] < K - k * BLOCK_SIZE_K a = tl.load( a_ptrs, - mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), + mask=a_mask, other=0.0, ) - b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=b_mask, other=0.0) # We accumulate along the K dimension. if use_int8_w8a16: accumulator = tl.dot(a, b.to(compute_type), acc=accumulator) @@ -498,12 +519,17 @@ def fused_moe_kernel( a_scale_ptrs + offs_ks * stride_ask, mask=token_mask, other=0.0 ) b_scale = tl.load(b_scale_ptrs + offs_ks * stride_bsk) - - accumulator += tl.dot(a, b) * a_scale[:, None] * b_scale[None, :] + if SWAP_AB: + accumulator += tl.dot(b, a) * b_scale[:, None] * a_scale[None, :] + else: + accumulator += tl.dot(a, b) * a_scale[:, None] * b_scale[None, :] else: if use_fp8_w8a8: # acc used to enable fp8_fast_accum - accumulator = tl.dot(a, b, acc=accumulator) + if SWAP_AB: + accumulator = tl.dot(b, a, acc=accumulator) + else: + accumulator = tl.dot(a, b, acc=accumulator) else: accumulator += tl.dot(a, b) else: @@ -512,6 +538,9 @@ def fused_moe_kernel( a_ptrs += BLOCK_SIZE_K * stride_ak b_ptrs += BLOCK_SIZE_K * stride_bk + if SWAP_AB: + accumulator = tl.trans(accumulator, (1, 0)) + # Dequantization for supported quantization schemes: # - int8_w8a16 # - fp8_w8a8 @@ -729,6 +758,11 @@ def invoke_fused_moe_triton_kernel( assert topk_weights is None or topk_weights.stride(1) == 1 assert sorted_token_ids is None or sorted_token_ids.stride(0) == 1 + if use_fp8_w8a8: + SWAP_AB = enable_swap_ab(config["BLOCK_SIZE_M"], config["BLOCK_SIZE_N"]) + else: + SWAP_AB = False + if use_fp8_w8a8 or use_int8_w8a8: assert B_scale is not None assert block_shape is None or triton.cdiv( @@ -810,6 +844,7 @@ def invoke_fused_moe_triton_kernel( naive_block_assignment=(sorted_token_ids is None), HAS_BIAS=HAS_BIAS, BLOCK_SIZE_K=BLOCK_SIZE_K, + SWAP_AB=SWAP_AB, **config, ) diff --git a/vllm/model_executor/layers/fused_moe/utils.py b/vllm/model_executor/layers/fused_moe/utils.py index f356ce6f4f..fce74346d6 100644 --- a/vllm/model_executor/layers/fused_moe/utils.py +++ b/vllm/model_executor/layers/fused_moe/utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools from math import prod import torch @@ -446,3 +447,12 @@ def swiglu_limit_func( up = torch.clamp(up, min=-swiglu_limit, max=swiglu_limit) output.copy_(F.silu(gate) * up) + + +@functools.lru_cache +def enable_swap_ab(BLOCK_SIZE_M: int, BLOCK_SIZE_N: int) -> bool: + return ( + current_platform.is_device_capability(90) + and BLOCK_SIZE_M < 64 + and BLOCK_SIZE_N >= 64 + )