[Kernel] Add swap AB optimization to fused_moe_kernel (#36559)
Signed-off-by: Xin Yang <xyangx@amazon.com>
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
enable_swap_ab,
|
||||
moe_kernel_quantize_input,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -343,6 +344,7 @@ def fused_moe_kernel(
|
||||
use_int8_w8a16: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
SWAP_AB: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
@@ -432,15 +434,25 @@ def fused_moe_kernel(
|
||||
|
||||
offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N
|
||||
offs_k = tl.arange(0, BLOCK_SIZE_K)
|
||||
a_ptrs = a_ptr + (
|
||||
offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak
|
||||
)
|
||||
if SWAP_AB:
|
||||
a_ptrs = a_ptr + (
|
||||
offs_k[:, None] * stride_ak + offs_token[None, :] // top_k * stride_am
|
||||
)
|
||||
b_ptrs = (
|
||||
b_ptr
|
||||
+ off_experts * stride_be
|
||||
+ (offs_bn[:, None] * stride_bn + offs_k[None, :] * stride_bk)
|
||||
)
|
||||
else:
|
||||
a_ptrs = a_ptr + (
|
||||
offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak
|
||||
)
|
||||
b_ptrs = (
|
||||
b_ptr
|
||||
+ off_experts * stride_be
|
||||
+ (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
|
||||
)
|
||||
|
||||
b_ptrs = (
|
||||
b_ptr
|
||||
+ off_experts * stride_be
|
||||
+ (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
|
||||
)
|
||||
if use_int8_w8a16:
|
||||
b_scale_ptrs = (
|
||||
b_scale_ptr + off_experts * stride_bse + offs_bn[None, :] * stride_bsn
|
||||
@@ -477,16 +489,25 @@ def fused_moe_kernel(
|
||||
# We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block
|
||||
# of fp32 values for higher accuracy.
|
||||
# `accumulator` will be converted back to fp16 after the loop.
|
||||
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
|
||||
if SWAP_AB:
|
||||
accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_M), dtype=tl.float32)
|
||||
else:
|
||||
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
|
||||
for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
|
||||
# Load the next block of A and B, generate a mask by checking the
|
||||
# K dimension.
|
||||
if SWAP_AB:
|
||||
a_mask = (offs_k[:, None] < K - k * BLOCK_SIZE_K) & token_mask[None, :]
|
||||
b_mask = offs_k[None, :] < K - k * BLOCK_SIZE_K
|
||||
else:
|
||||
a_mask = token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K)
|
||||
b_mask = offs_k[:, None] < K - k * BLOCK_SIZE_K
|
||||
a = tl.load(
|
||||
a_ptrs,
|
||||
mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K),
|
||||
mask=a_mask,
|
||||
other=0.0,
|
||||
)
|
||||
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0)
|
||||
b = tl.load(b_ptrs, mask=b_mask, other=0.0)
|
||||
# We accumulate along the K dimension.
|
||||
if use_int8_w8a16:
|
||||
accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
|
||||
@@ -498,12 +519,17 @@ def fused_moe_kernel(
|
||||
a_scale_ptrs + offs_ks * stride_ask, mask=token_mask, other=0.0
|
||||
)
|
||||
b_scale = tl.load(b_scale_ptrs + offs_ks * stride_bsk)
|
||||
|
||||
accumulator += tl.dot(a, b) * a_scale[:, None] * b_scale[None, :]
|
||||
if SWAP_AB:
|
||||
accumulator += tl.dot(b, a) * b_scale[:, None] * a_scale[None, :]
|
||||
else:
|
||||
accumulator += tl.dot(a, b) * a_scale[:, None] * b_scale[None, :]
|
||||
else:
|
||||
if use_fp8_w8a8:
|
||||
# acc used to enable fp8_fast_accum
|
||||
accumulator = tl.dot(a, b, acc=accumulator)
|
||||
if SWAP_AB:
|
||||
accumulator = tl.dot(b, a, acc=accumulator)
|
||||
else:
|
||||
accumulator = tl.dot(a, b, acc=accumulator)
|
||||
else:
|
||||
accumulator += tl.dot(a, b)
|
||||
else:
|
||||
@@ -512,6 +538,9 @@ def fused_moe_kernel(
|
||||
a_ptrs += BLOCK_SIZE_K * stride_ak
|
||||
b_ptrs += BLOCK_SIZE_K * stride_bk
|
||||
|
||||
if SWAP_AB:
|
||||
accumulator = tl.trans(accumulator, (1, 0))
|
||||
|
||||
# Dequantization for supported quantization schemes:
|
||||
# - int8_w8a16
|
||||
# - fp8_w8a8
|
||||
@@ -729,6 +758,11 @@ def invoke_fused_moe_triton_kernel(
|
||||
assert topk_weights is None or topk_weights.stride(1) == 1
|
||||
assert sorted_token_ids is None or sorted_token_ids.stride(0) == 1
|
||||
|
||||
if use_fp8_w8a8:
|
||||
SWAP_AB = enable_swap_ab(config["BLOCK_SIZE_M"], config["BLOCK_SIZE_N"])
|
||||
else:
|
||||
SWAP_AB = False
|
||||
|
||||
if use_fp8_w8a8 or use_int8_w8a8:
|
||||
assert B_scale is not None
|
||||
assert block_shape is None or triton.cdiv(
|
||||
@@ -810,6 +844,7 @@ def invoke_fused_moe_triton_kernel(
|
||||
naive_block_assignment=(sorted_token_ids is None),
|
||||
HAS_BIAS=HAS_BIAS,
|
||||
BLOCK_SIZE_K=BLOCK_SIZE_K,
|
||||
SWAP_AB=SWAP_AB,
|
||||
**config,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
from math import prod
|
||||
|
||||
import torch
|
||||
@@ -446,3 +447,12 @@ def swiglu_limit_func(
|
||||
up = torch.clamp(up, min=-swiglu_limit, max=swiglu_limit)
|
||||
|
||||
output.copy_(F.silu(gate) * up)
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def enable_swap_ab(BLOCK_SIZE_M: int, BLOCK_SIZE_N: int) -> bool:
|
||||
return (
|
||||
current_platform.is_device_capability(90)
|
||||
and BLOCK_SIZE_M < 64
|
||||
and BLOCK_SIZE_N >= 64
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user