[Kernel] TD operand loads for batched MoE GEMM (moe_mmk) on XPU (#46340)

Signed-off-by: oonyshch <xonyshch@gmail.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Lena Onyshchenko
2026-07-26 08:50:07 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent d30b1ecd1b
commit 0111002323
9 changed files with 331 additions and 33 deletions
+184
View File
@@ -351,3 +351,187 @@ def test_fused_moe_batched_experts(
torch.testing.assert_close(batched_output, baseline_output, atol=3e-2, rtol=2e-2)
torch.testing.assert_close(triton_output, batched_output, atol=2e-2, rtol=2e-2)
# USE_TD in moe_mmk
# K % 8 == 0 (bf16, 16-byte alignment)
_TD_SHAPES = [
(4, 1, 128, 256),
(8, 64, 512, 512),
(4, 256, 1024, 2048),
]
def _td_supported() -> bool:
return current_platform.is_xpu() or (
current_platform.is_cuda() and current_platform.has_device_capability(90)
)
def _run_td(A, B, num_expert_tokens, use_td: bool):
out = torch.zeros(
A.shape[0],
A.shape[1],
B.shape[1],
dtype=torch.bfloat16,
device=A.device,
)
import triton
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
batched_triton_kernel,
)
if use_td:
from vllm.triton_utils.allocation import set_triton_allocator
set_triton_allocator(A.device)
E = A.shape[0]
max_tokens = A.shape[1]
K = A.shape[2]
N = B.shape[1]
BM = BN = BK = 64
grid = (E, triton.cdiv(max_tokens, BM) * triton.cdiv(N, BN))
batched_triton_kernel[grid](
A,
B,
out,
num_expert_tokens,
tl.bfloat16,
max_tokens,
K,
N,
None,
None,
None,
A.stride(0),
A.stride(1),
A.stride(2),
B.stride(0),
B.stride(2),
B.stride(1),
out.stride(0),
out.stride(1),
out.stride(2),
0,
0,
0,
0,
0,
0,
0,
0,
False,
False,
False,
BLOCK_M=BM,
BLOCK_N=BN,
BLOCK_K=BK,
USE_TD=use_td,
)
return out
@pytest.mark.parametrize("num_experts,max_tokens_per_expert,K,N", _TD_SHAPES)
def test_batched_mm_td_matches_plain(num_experts, max_tokens_per_expert, K, N):
if not _td_supported():
pytest.skip("TD requires XPU or CUDA sm_90+")
set_random_seed(42)
device = current_platform.device_type
A = (
torch.randn(
num_experts, max_tokens_per_expert, K, device=device, dtype=torch.bfloat16
)
/ 10
)
B = torch.randn(num_experts, N, K, device=device, dtype=torch.bfloat16)
num_expert_tokens = torch.randint(
1,
max_tokens_per_expert + 1,
size=(num_experts,),
device=device,
dtype=torch.int32,
)
out_plain = _run_td(A, B, num_expert_tokens, False)
out_td = _run_td(A, B, num_expert_tokens, True)
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
def test_batched_mm_td_zero_expert_tokens():
if not _td_supported():
pytest.skip("TD requires XPU or CUDA sm_90+")
set_random_seed(42)
device = current_platform.device_type
E, M, K, N = 8, 32, 256, 256
A = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) / 10
B = torch.randn(E, N, K, device=device, dtype=torch.bfloat16)
num_expert_tokens = torch.zeros(E, device=device, dtype=torch.int32)
num_expert_tokens[::2] = M
out_plain = _run_td(A, B, num_expert_tokens, False)
out_td = _run_td(A, B, num_expert_tokens, True)
for e in range(E):
if num_expert_tokens[e].item() == 0:
assert out_plain[e].abs().max().item() == 0.0
assert out_td[e].abs().max().item() == 0.0
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
# BatchedTritonExperts device enablement (XPU)
def test_batched_triton_experts_supports_current_device():
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
)
if current_platform.is_xpu() or current_platform.is_cuda_alike():
assert BatchedTritonExperts._supports_current_device()
else:
pytest.skip("No GPU device available")
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
def test_batched_experts_end_to_end(m, n, k, e, topk):
"""End-to-end BatchedTritonExperts via the reference (no-comms)
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
the device-enablement path."""
if not (current_platform.is_xpu() or current_platform.is_cuda_alike()):
pytest.skip("No GPU device available")
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
device = current_platform.device_type
init_workspace_manager(torch.device(f"{device}:0"))
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
# w1 is the fused gate+up projection [E, 2N, K]; w2 is [E, K, N].
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
with set_current_vllm_config(vllm_config):
topk_weight, topk_ids, _ = fused_topk(a, score, topk, False)
baseline_output = torch_experts(a, w1, w2, topk_weight, topk_ids)
triton_output = batched_moe(a, w1, w2, topk_weight, topk_ids)
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
def test_batched_triton_backend_mapping():
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
map_unquantized_backend,
)
assert (
map_unquantized_backend("batched_triton")
== UnquantizedMoeBackend.BATCHED_TRITON
)
assert map_unquantized_backend("triton") == UnquantizedMoeBackend.TRITON
+3
View File
@@ -122,6 +122,7 @@ class IrOpPriorityConfig:
MoEBackend = Literal[
"auto",
"triton",
"batched_triton",
"deep_gemm",
"deep_gemm_mega_moe",
"cutlass",
@@ -192,6 +193,8 @@ class KernelConfig:
- "auto": Automatically select the best backend based on model and hardware
- "triton": Use Triton-based fused MoE kernels
- "batched_triton": Use batched Triton experts (moe_mmk) on the batched
activation format ([E_local, max_num_tokens, K])
- "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only)
- "deep_gemm_mega_moe": Use DeepGEMM mega MoE kernels
- "cutlass": Use vLLM CUTLASS kernels
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
BatchedPrepareAndFinalize,
make_moe_prepare_and_finalize_naive_dp_ep,
make_moe_prepare_and_finalize_no_dp_ep,
)
@@ -140,6 +141,16 @@ def maybe_make_prepare_finalize(
if not allow_new_interface:
return None
# Opt-in XPU batched path: reorganize tokens into E x T x K locally
# (no all-to-all) so BatchedTritonExperts (moe_mmk TD) can run.
if current_platform.is_xpu() and moe.moe_backend == "batched_triton":
return BatchedPrepareAndFinalize(
max_num_tokens=moe.max_num_tokens,
num_local_experts=moe.num_local_experts,
num_dispatchers=1,
rank=moe.moe_parallel_config.ep_rank,
)
# For DP/TP case, fall back to naive P/F.
if moe.moe_parallel_config.dp_size > 1:
logger.info_once(
@@ -32,7 +32,15 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.triton_utils import tl, triton, use_tensor_descriptor
from vllm.triton_utils.allocation import set_triton_allocator
def _is_capturing_or_compiling() -> bool:
# torch.cuda.is_current_stream_capturing() is unavailable on non-CUDA (XPU) torch.
return torch.compiler.is_compiling() or (
current_platform.is_cuda_alike() and torch.cuda.is_current_stream_capturing()
)
@triton.jit
@@ -71,9 +79,32 @@ def moe_mmk(
use_w8a8: tl.constexpr,
use_w8a16: tl.constexpr,
per_act_token_quant: tl.constexpr,
# TD: a_base_ptr/b_base_ptr are the expert/CTA-offset bases of A[M,K]/B[N,K].
a_base_ptr=None,
b_base_ptr=None,
M=0,
N=0,
stride_am: tl.int64 = 0,
stride_bn: tl.int64 = 0,
USE_TD: tl.constexpr = False,
):
offs_k = tl.arange(0, BLOCK_K)
if USE_TD:
# make_tensor_descriptor requires the last (K) stride to be a
# compile-time 1; the launcher only enables USE_TD for K-contiguous A/B.
a_desc = tl.make_tensor_descriptor(
a_base_ptr,
shape=[M, K],
strides=[stride_am, 1],
block_shape=[BLOCK_M, BLOCK_K],
)
b_desc = tl.make_tensor_descriptor(
b_base_ptr,
shape=[N, K],
strides=[stride_bn, 1],
block_shape=[BLOCK_N, BLOCK_K],
)
if use_w8a16:
b_scale_ptrs = (
b_scale_ptr + expert_id * stride_bse + offs_n[None, :] * stride_bsn
@@ -110,12 +141,17 @@ def moe_mmk(
for k in range(0, tl.cdiv(K, BLOCK_K)):
# Load the next block of A and B, generate a mask by checking the
# K dimension.
a = tl.load(
a_ptrs,
mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K),
other=0.0,
)
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0)
if USE_TD:
# B is [N, K]; tile is [BLOCK_N, BLOCK_K], transposed for dot.
a = a_desc.load([0, k * BLOCK_K])
b = tl.trans(b_desc.load([0, k * BLOCK_K]))
else:
a = tl.load(
a_ptrs,
mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K),
other=0.0,
)
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0)
# We accumulate along the K dimension.
if use_w8a16:
accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
@@ -193,6 +229,7 @@ def expert_triton_kernel(
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
USE_TD: tl.constexpr = False,
):
offs_m = tl.arange(0, BLOCK_M)
offs_n = tl.arange(0, BLOCK_N) % N
@@ -238,6 +275,13 @@ def expert_triton_kernel(
use_fp8_w8a8,
use_int8_w8a16,
per_act_token_quant,
a_ptr,
b_ptr,
M,
N,
stride_am,
stride_bn,
USE_TD,
)
# store in C
@@ -292,6 +336,7 @@ def batched_triton_kernel(
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
USE_TD: tl.constexpr = False,
):
expert_id = tl.program_id(axis=0)
e_num_tokens = tl.load(expert_num_tokens + expert_id)
@@ -372,6 +417,7 @@ def batched_triton_kernel(
BLOCK_M,
BLOCK_N,
BLOCK_K,
USE_TD,
)
@@ -444,6 +490,18 @@ def invoke_moe_batched_triton_kernel(
stride_asm = 0
stride_ask = 0
use_td = (
use_tensor_descriptor()
and A.stride(2) == 1
and B.stride(2) == 1
and (K * A.element_size()) % 16 == 0
and (BLOCK_M & (BLOCK_M - 1)) == 0
and (BLOCK_N & (BLOCK_N - 1)) == 0
and (BLOCK_K & (BLOCK_K - 1)) == 0
)
if use_td:
set_triton_allocator(A.device)
batched_triton_kernel[grid](
A,
B,
@@ -485,6 +543,7 @@ def invoke_moe_batched_triton_kernel(
BLOCK_M=BLOCK_M,
BLOCK_N=BLOCK_N,
BLOCK_K=BLOCK_K,
USE_TD=use_td,
)
@@ -616,10 +675,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
for expert in range(num_local_experts):
# Indexing expert_num_tokens doesn't work w/cudagraphs or inductor
if (
torch.compiler.is_compiling()
or torch.cuda.is_current_stream_capturing()
):
if _is_capturing_or_compiling():
num = hidden_states.shape[1]
else:
num = int(expert_num_tokens[expert].item())
@@ -659,7 +715,7 @@ def batched_moe_kernel_quantize_input(
per_act_token_quant: bool,
block_shape: list[int] | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing():
if _is_capturing_or_compiling():
# Note: this does a bunch of extra work because expert_num_tokens is
# ignored but it does support torch.compile + cudagraphs.
hidden_dim = A.size(-1)
@@ -745,7 +801,7 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cuda_alike()
return current_platform.is_cuda_alike() or current_platform.is_xpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
@@ -151,6 +151,7 @@ def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend
"""Map user's MoEBackend to UnquantizedMoeBackend."""
mapping = {
"triton": UnquantizedMoeBackend.TRITON,
"batched_triton": UnquantizedMoeBackend.BATCHED_TRITON,
"flashinfer_trtllm": UnquantizedMoeBackend.FLASHINFER_TRTLLM,
"flashinfer_cutlass": UnquantizedMoeBackend.FLASHINFER_CUTLASS,
"aiter": UnquantizedMoeBackend.AITER,
@@ -233,6 +234,7 @@ def select_unquantized_moe_backend(
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
if moe_config.moe_parallel_config.use_batched_activation_format
or moe_config.moe_backend == "batched_triton"
else mk.FusedMoEActivationFormat.Standard
)
@@ -112,15 +112,28 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
a1_scale = normalize_scales_shape(quant_config.a1_scale)
for expert_id in range(first_expert, last_expert):
topks = torch.any(topk_ids == expert_id, dim=1).flatten()
rows = torch.count_nonzero(topks.flatten())
if rows == 0:
continue
idx = expert_id - first_expert
tokens_per_expert[idx] = rows
rhs = a1[: topks.numel()][topks]
if quant_config.quant_dtype is not None:
if quant_config.quant_dtype is None:
# Vectorized dispatch: compute the [E_local, T] hit mask and
# per-expert slot offsets in one shot (single nonzero sync instead
# of a launch-bound per-expert Python loop).
local_ids = torch.arange(first_expert, last_expert, device=a1.device).view(
-1, 1, 1
)
hits = (topk_ids.unsqueeze(0) == local_ids).any(dim=2) # [E_local, T]
tokens_per_expert[:num_local_experts] = hits.sum(dim=1).to(torch.int32)
# slot of each token within its expert batch, preserving token order
slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T]
e_idx, t_idx = hits.nonzero(as_tuple=True)
b_a1[e_idx, slots[e_idx, t_idx]] = a1[t_idx].to(b_type)
else:
for expert_id in range(first_expert, last_expert):
topks = torch.any(topk_ids == expert_id, dim=1).flatten()
rows = torch.count_nonzero(topks.flatten())
if rows == 0:
continue
idx = expert_id - first_expert
tokens_per_expert[idx] = rows
rhs = a1[: topks.numel()][topks]
if a1_scale is not None:
if quant_config.is_per_act_token:
rhs_a1_scale = a1_scale[: topks.numel()][topks]
@@ -140,8 +153,6 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
b_a1_scale[idx, :rows] = b_s[:rows]
else:
b_a1_scale[idx, : b_s.shape[0]] = b_s
else:
b_a1[idx, :rows, :] = rhs
assert b_a1_scale is None or b_a1_scale.ndim == 3
@@ -164,13 +164,20 @@ class TopKWeightAndReduceNaiveBatched(mk.TopKWeightAndReduce):
first_expert = num_local_experts * self.rank
last_expert = first_expert + num_local_experts
for expert_id in range(first_expert, last_expert):
matching_tokens = topk_ids == expert_id
topks = torch.any(matching_tokens, dim=1).flatten()
rows = torch.count_nonzero(topks)
rhs = fused_expert_output[expert_id - first_expert, :rows, :]
if not apply_router_weight_on_input:
rhs.mul_(topk_weights[matching_tokens].view(rhs.size(0), 1))
output[topks] = output[topks] + rhs
# Vectorized weighted scatter-add (single nonzero sync instead of a
# per-expert loop; mirrors the dispatch path).
local_ids = torch.arange(
first_expert, last_expert, device=topk_ids.device
).view(-1, 1, 1)
matching = topk_ids.unsqueeze(0) == local_ids # [E_local, T, topk]
hits = matching.any(dim=2) # [E_local, T]
slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T]
e_idx, t_idx = hits.nonzero(as_tuple=True)
gathered = fused_expert_output[e_idx, slots[e_idx, t_idx], :]
if not apply_router_weight_on_input:
# weight of token t at expert e: the topk weight on the matching slot
weights = (matching * topk_weights.unsqueeze(0)).sum(dim=2) # [E_local, T]
gathered = gathered * weights[e_idx, t_idx].unsqueeze(1).to(gathered.dtype)
output.index_add_(0, t_idx, gathered.to(output.dtype))
return output
+11 -1
View File
@@ -17,7 +17,17 @@ else:
tl = TritonLanguagePlaceholder()
tldevice = TritonLanguagePlaceholder()
from vllm.triton_utils.tensor_descriptor import use_tensor_descriptor
LOG2E = 1.4426950408889634
LOGE2 = 0.6931471805599453
__all__ = ["HAS_TRITON", "triton", "tl", "tldevice", "LOG2E", "LOGE2"]
__all__ = [
"HAS_TRITON",
"triton",
"tl",
"tldevice",
"LOG2E",
"LOGE2",
"use_tensor_descriptor",
]
+14
View File
@@ -0,0 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
def use_tensor_descriptor(override: bool | None = None) -> bool:
"""Tri-state VLLM_TRITON_USE_TD: unset=auto (on for XPU), 1/0=force on/off."""
from vllm import envs
from vllm.platforms import current_platform
if override is None:
override = envs.VLLM_TRITON_USE_TD
if override is not None:
return override
return current_platform.is_xpu()