[Kernel] TD operand loads for batched MoE GEMM (moe_mmk) on XPU (#46340)

Signed-off-by: oonyshch <xonyshch@gmail.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Lena Onyshchenko
2026-07-26 08:50:07 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent d30b1ecd1b
commit 0111002323
9 changed files with 331 additions and 33 deletions
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
BatchedPrepareAndFinalize,
make_moe_prepare_and_finalize_naive_dp_ep,
make_moe_prepare_and_finalize_no_dp_ep,
)
@@ -140,6 +141,16 @@ def maybe_make_prepare_finalize(
if not allow_new_interface:
return None
# Opt-in XPU batched path: reorganize tokens into E x T x K locally
# (no all-to-all) so BatchedTritonExperts (moe_mmk TD) can run.
if current_platform.is_xpu() and moe.moe_backend == "batched_triton":
return BatchedPrepareAndFinalize(
max_num_tokens=moe.max_num_tokens,
num_local_experts=moe.num_local_experts,
num_dispatchers=1,
rank=moe.moe_parallel_config.ep_rank,
)
# For DP/TP case, fall back to naive P/F.
if moe.moe_parallel_config.dp_size > 1:
logger.info_once(
@@ -32,7 +32,15 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.triton_utils import tl, triton, use_tensor_descriptor
from vllm.triton_utils.allocation import set_triton_allocator
def _is_capturing_or_compiling() -> bool:
# torch.cuda.is_current_stream_capturing() is unavailable on non-CUDA (XPU) torch.
return torch.compiler.is_compiling() or (
current_platform.is_cuda_alike() and torch.cuda.is_current_stream_capturing()
)
@triton.jit
@@ -71,9 +79,32 @@ def moe_mmk(
use_w8a8: tl.constexpr,
use_w8a16: tl.constexpr,
per_act_token_quant: tl.constexpr,
# TD: a_base_ptr/b_base_ptr are the expert/CTA-offset bases of A[M,K]/B[N,K].
a_base_ptr=None,
b_base_ptr=None,
M=0,
N=0,
stride_am: tl.int64 = 0,
stride_bn: tl.int64 = 0,
USE_TD: tl.constexpr = False,
):
offs_k = tl.arange(0, BLOCK_K)
if USE_TD:
# make_tensor_descriptor requires the last (K) stride to be a
# compile-time 1; the launcher only enables USE_TD for K-contiguous A/B.
a_desc = tl.make_tensor_descriptor(
a_base_ptr,
shape=[M, K],
strides=[stride_am, 1],
block_shape=[BLOCK_M, BLOCK_K],
)
b_desc = tl.make_tensor_descriptor(
b_base_ptr,
shape=[N, K],
strides=[stride_bn, 1],
block_shape=[BLOCK_N, BLOCK_K],
)
if use_w8a16:
b_scale_ptrs = (
b_scale_ptr + expert_id * stride_bse + offs_n[None, :] * stride_bsn
@@ -110,12 +141,17 @@ def moe_mmk(
for k in range(0, tl.cdiv(K, BLOCK_K)):
# Load the next block of A and B, generate a mask by checking the
# K dimension.
a = tl.load(
a_ptrs,
mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K),
other=0.0,
)
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0)
if USE_TD:
# B is [N, K]; tile is [BLOCK_N, BLOCK_K], transposed for dot.
a = a_desc.load([0, k * BLOCK_K])
b = tl.trans(b_desc.load([0, k * BLOCK_K]))
else:
a = tl.load(
a_ptrs,
mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K),
other=0.0,
)
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0)
# We accumulate along the K dimension.
if use_w8a16:
accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
@@ -193,6 +229,7 @@ def expert_triton_kernel(
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
USE_TD: tl.constexpr = False,
):
offs_m = tl.arange(0, BLOCK_M)
offs_n = tl.arange(0, BLOCK_N) % N
@@ -238,6 +275,13 @@ def expert_triton_kernel(
use_fp8_w8a8,
use_int8_w8a16,
per_act_token_quant,
a_ptr,
b_ptr,
M,
N,
stride_am,
stride_bn,
USE_TD,
)
# store in C
@@ -292,6 +336,7 @@ def batched_triton_kernel(
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
USE_TD: tl.constexpr = False,
):
expert_id = tl.program_id(axis=0)
e_num_tokens = tl.load(expert_num_tokens + expert_id)
@@ -372,6 +417,7 @@ def batched_triton_kernel(
BLOCK_M,
BLOCK_N,
BLOCK_K,
USE_TD,
)
@@ -444,6 +490,18 @@ def invoke_moe_batched_triton_kernel(
stride_asm = 0
stride_ask = 0
use_td = (
use_tensor_descriptor()
and A.stride(2) == 1
and B.stride(2) == 1
and (K * A.element_size()) % 16 == 0
and (BLOCK_M & (BLOCK_M - 1)) == 0
and (BLOCK_N & (BLOCK_N - 1)) == 0
and (BLOCK_K & (BLOCK_K - 1)) == 0
)
if use_td:
set_triton_allocator(A.device)
batched_triton_kernel[grid](
A,
B,
@@ -485,6 +543,7 @@ def invoke_moe_batched_triton_kernel(
BLOCK_M=BLOCK_M,
BLOCK_N=BLOCK_N,
BLOCK_K=BLOCK_K,
USE_TD=use_td,
)
@@ -616,10 +675,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
for expert in range(num_local_experts):
# Indexing expert_num_tokens doesn't work w/cudagraphs or inductor
if (
torch.compiler.is_compiling()
or torch.cuda.is_current_stream_capturing()
):
if _is_capturing_or_compiling():
num = hidden_states.shape[1]
else:
num = int(expert_num_tokens[expert].item())
@@ -659,7 +715,7 @@ def batched_moe_kernel_quantize_input(
per_act_token_quant: bool,
block_shape: list[int] | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing():
if _is_capturing_or_compiling():
# Note: this does a bunch of extra work because expert_num_tokens is
# ignored but it does support torch.compile + cudagraphs.
hidden_dim = A.size(-1)
@@ -745,7 +801,7 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cuda_alike()
return current_platform.is_cuda_alike() or current_platform.is_xpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
@@ -151,6 +151,7 @@ def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend
"""Map user's MoEBackend to UnquantizedMoeBackend."""
mapping = {
"triton": UnquantizedMoeBackend.TRITON,
"batched_triton": UnquantizedMoeBackend.BATCHED_TRITON,
"flashinfer_trtllm": UnquantizedMoeBackend.FLASHINFER_TRTLLM,
"flashinfer_cutlass": UnquantizedMoeBackend.FLASHINFER_CUTLASS,
"aiter": UnquantizedMoeBackend.AITER,
@@ -233,6 +234,7 @@ def select_unquantized_moe_backend(
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
if moe_config.moe_parallel_config.use_batched_activation_format
or moe_config.moe_backend == "batched_triton"
else mk.FusedMoEActivationFormat.Standard
)
@@ -112,15 +112,28 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
a1_scale = normalize_scales_shape(quant_config.a1_scale)
for expert_id in range(first_expert, last_expert):
topks = torch.any(topk_ids == expert_id, dim=1).flatten()
rows = torch.count_nonzero(topks.flatten())
if rows == 0:
continue
idx = expert_id - first_expert
tokens_per_expert[idx] = rows
rhs = a1[: topks.numel()][topks]
if quant_config.quant_dtype is not None:
if quant_config.quant_dtype is None:
# Vectorized dispatch: compute the [E_local, T] hit mask and
# per-expert slot offsets in one shot (single nonzero sync instead
# of a launch-bound per-expert Python loop).
local_ids = torch.arange(first_expert, last_expert, device=a1.device).view(
-1, 1, 1
)
hits = (topk_ids.unsqueeze(0) == local_ids).any(dim=2) # [E_local, T]
tokens_per_expert[:num_local_experts] = hits.sum(dim=1).to(torch.int32)
# slot of each token within its expert batch, preserving token order
slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T]
e_idx, t_idx = hits.nonzero(as_tuple=True)
b_a1[e_idx, slots[e_idx, t_idx]] = a1[t_idx].to(b_type)
else:
for expert_id in range(first_expert, last_expert):
topks = torch.any(topk_ids == expert_id, dim=1).flatten()
rows = torch.count_nonzero(topks.flatten())
if rows == 0:
continue
idx = expert_id - first_expert
tokens_per_expert[idx] = rows
rhs = a1[: topks.numel()][topks]
if a1_scale is not None:
if quant_config.is_per_act_token:
rhs_a1_scale = a1_scale[: topks.numel()][topks]
@@ -140,8 +153,6 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
b_a1_scale[idx, :rows] = b_s[:rows]
else:
b_a1_scale[idx, : b_s.shape[0]] = b_s
else:
b_a1[idx, :rows, :] = rhs
assert b_a1_scale is None or b_a1_scale.ndim == 3
@@ -164,13 +164,20 @@ class TopKWeightAndReduceNaiveBatched(mk.TopKWeightAndReduce):
first_expert = num_local_experts * self.rank
last_expert = first_expert + num_local_experts
for expert_id in range(first_expert, last_expert):
matching_tokens = topk_ids == expert_id
topks = torch.any(matching_tokens, dim=1).flatten()
rows = torch.count_nonzero(topks)
rhs = fused_expert_output[expert_id - first_expert, :rows, :]
if not apply_router_weight_on_input:
rhs.mul_(topk_weights[matching_tokens].view(rhs.size(0), 1))
output[topks] = output[topks] + rhs
# Vectorized weighted scatter-add (single nonzero sync instead of a
# per-expert loop; mirrors the dispatch path).
local_ids = torch.arange(
first_expert, last_expert, device=topk_ids.device
).view(-1, 1, 1)
matching = topk_ids.unsqueeze(0) == local_ids # [E_local, T, topk]
hits = matching.any(dim=2) # [E_local, T]
slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T]
e_idx, t_idx = hits.nonzero(as_tuple=True)
gathered = fused_expert_output[e_idx, slots[e_idx, t_idx], :]
if not apply_router_weight_on_input:
# weight of token t at expert e: the topk weight on the matching slot
weights = (matching * topk_weights.unsqueeze(0)).sum(dim=2) # [E_local, T]
gathered = gathered * weights[e_idx, t_idx].unsqueeze(1).to(gathered.dtype)
output.index_add_(0, t_idx, gathered.to(output.dtype))
return output