[Kernel] TD operand loads for batched MoE GEMM (moe_mmk) on XPU (#46340)
Signed-off-by: oonyshch <xonyshch@gmail.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
d30b1ecd1b
commit
0111002323
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
BatchedPrepareAndFinalize,
|
||||
make_moe_prepare_and_finalize_naive_dp_ep,
|
||||
make_moe_prepare_and_finalize_no_dp_ep,
|
||||
)
|
||||
@@ -140,6 +141,16 @@ def maybe_make_prepare_finalize(
|
||||
if not allow_new_interface:
|
||||
return None
|
||||
|
||||
# Opt-in XPU batched path: reorganize tokens into E x T x K locally
|
||||
# (no all-to-all) so BatchedTritonExperts (moe_mmk TD) can run.
|
||||
if current_platform.is_xpu() and moe.moe_backend == "batched_triton":
|
||||
return BatchedPrepareAndFinalize(
|
||||
max_num_tokens=moe.max_num_tokens,
|
||||
num_local_experts=moe.num_local_experts,
|
||||
num_dispatchers=1,
|
||||
rank=moe.moe_parallel_config.ep_rank,
|
||||
)
|
||||
|
||||
# For DP/TP case, fall back to naive P/F.
|
||||
if moe.moe_parallel_config.dp_size > 1:
|
||||
logger.info_once(
|
||||
|
||||
@@ -32,7 +32,15 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.triton_utils import tl, triton, use_tensor_descriptor
|
||||
from vllm.triton_utils.allocation import set_triton_allocator
|
||||
|
||||
|
||||
def _is_capturing_or_compiling() -> bool:
|
||||
# torch.cuda.is_current_stream_capturing() is unavailable on non-CUDA (XPU) torch.
|
||||
return torch.compiler.is_compiling() or (
|
||||
current_platform.is_cuda_alike() and torch.cuda.is_current_stream_capturing()
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -71,9 +79,32 @@ def moe_mmk(
|
||||
use_w8a8: tl.constexpr,
|
||||
use_w8a16: tl.constexpr,
|
||||
per_act_token_quant: tl.constexpr,
|
||||
# TD: a_base_ptr/b_base_ptr are the expert/CTA-offset bases of A[M,K]/B[N,K].
|
||||
a_base_ptr=None,
|
||||
b_base_ptr=None,
|
||||
M=0,
|
||||
N=0,
|
||||
stride_am: tl.int64 = 0,
|
||||
stride_bn: tl.int64 = 0,
|
||||
USE_TD: tl.constexpr = False,
|
||||
):
|
||||
offs_k = tl.arange(0, BLOCK_K)
|
||||
|
||||
if USE_TD:
|
||||
# make_tensor_descriptor requires the last (K) stride to be a
|
||||
# compile-time 1; the launcher only enables USE_TD for K-contiguous A/B.
|
||||
a_desc = tl.make_tensor_descriptor(
|
||||
a_base_ptr,
|
||||
shape=[M, K],
|
||||
strides=[stride_am, 1],
|
||||
block_shape=[BLOCK_M, BLOCK_K],
|
||||
)
|
||||
b_desc = tl.make_tensor_descriptor(
|
||||
b_base_ptr,
|
||||
shape=[N, K],
|
||||
strides=[stride_bn, 1],
|
||||
block_shape=[BLOCK_N, BLOCK_K],
|
||||
)
|
||||
if use_w8a16:
|
||||
b_scale_ptrs = (
|
||||
b_scale_ptr + expert_id * stride_bse + offs_n[None, :] * stride_bsn
|
||||
@@ -110,12 +141,17 @@ def moe_mmk(
|
||||
for k in range(0, tl.cdiv(K, BLOCK_K)):
|
||||
# Load the next block of A and B, generate a mask by checking the
|
||||
# K dimension.
|
||||
a = tl.load(
|
||||
a_ptrs,
|
||||
mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K),
|
||||
other=0.0,
|
||||
)
|
||||
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0)
|
||||
if USE_TD:
|
||||
# B is [N, K]; tile is [BLOCK_N, BLOCK_K], transposed for dot.
|
||||
a = a_desc.load([0, k * BLOCK_K])
|
||||
b = tl.trans(b_desc.load([0, k * BLOCK_K]))
|
||||
else:
|
||||
a = tl.load(
|
||||
a_ptrs,
|
||||
mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K),
|
||||
other=0.0,
|
||||
)
|
||||
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0)
|
||||
# We accumulate along the K dimension.
|
||||
if use_w8a16:
|
||||
accumulator = tl.dot(a, b.to(compute_type), acc=accumulator)
|
||||
@@ -193,6 +229,7 @@ def expert_triton_kernel(
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
USE_TD: tl.constexpr = False,
|
||||
):
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.arange(0, BLOCK_N) % N
|
||||
@@ -238,6 +275,13 @@ def expert_triton_kernel(
|
||||
use_fp8_w8a8,
|
||||
use_int8_w8a16,
|
||||
per_act_token_quant,
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
M,
|
||||
N,
|
||||
stride_am,
|
||||
stride_bn,
|
||||
USE_TD,
|
||||
)
|
||||
|
||||
# store in C
|
||||
@@ -292,6 +336,7 @@ def batched_triton_kernel(
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
USE_TD: tl.constexpr = False,
|
||||
):
|
||||
expert_id = tl.program_id(axis=0)
|
||||
e_num_tokens = tl.load(expert_num_tokens + expert_id)
|
||||
@@ -372,6 +417,7 @@ def batched_triton_kernel(
|
||||
BLOCK_M,
|
||||
BLOCK_N,
|
||||
BLOCK_K,
|
||||
USE_TD,
|
||||
)
|
||||
|
||||
|
||||
@@ -444,6 +490,18 @@ def invoke_moe_batched_triton_kernel(
|
||||
stride_asm = 0
|
||||
stride_ask = 0
|
||||
|
||||
use_td = (
|
||||
use_tensor_descriptor()
|
||||
and A.stride(2) == 1
|
||||
and B.stride(2) == 1
|
||||
and (K * A.element_size()) % 16 == 0
|
||||
and (BLOCK_M & (BLOCK_M - 1)) == 0
|
||||
and (BLOCK_N & (BLOCK_N - 1)) == 0
|
||||
and (BLOCK_K & (BLOCK_K - 1)) == 0
|
||||
)
|
||||
if use_td:
|
||||
set_triton_allocator(A.device)
|
||||
|
||||
batched_triton_kernel[grid](
|
||||
A,
|
||||
B,
|
||||
@@ -485,6 +543,7 @@ def invoke_moe_batched_triton_kernel(
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_K=BLOCK_K,
|
||||
USE_TD=use_td,
|
||||
)
|
||||
|
||||
|
||||
@@ -616,10 +675,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
for expert in range(num_local_experts):
|
||||
# Indexing expert_num_tokens doesn't work w/cudagraphs or inductor
|
||||
if (
|
||||
torch.compiler.is_compiling()
|
||||
or torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
if _is_capturing_or_compiling():
|
||||
num = hidden_states.shape[1]
|
||||
else:
|
||||
num = int(expert_num_tokens[expert].item())
|
||||
@@ -659,7 +715,7 @@ def batched_moe_kernel_quantize_input(
|
||||
per_act_token_quant: bool,
|
||||
block_shape: list[int] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing():
|
||||
if _is_capturing_or_compiling():
|
||||
# Note: this does a bunch of extra work because expert_num_tokens is
|
||||
# ignored but it does support torch.compile + cudagraphs.
|
||||
hidden_dim = A.size(-1)
|
||||
@@ -745,7 +801,7 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cuda_alike()
|
||||
return current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
|
||||
@@ -151,6 +151,7 @@ def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend
|
||||
"""Map user's MoEBackend to UnquantizedMoeBackend."""
|
||||
mapping = {
|
||||
"triton": UnquantizedMoeBackend.TRITON,
|
||||
"batched_triton": UnquantizedMoeBackend.BATCHED_TRITON,
|
||||
"flashinfer_trtllm": UnquantizedMoeBackend.FLASHINFER_TRTLLM,
|
||||
"flashinfer_cutlass": UnquantizedMoeBackend.FLASHINFER_CUTLASS,
|
||||
"aiter": UnquantizedMoeBackend.AITER,
|
||||
@@ -233,6 +234,7 @@ def select_unquantized_moe_backend(
|
||||
activation_format = (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
if moe_config.moe_parallel_config.use_batched_activation_format
|
||||
or moe_config.moe_backend == "batched_triton"
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
|
||||
@@ -112,15 +112,28 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
|
||||
a1_scale = normalize_scales_shape(quant_config.a1_scale)
|
||||
|
||||
for expert_id in range(first_expert, last_expert):
|
||||
topks = torch.any(topk_ids == expert_id, dim=1).flatten()
|
||||
rows = torch.count_nonzero(topks.flatten())
|
||||
if rows == 0:
|
||||
continue
|
||||
idx = expert_id - first_expert
|
||||
tokens_per_expert[idx] = rows
|
||||
rhs = a1[: topks.numel()][topks]
|
||||
if quant_config.quant_dtype is not None:
|
||||
if quant_config.quant_dtype is None:
|
||||
# Vectorized dispatch: compute the [E_local, T] hit mask and
|
||||
# per-expert slot offsets in one shot (single nonzero sync instead
|
||||
# of a launch-bound per-expert Python loop).
|
||||
local_ids = torch.arange(first_expert, last_expert, device=a1.device).view(
|
||||
-1, 1, 1
|
||||
)
|
||||
hits = (topk_ids.unsqueeze(0) == local_ids).any(dim=2) # [E_local, T]
|
||||
tokens_per_expert[:num_local_experts] = hits.sum(dim=1).to(torch.int32)
|
||||
# slot of each token within its expert batch, preserving token order
|
||||
slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T]
|
||||
e_idx, t_idx = hits.nonzero(as_tuple=True)
|
||||
b_a1[e_idx, slots[e_idx, t_idx]] = a1[t_idx].to(b_type)
|
||||
else:
|
||||
for expert_id in range(first_expert, last_expert):
|
||||
topks = torch.any(topk_ids == expert_id, dim=1).flatten()
|
||||
rows = torch.count_nonzero(topks.flatten())
|
||||
if rows == 0:
|
||||
continue
|
||||
idx = expert_id - first_expert
|
||||
tokens_per_expert[idx] = rows
|
||||
rhs = a1[: topks.numel()][topks]
|
||||
if a1_scale is not None:
|
||||
if quant_config.is_per_act_token:
|
||||
rhs_a1_scale = a1_scale[: topks.numel()][topks]
|
||||
@@ -140,8 +153,6 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
b_a1_scale[idx, :rows] = b_s[:rows]
|
||||
else:
|
||||
b_a1_scale[idx, : b_s.shape[0]] = b_s
|
||||
else:
|
||||
b_a1[idx, :rows, :] = rhs
|
||||
|
||||
assert b_a1_scale is None or b_a1_scale.ndim == 3
|
||||
|
||||
|
||||
@@ -164,13 +164,20 @@ class TopKWeightAndReduceNaiveBatched(mk.TopKWeightAndReduce):
|
||||
first_expert = num_local_experts * self.rank
|
||||
last_expert = first_expert + num_local_experts
|
||||
|
||||
for expert_id in range(first_expert, last_expert):
|
||||
matching_tokens = topk_ids == expert_id
|
||||
topks = torch.any(matching_tokens, dim=1).flatten()
|
||||
rows = torch.count_nonzero(topks)
|
||||
rhs = fused_expert_output[expert_id - first_expert, :rows, :]
|
||||
if not apply_router_weight_on_input:
|
||||
rhs.mul_(topk_weights[matching_tokens].view(rhs.size(0), 1))
|
||||
output[topks] = output[topks] + rhs
|
||||
# Vectorized weighted scatter-add (single nonzero sync instead of a
|
||||
# per-expert loop; mirrors the dispatch path).
|
||||
local_ids = torch.arange(
|
||||
first_expert, last_expert, device=topk_ids.device
|
||||
).view(-1, 1, 1)
|
||||
matching = topk_ids.unsqueeze(0) == local_ids # [E_local, T, topk]
|
||||
hits = matching.any(dim=2) # [E_local, T]
|
||||
slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T]
|
||||
e_idx, t_idx = hits.nonzero(as_tuple=True)
|
||||
gathered = fused_expert_output[e_idx, slots[e_idx, t_idx], :]
|
||||
if not apply_router_weight_on_input:
|
||||
# weight of token t at expert e: the topk weight on the matching slot
|
||||
weights = (matching * topk_weights.unsqueeze(0)).sum(dim=2) # [E_local, T]
|
||||
gathered = gathered * weights[e_idx, t_idx].unsqueeze(1).to(gathered.dtype)
|
||||
output.index_add_(0, t_idx, gathered.to(output.dtype))
|
||||
|
||||
return output
|
||||
|
||||
Reference in New Issue
Block a user