From 0111002323e226844a18e7f3e5834e9d0e1b6dcd Mon Sep 17 00:00:00 2001 From: Lena Onyshchenko Date: Sun, 26 Jul 2026 02:50:07 +0200 Subject: [PATCH] [Kernel] TD operand loads for batched MoE GEMM (moe_mmk) on XPU (#46340) Signed-off-by: oonyshch Co-authored-by: Kunshang Ji --- tests/kernels/moe/test_batched_moe.py | 184 ++++++++++++++++++ vllm/config/kernel.py | 3 + .../layers/fused_moe/all2all_utils.py | 11 ++ .../fused_moe/experts/fused_batched_moe.py | 82 ++++++-- .../layers/fused_moe/oracle/unquantized.py | 2 + .../fused_moe/prepare_finalize/batched.py | 33 ++-- .../fused_moe/topk_weight_and_reduce.py | 23 ++- vllm/triton_utils/__init__.py | 12 +- vllm/triton_utils/tensor_descriptor.py | 14 ++ 9 files changed, 331 insertions(+), 33 deletions(-) create mode 100644 vllm/triton_utils/tensor_descriptor.py diff --git a/tests/kernels/moe/test_batched_moe.py b/tests/kernels/moe/test_batched_moe.py index b9fe8ceafc..3c605ced68 100644 --- a/tests/kernels/moe/test_batched_moe.py +++ b/tests/kernels/moe/test_batched_moe.py @@ -351,3 +351,187 @@ def test_fused_moe_batched_experts( torch.testing.assert_close(batched_output, baseline_output, atol=3e-2, rtol=2e-2) torch.testing.assert_close(triton_output, batched_output, atol=2e-2, rtol=2e-2) + + +# USE_TD in moe_mmk + +# K % 8 == 0 (bf16, 16-byte alignment) +_TD_SHAPES = [ + (4, 1, 128, 256), + (8, 64, 512, 512), + (4, 256, 1024, 2048), +] + + +def _td_supported() -> bool: + return current_platform.is_xpu() or ( + current_platform.is_cuda() and current_platform.has_device_capability(90) + ) + + +def _run_td(A, B, num_expert_tokens, use_td: bool): + out = torch.zeros( + A.shape[0], + A.shape[1], + B.shape[1], + dtype=torch.bfloat16, + device=A.device, + ) + import triton + + from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import ( + batched_triton_kernel, + ) + + if use_td: + from vllm.triton_utils.allocation import set_triton_allocator + + set_triton_allocator(A.device) + + E = A.shape[0] + max_tokens = A.shape[1] + K = A.shape[2] + N = B.shape[1] + BM = BN = BK = 64 + grid = (E, triton.cdiv(max_tokens, BM) * triton.cdiv(N, BN)) + batched_triton_kernel[grid]( + A, + B, + out, + num_expert_tokens, + tl.bfloat16, + max_tokens, + K, + N, + None, + None, + None, + A.stride(0), + A.stride(1), + A.stride(2), + B.stride(0), + B.stride(2), + B.stride(1), + out.stride(0), + out.stride(1), + out.stride(2), + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + False, + False, + False, + BLOCK_M=BM, + BLOCK_N=BN, + BLOCK_K=BK, + USE_TD=use_td, + ) + return out + + +@pytest.mark.parametrize("num_experts,max_tokens_per_expert,K,N", _TD_SHAPES) +def test_batched_mm_td_matches_plain(num_experts, max_tokens_per_expert, K, N): + if not _td_supported(): + pytest.skip("TD requires XPU or CUDA sm_90+") + set_random_seed(42) + device = current_platform.device_type + A = ( + torch.randn( + num_experts, max_tokens_per_expert, K, device=device, dtype=torch.bfloat16 + ) + / 10 + ) + B = torch.randn(num_experts, N, K, device=device, dtype=torch.bfloat16) + num_expert_tokens = torch.randint( + 1, + max_tokens_per_expert + 1, + size=(num_experts,), + device=device, + dtype=torch.int32, + ) + + out_plain = _run_td(A, B, num_expert_tokens, False) + out_td = _run_td(A, B, num_expert_tokens, True) + + torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2) + + +def test_batched_mm_td_zero_expert_tokens(): + if not _td_supported(): + pytest.skip("TD requires XPU or CUDA sm_90+") + set_random_seed(42) + device = current_platform.device_type + E, M, K, N = 8, 32, 256, 256 + A = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) / 10 + B = torch.randn(E, N, K, device=device, dtype=torch.bfloat16) + num_expert_tokens = torch.zeros(E, device=device, dtype=torch.int32) + num_expert_tokens[::2] = M + + out_plain = _run_td(A, B, num_expert_tokens, False) + out_td = _run_td(A, B, num_expert_tokens, True) + + for e in range(E): + if num_expert_tokens[e].item() == 0: + assert out_plain[e].abs().max().item() == 0.0 + assert out_td[e].abs().max().item() == 0.0 + + torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2) + + +# BatchedTritonExperts device enablement (XPU) +def test_batched_triton_experts_supports_current_device(): + from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import ( + BatchedTritonExperts, + ) + + if current_platform.is_xpu() or current_platform.is_cuda_alike(): + assert BatchedTritonExperts._supports_current_device() + else: + pytest.skip("No GPU device available") + + +@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)]) +def test_batched_experts_end_to_end(m, n, k, e, topk): + """End-to-end BatchedTritonExperts via the reference (no-comms) + BatchedPrepareAndFinalize, validated against a torch reference. Exercises + the device-enablement path.""" + if not (current_platform.is_xpu() or current_platform.is_cuda_alike()): + pytest.skip("No GPU device available") + + from vllm.v1.worker.workspace import init_workspace_manager + + set_random_seed(7) + device = current_platform.device_type + init_workspace_manager(torch.device(f"{device}:0")) + + a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10 + score = torch.randn((m, e), device=device, dtype=torch.bfloat16) + # w1 is the fused gate+up projection [E, 2N, K]; w2 is [E, K, N]. + w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15 + w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15 + + with set_current_vllm_config(vllm_config): + topk_weight, topk_ids, _ = fused_topk(a, score, topk, False) + + baseline_output = torch_experts(a, w1, w2, topk_weight, topk_ids) + triton_output = batched_moe(a, w1, w2, topk_weight, topk_ids) + + torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2) + + +def test_batched_triton_backend_mapping(): + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, + map_unquantized_backend, + ) + + assert ( + map_unquantized_backend("batched_triton") + == UnquantizedMoeBackend.BATCHED_TRITON + ) + assert map_unquantized_backend("triton") == UnquantizedMoeBackend.TRITON diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index 16856dc358..59195f307b 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -122,6 +122,7 @@ class IrOpPriorityConfig: MoEBackend = Literal[ "auto", "triton", + "batched_triton", "deep_gemm", "deep_gemm_mega_moe", "cutlass", @@ -192,6 +193,8 @@ class KernelConfig: - "auto": Automatically select the best backend based on model and hardware - "triton": Use Triton-based fused MoE kernels + - "batched_triton": Use batched Triton experts (moe_mmk) on the batched + activation format ([E_local, max_num_tokens, K]) - "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only) - "deep_gemm_mega_moe": Use DeepGEMM mega MoE kernels - "cutlass": Use vLLM CUTLASS kernels diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index 6af93bfde9..58c9c8d9f6 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import ( FusedMoEPrepareAndFinalize, ) from vllm.model_executor.layers.fused_moe.prepare_finalize import ( + BatchedPrepareAndFinalize, make_moe_prepare_and_finalize_naive_dp_ep, make_moe_prepare_and_finalize_no_dp_ep, ) @@ -140,6 +141,16 @@ def maybe_make_prepare_finalize( if not allow_new_interface: return None + # Opt-in XPU batched path: reorganize tokens into E x T x K locally + # (no all-to-all) so BatchedTritonExperts (moe_mmk TD) can run. + if current_platform.is_xpu() and moe.moe_backend == "batched_triton": + return BatchedPrepareAndFinalize( + max_num_tokens=moe.max_num_tokens, + num_local_experts=moe.num_local_experts, + num_dispatchers=1, + rank=moe.moe_parallel_config.ep_rank, + ) + # For DP/TP case, fall back to naive P/F. if moe.moe_parallel_config.dp_size > 1: logger.info_once( diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py index 21bda8e173..609b28a959 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py @@ -32,7 +32,15 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, ) from vllm.platforms import current_platform -from vllm.triton_utils import tl, triton +from vllm.triton_utils import tl, triton, use_tensor_descriptor +from vllm.triton_utils.allocation import set_triton_allocator + + +def _is_capturing_or_compiling() -> bool: + # torch.cuda.is_current_stream_capturing() is unavailable on non-CUDA (XPU) torch. + return torch.compiler.is_compiling() or ( + current_platform.is_cuda_alike() and torch.cuda.is_current_stream_capturing() + ) @triton.jit @@ -71,9 +79,32 @@ def moe_mmk( use_w8a8: tl.constexpr, use_w8a16: tl.constexpr, per_act_token_quant: tl.constexpr, + # TD: a_base_ptr/b_base_ptr are the expert/CTA-offset bases of A[M,K]/B[N,K]. + a_base_ptr=None, + b_base_ptr=None, + M=0, + N=0, + stride_am: tl.int64 = 0, + stride_bn: tl.int64 = 0, + USE_TD: tl.constexpr = False, ): offs_k = tl.arange(0, BLOCK_K) + if USE_TD: + # make_tensor_descriptor requires the last (K) stride to be a + # compile-time 1; the launcher only enables USE_TD for K-contiguous A/B. + a_desc = tl.make_tensor_descriptor( + a_base_ptr, + shape=[M, K], + strides=[stride_am, 1], + block_shape=[BLOCK_M, BLOCK_K], + ) + b_desc = tl.make_tensor_descriptor( + b_base_ptr, + shape=[N, K], + strides=[stride_bn, 1], + block_shape=[BLOCK_N, BLOCK_K], + ) if use_w8a16: b_scale_ptrs = ( b_scale_ptr + expert_id * stride_bse + offs_n[None, :] * stride_bsn @@ -110,12 +141,17 @@ def moe_mmk( for k in range(0, tl.cdiv(K, BLOCK_K)): # Load the next block of A and B, generate a mask by checking the # K dimension. - a = tl.load( - a_ptrs, - mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K), - other=0.0, - ) - b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0) + if USE_TD: + # B is [N, K]; tile is [BLOCK_N, BLOCK_K], transposed for dot. + a = a_desc.load([0, k * BLOCK_K]) + b = tl.trans(b_desc.load([0, k * BLOCK_K])) + else: + a = tl.load( + a_ptrs, + mask=mask_m[:, None] & (offs_k[None, :] < K - k * BLOCK_K), + other=0.0, + ) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_K, other=0.0) # We accumulate along the K dimension. if use_w8a16: accumulator = tl.dot(a, b.to(compute_type), acc=accumulator) @@ -193,6 +229,7 @@ def expert_triton_kernel( BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, + USE_TD: tl.constexpr = False, ): offs_m = tl.arange(0, BLOCK_M) offs_n = tl.arange(0, BLOCK_N) % N @@ -238,6 +275,13 @@ def expert_triton_kernel( use_fp8_w8a8, use_int8_w8a16, per_act_token_quant, + a_ptr, + b_ptr, + M, + N, + stride_am, + stride_bn, + USE_TD, ) # store in C @@ -292,6 +336,7 @@ def batched_triton_kernel( BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, + USE_TD: tl.constexpr = False, ): expert_id = tl.program_id(axis=0) e_num_tokens = tl.load(expert_num_tokens + expert_id) @@ -372,6 +417,7 @@ def batched_triton_kernel( BLOCK_M, BLOCK_N, BLOCK_K, + USE_TD, ) @@ -444,6 +490,18 @@ def invoke_moe_batched_triton_kernel( stride_asm = 0 stride_ask = 0 + use_td = ( + use_tensor_descriptor() + and A.stride(2) == 1 + and B.stride(2) == 1 + and (K * A.element_size()) % 16 == 0 + and (BLOCK_M & (BLOCK_M - 1)) == 0 + and (BLOCK_N & (BLOCK_N - 1)) == 0 + and (BLOCK_K & (BLOCK_K - 1)) == 0 + ) + if use_td: + set_triton_allocator(A.device) + batched_triton_kernel[grid]( A, B, @@ -485,6 +543,7 @@ def invoke_moe_batched_triton_kernel( BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, + USE_TD=use_td, ) @@ -616,10 +675,7 @@ class NaiveBatchedExperts(mk.FusedMoEExpertsModular): for expert in range(num_local_experts): # Indexing expert_num_tokens doesn't work w/cudagraphs or inductor - if ( - torch.compiler.is_compiling() - or torch.cuda.is_current_stream_capturing() - ): + if _is_capturing_or_compiling(): num = hidden_states.shape[1] else: num = int(expert_num_tokens[expert].item()) @@ -659,7 +715,7 @@ def batched_moe_kernel_quantize_input( per_act_token_quant: bool, block_shape: list[int] | None = None, ) -> tuple[torch.Tensor, torch.Tensor | None]: - if torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing(): + if _is_capturing_or_compiling(): # Note: this does a bunch of extra work because expert_num_tokens is # ignored but it does support torch.compile + cudagraphs. hidden_dim = A.size(-1) @@ -745,7 +801,7 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular): @staticmethod def _supports_current_device() -> bool: - return current_platform.is_cuda_alike() + return current_platform.is_cuda_alike() or current_platform.is_xpu() @staticmethod def _supports_no_act_and_mul() -> bool: diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index 5d4c733631..8fa1a0c265 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -151,6 +151,7 @@ def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend """Map user's MoEBackend to UnquantizedMoeBackend.""" mapping = { "triton": UnquantizedMoeBackend.TRITON, + "batched_triton": UnquantizedMoeBackend.BATCHED_TRITON, "flashinfer_trtllm": UnquantizedMoeBackend.FLASHINFER_TRTLLM, "flashinfer_cutlass": UnquantizedMoeBackend.FLASHINFER_CUTLASS, "aiter": UnquantizedMoeBackend.AITER, @@ -233,6 +234,7 @@ def select_unquantized_moe_backend( activation_format = ( mk.FusedMoEActivationFormat.BatchedExperts if moe_config.moe_parallel_config.use_batched_activation_format + or moe_config.moe_backend == "batched_triton" else mk.FusedMoEActivationFormat.Standard ) diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/batched.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/batched.py index 943027717b..257b5f04dc 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/batched.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/batched.py @@ -112,15 +112,28 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): a1_scale = normalize_scales_shape(quant_config.a1_scale) - for expert_id in range(first_expert, last_expert): - topks = torch.any(topk_ids == expert_id, dim=1).flatten() - rows = torch.count_nonzero(topks.flatten()) - if rows == 0: - continue - idx = expert_id - first_expert - tokens_per_expert[idx] = rows - rhs = a1[: topks.numel()][topks] - if quant_config.quant_dtype is not None: + if quant_config.quant_dtype is None: + # Vectorized dispatch: compute the [E_local, T] hit mask and + # per-expert slot offsets in one shot (single nonzero sync instead + # of a launch-bound per-expert Python loop). + local_ids = torch.arange(first_expert, last_expert, device=a1.device).view( + -1, 1, 1 + ) + hits = (topk_ids.unsqueeze(0) == local_ids).any(dim=2) # [E_local, T] + tokens_per_expert[:num_local_experts] = hits.sum(dim=1).to(torch.int32) + # slot of each token within its expert batch, preserving token order + slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T] + e_idx, t_idx = hits.nonzero(as_tuple=True) + b_a1[e_idx, slots[e_idx, t_idx]] = a1[t_idx].to(b_type) + else: + for expert_id in range(first_expert, last_expert): + topks = torch.any(topk_ids == expert_id, dim=1).flatten() + rows = torch.count_nonzero(topks.flatten()) + if rows == 0: + continue + idx = expert_id - first_expert + tokens_per_expert[idx] = rows + rhs = a1[: topks.numel()][topks] if a1_scale is not None: if quant_config.is_per_act_token: rhs_a1_scale = a1_scale[: topks.numel()][topks] @@ -140,8 +153,6 @@ class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): b_a1_scale[idx, :rows] = b_s[:rows] else: b_a1_scale[idx, : b_s.shape[0]] = b_s - else: - b_a1[idx, :rows, :] = rhs assert b_a1_scale is None or b_a1_scale.ndim == 3 diff --git a/vllm/model_executor/layers/fused_moe/topk_weight_and_reduce.py b/vllm/model_executor/layers/fused_moe/topk_weight_and_reduce.py index 81c84d64be..f9543358ec 100644 --- a/vllm/model_executor/layers/fused_moe/topk_weight_and_reduce.py +++ b/vllm/model_executor/layers/fused_moe/topk_weight_and_reduce.py @@ -164,13 +164,20 @@ class TopKWeightAndReduceNaiveBatched(mk.TopKWeightAndReduce): first_expert = num_local_experts * self.rank last_expert = first_expert + num_local_experts - for expert_id in range(first_expert, last_expert): - matching_tokens = topk_ids == expert_id - topks = torch.any(matching_tokens, dim=1).flatten() - rows = torch.count_nonzero(topks) - rhs = fused_expert_output[expert_id - first_expert, :rows, :] - if not apply_router_weight_on_input: - rhs.mul_(topk_weights[matching_tokens].view(rhs.size(0), 1)) - output[topks] = output[topks] + rhs + # Vectorized weighted scatter-add (single nonzero sync instead of a + # per-expert loop; mirrors the dispatch path). + local_ids = torch.arange( + first_expert, last_expert, device=topk_ids.device + ).view(-1, 1, 1) + matching = topk_ids.unsqueeze(0) == local_ids # [E_local, T, topk] + hits = matching.any(dim=2) # [E_local, T] + slots = hits.to(torch.int32).cumsum(dim=1) - 1 # [E_local, T] + e_idx, t_idx = hits.nonzero(as_tuple=True) + gathered = fused_expert_output[e_idx, slots[e_idx, t_idx], :] + if not apply_router_weight_on_input: + # weight of token t at expert e: the topk weight on the matching slot + weights = (matching * topk_weights.unsqueeze(0)).sum(dim=2) # [E_local, T] + gathered = gathered * weights[e_idx, t_idx].unsqueeze(1).to(gathered.dtype) + output.index_add_(0, t_idx, gathered.to(output.dtype)) return output diff --git a/vllm/triton_utils/__init__.py b/vllm/triton_utils/__init__.py index f4866a702d..e20cb002cd 100644 --- a/vllm/triton_utils/__init__.py +++ b/vllm/triton_utils/__init__.py @@ -17,7 +17,17 @@ else: tl = TritonLanguagePlaceholder() tldevice = TritonLanguagePlaceholder() +from vllm.triton_utils.tensor_descriptor import use_tensor_descriptor + LOG2E = 1.4426950408889634 LOGE2 = 0.6931471805599453 -__all__ = ["HAS_TRITON", "triton", "tl", "tldevice", "LOG2E", "LOGE2"] +__all__ = [ + "HAS_TRITON", + "triton", + "tl", + "tldevice", + "LOG2E", + "LOGE2", + "use_tensor_descriptor", +] diff --git a/vllm/triton_utils/tensor_descriptor.py b/vllm/triton_utils/tensor_descriptor.py new file mode 100644 index 0000000000..4472d39c9c --- /dev/null +++ b/vllm/triton_utils/tensor_descriptor.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + + +def use_tensor_descriptor(override: bool | None = None) -> bool: + """Tri-state VLLM_TRITON_USE_TD: unset=auto (on for XPU), 1/0=force on/off.""" + from vllm import envs + from vllm.platforms import current_platform + + if override is None: + override = envs.VLLM_TRITON_USE_TD + if override is not None: + return override + return current_platform.is_xpu()