[Kernel] TD operand loads for batched MoE GEMM (moe_mmk) on XPU (#46340)

Signed-off-by: oonyshch <xonyshch@gmail.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Lena Onyshchenko
2026-07-26 08:50:07 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent d30b1ecd1b
commit 0111002323
9 changed files with 331 additions and 33 deletions
+184
View File
@@ -351,3 +351,187 @@ def test_fused_moe_batched_experts(
torch.testing.assert_close(batched_output, baseline_output, atol=3e-2, rtol=2e-2)
torch.testing.assert_close(triton_output, batched_output, atol=2e-2, rtol=2e-2)
# USE_TD in moe_mmk
# K % 8 == 0 (bf16, 16-byte alignment)
_TD_SHAPES = [
(4, 1, 128, 256),
(8, 64, 512, 512),
(4, 256, 1024, 2048),
]
def _td_supported() -> bool:
return current_platform.is_xpu() or (
current_platform.is_cuda() and current_platform.has_device_capability(90)
)
def _run_td(A, B, num_expert_tokens, use_td: bool):
out = torch.zeros(
A.shape[0],
A.shape[1],
B.shape[1],
dtype=torch.bfloat16,
device=A.device,
)
import triton
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
batched_triton_kernel,
)
if use_td:
from vllm.triton_utils.allocation import set_triton_allocator
set_triton_allocator(A.device)
E = A.shape[0]
max_tokens = A.shape[1]
K = A.shape[2]
N = B.shape[1]
BM = BN = BK = 64
grid = (E, triton.cdiv(max_tokens, BM) * triton.cdiv(N, BN))
batched_triton_kernel[grid](
A,
B,
out,
num_expert_tokens,
tl.bfloat16,
max_tokens,
K,
N,
None,
None,
None,
A.stride(0),
A.stride(1),
A.stride(2),
B.stride(0),
B.stride(2),
B.stride(1),
out.stride(0),
out.stride(1),
out.stride(2),
0,
0,
0,
0,
0,
0,
0,
0,
False,
False,
False,
BLOCK_M=BM,
BLOCK_N=BN,
BLOCK_K=BK,
USE_TD=use_td,
)
return out
@pytest.mark.parametrize("num_experts,max_tokens_per_expert,K,N", _TD_SHAPES)
def test_batched_mm_td_matches_plain(num_experts, max_tokens_per_expert, K, N):
if not _td_supported():
pytest.skip("TD requires XPU or CUDA sm_90+")
set_random_seed(42)
device = current_platform.device_type
A = (
torch.randn(
num_experts, max_tokens_per_expert, K, device=device, dtype=torch.bfloat16
)
/ 10
)
B = torch.randn(num_experts, N, K, device=device, dtype=torch.bfloat16)
num_expert_tokens = torch.randint(
1,
max_tokens_per_expert + 1,
size=(num_experts,),
device=device,
dtype=torch.int32,
)
out_plain = _run_td(A, B, num_expert_tokens, False)
out_td = _run_td(A, B, num_expert_tokens, True)
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
def test_batched_mm_td_zero_expert_tokens():
if not _td_supported():
pytest.skip("TD requires XPU or CUDA sm_90+")
set_random_seed(42)
device = current_platform.device_type
E, M, K, N = 8, 32, 256, 256
A = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) / 10
B = torch.randn(E, N, K, device=device, dtype=torch.bfloat16)
num_expert_tokens = torch.zeros(E, device=device, dtype=torch.int32)
num_expert_tokens[::2] = M
out_plain = _run_td(A, B, num_expert_tokens, False)
out_td = _run_td(A, B, num_expert_tokens, True)
for e in range(E):
if num_expert_tokens[e].item() == 0:
assert out_plain[e].abs().max().item() == 0.0
assert out_td[e].abs().max().item() == 0.0
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
# BatchedTritonExperts device enablement (XPU)
def test_batched_triton_experts_supports_current_device():
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
)
if current_platform.is_xpu() or current_platform.is_cuda_alike():
assert BatchedTritonExperts._supports_current_device()
else:
pytest.skip("No GPU device available")
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
def test_batched_experts_end_to_end(m, n, k, e, topk):
"""End-to-end BatchedTritonExperts via the reference (no-comms)
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
the device-enablement path."""
if not (current_platform.is_xpu() or current_platform.is_cuda_alike()):
pytest.skip("No GPU device available")
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
device = current_platform.device_type
init_workspace_manager(torch.device(f"{device}:0"))
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
# w1 is the fused gate+up projection [E, 2N, K]; w2 is [E, K, N].
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
with set_current_vllm_config(vllm_config):
topk_weight, topk_ids, _ = fused_topk(a, score, topk, False)
baseline_output = torch_experts(a, w1, w2, topk_weight, topk_ids)
triton_output = batched_moe(a, w1, w2, topk_weight, topk_ids)
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
def test_batched_triton_backend_mapping():
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
map_unquantized_backend,
)
assert (
map_unquantized_backend("batched_triton")
== UnquantizedMoeBackend.BATCHED_TRITON
)
assert map_unquantized_backend("triton") == UnquantizedMoeBackend.TRITON