[Kernel] TD operand loads for batched MoE GEMM (moe_mmk) on XPU (#46340)
Signed-off-by: oonyshch <xonyshch@gmail.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
d30b1ecd1b
commit
0111002323
@@ -351,3 +351,187 @@ def test_fused_moe_batched_experts(
|
||||
torch.testing.assert_close(batched_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
torch.testing.assert_close(triton_output, batched_output, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
# USE_TD in moe_mmk
|
||||
|
||||
# K % 8 == 0 (bf16, 16-byte alignment)
|
||||
_TD_SHAPES = [
|
||||
(4, 1, 128, 256),
|
||||
(8, 64, 512, 512),
|
||||
(4, 256, 1024, 2048),
|
||||
]
|
||||
|
||||
|
||||
def _td_supported() -> bool:
|
||||
return current_platform.is_xpu() or (
|
||||
current_platform.is_cuda() and current_platform.has_device_capability(90)
|
||||
)
|
||||
|
||||
|
||||
def _run_td(A, B, num_expert_tokens, use_td: bool):
|
||||
out = torch.zeros(
|
||||
A.shape[0],
|
||||
A.shape[1],
|
||||
B.shape[1],
|
||||
dtype=torch.bfloat16,
|
||||
device=A.device,
|
||||
)
|
||||
import triton
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
batched_triton_kernel,
|
||||
)
|
||||
|
||||
if use_td:
|
||||
from vllm.triton_utils.allocation import set_triton_allocator
|
||||
|
||||
set_triton_allocator(A.device)
|
||||
|
||||
E = A.shape[0]
|
||||
max_tokens = A.shape[1]
|
||||
K = A.shape[2]
|
||||
N = B.shape[1]
|
||||
BM = BN = BK = 64
|
||||
grid = (E, triton.cdiv(max_tokens, BM) * triton.cdiv(N, BN))
|
||||
batched_triton_kernel[grid](
|
||||
A,
|
||||
B,
|
||||
out,
|
||||
num_expert_tokens,
|
||||
tl.bfloat16,
|
||||
max_tokens,
|
||||
K,
|
||||
N,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
A.stride(0),
|
||||
A.stride(1),
|
||||
A.stride(2),
|
||||
B.stride(0),
|
||||
B.stride(2),
|
||||
B.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
BLOCK_M=BM,
|
||||
BLOCK_N=BN,
|
||||
BLOCK_K=BK,
|
||||
USE_TD=use_td,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts,max_tokens_per_expert,K,N", _TD_SHAPES)
|
||||
def test_batched_mm_td_matches_plain(num_experts, max_tokens_per_expert, K, N):
|
||||
if not _td_supported():
|
||||
pytest.skip("TD requires XPU or CUDA sm_90+")
|
||||
set_random_seed(42)
|
||||
device = current_platform.device_type
|
||||
A = (
|
||||
torch.randn(
|
||||
num_experts, max_tokens_per_expert, K, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
/ 10
|
||||
)
|
||||
B = torch.randn(num_experts, N, K, device=device, dtype=torch.bfloat16)
|
||||
num_expert_tokens = torch.randint(
|
||||
1,
|
||||
max_tokens_per_expert + 1,
|
||||
size=(num_experts,),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
out_plain = _run_td(A, B, num_expert_tokens, False)
|
||||
out_td = _run_td(A, B, num_expert_tokens, True)
|
||||
|
||||
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
|
||||
|
||||
|
||||
def test_batched_mm_td_zero_expert_tokens():
|
||||
if not _td_supported():
|
||||
pytest.skip("TD requires XPU or CUDA sm_90+")
|
||||
set_random_seed(42)
|
||||
device = current_platform.device_type
|
||||
E, M, K, N = 8, 32, 256, 256
|
||||
A = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) / 10
|
||||
B = torch.randn(E, N, K, device=device, dtype=torch.bfloat16)
|
||||
num_expert_tokens = torch.zeros(E, device=device, dtype=torch.int32)
|
||||
num_expert_tokens[::2] = M
|
||||
|
||||
out_plain = _run_td(A, B, num_expert_tokens, False)
|
||||
out_td = _run_td(A, B, num_expert_tokens, True)
|
||||
|
||||
for e in range(E):
|
||||
if num_expert_tokens[e].item() == 0:
|
||||
assert out_plain[e].abs().max().item() == 0.0
|
||||
assert out_td[e].abs().max().item() == 0.0
|
||||
|
||||
torch.testing.assert_close(out_td, out_plain, atol=6e-2, rtol=6e-2)
|
||||
|
||||
|
||||
# BatchedTritonExperts device enablement (XPU)
|
||||
def test_batched_triton_experts_supports_current_device():
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
)
|
||||
|
||||
if current_platform.is_xpu() or current_platform.is_cuda_alike():
|
||||
assert BatchedTritonExperts._supports_current_device()
|
||||
else:
|
||||
pytest.skip("No GPU device available")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k,e,topk", [(32, 512, 512, 8, 2), (45, 1024, 128, 8, 1)])
|
||||
def test_batched_experts_end_to_end(m, n, k, e, topk):
|
||||
"""End-to-end BatchedTritonExperts via the reference (no-comms)
|
||||
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
|
||||
the device-enablement path."""
|
||||
if not (current_platform.is_xpu() or current_platform.is_cuda_alike()):
|
||||
pytest.skip("No GPU device available")
|
||||
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
device = current_platform.device_type
|
||||
init_workspace_manager(torch.device(f"{device}:0"))
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
|
||||
# w1 is the fused gate+up projection [E, 2N, K]; w2 is [E, K, N].
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=torch.bfloat16) / 15
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=torch.bfloat16) / 15
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
topk_weight, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
|
||||
baseline_output = torch_experts(a, w1, w2, topk_weight, topk_ids)
|
||||
triton_output = batched_moe(a, w1, w2, topk_weight, topk_ids)
|
||||
|
||||
torch.testing.assert_close(triton_output, baseline_output, atol=3e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def test_batched_triton_backend_mapping():
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
map_unquantized_backend,
|
||||
)
|
||||
|
||||
assert (
|
||||
map_unquantized_backend("batched_triton")
|
||||
== UnquantizedMoeBackend.BATCHED_TRITON
|
||||
)
|
||||
assert map_unquantized_backend("triton") == UnquantizedMoeBackend.TRITON
|
||||
|
||||
Reference in New Issue
Block a user