[Perf][MoE] Skip top-k slots routed to non-local experts in TritonExp… (#58051)
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Signed-off-by: Shuolei Wang <948904026@qq.com>
This commit is contained in:
@@ -107,7 +107,16 @@ SEEDS = [0]
|
||||
|
||||
|
||||
def torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
w1_s,
|
||||
w2_s,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
block_shape,
|
||||
silu_fp32=False,
|
||||
expert_map=None,
|
||||
):
|
||||
"""Fused MoE with block-wise fp8 quantization using native torch.
|
||||
|
||||
@@ -123,6 +132,9 @@ def torch_w8a8_block_fp8_moe(
|
||||
|
||||
topk_weight = topk_weight.view(-1)
|
||||
topk_ids = topk_ids.view(-1)
|
||||
if expert_map is not None:
|
||||
# w1/w2 hold this rank's experts; slots mapped to -1 contribute nothing.
|
||||
topk_ids = expert_map[topk_ids]
|
||||
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
# Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the
|
||||
@@ -301,6 +313,54 @@ def test_w8a8_block_fp8_clamp_skips_fused_silu_quant(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), [(83, 512, 512), (2048, 4608, 512)])
|
||||
@pytest.mark.parametrize("topk", [2, 6])
|
||||
@pytest.mark.parametrize("ep_size", [2, 4])
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_fused_moe_expert_map(M, N, K, topk, ep_size, workspace_init):
|
||||
"""One EP rank: local expert shard plus an expert_map with -1 entries."""
|
||||
E, block_size, dtype = 16, [128, 128], torch.bfloat16
|
||||
torch.manual_seed(0)
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
score = torch.randn((M, E), dtype=dtype)
|
||||
w1, w2, quant_config = make_test_quant_config(
|
||||
E, N, K, dtype, quant_dtype=torch.float8_e4m3fn, block_shape=block_size
|
||||
)
|
||||
|
||||
local_e = E // ep_size
|
||||
e_ids = torch.randperm(E, dtype=torch.int32)[:local_e]
|
||||
e_map = torch.full((E,), -1, dtype=torch.int32)
|
||||
e_map[e_ids] = torch.arange(local_e, dtype=torch.int32)
|
||||
w1, w2 = w1[e_ids].contiguous(), w2[e_ids].contiguous()
|
||||
w1_s = quant_config.w1_scale[e_ids].contiguous()
|
||||
w2_s = quant_config.w2_scale[e_ids].contiguous()
|
||||
quant_config = fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_s, w2_scale=w2_s, block_shape=block_size
|
||||
)
|
||||
|
||||
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = torch_w8a8_block_fp8_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size, expert_map=e_map
|
||||
)
|
||||
m_out = m_fused_moe.apply(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=e_map,
|
||||
global_num_experts=E,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(m_out, ref_out, atol=0.035, rtol=0.035)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
|
||||
@pytest.mark.parametrize("E", E)
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
|
||||
Reference in New Issue
Block a user