[Perf][MoE] Cheaper triton_kernels 3.8 routing build in the OAI Triton MoE (#58897)
Signed-off-by: Zheng Gong <zgong@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Cursor
Shanshan Shen
parent
a1f51a0226
commit
2308b6f00b
@@ -431,11 +431,6 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||
|
||||
if gptoss_moe.triton_kernels_version == "3.5.1":
|
||||
pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+")
|
||||
if gptoss_moe.triton_kernels_version == "3.8":
|
||||
pytest.skip(
|
||||
"3.8 rebuilds routing from raw ids, so routing_data_from_sparse_topk "
|
||||
"just forwards to make_routing_data and the comparison is vacuous"
|
||||
)
|
||||
|
||||
from triton_kernels.topk import topk as topk_fn
|
||||
|
||||
@@ -467,11 +462,52 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||
elif hasattr(ref, "__dict__"):
|
||||
for k, v in vars(ref).items():
|
||||
assert_equivalent(getattr(new, k), v, f"{path}.{k}")
|
||||
elif hasattr(type(ref), "__slots__"):
|
||||
for k in type(ref).__slots__:
|
||||
assert_equivalent(getattr(new, k), getattr(ref, k), f"{path}.{k}")
|
||||
else:
|
||||
assert type(new) is type(ref), f"{path}: {type(new)} vs {type(ref)}"
|
||||
|
||||
assert_equivalent(rd_new, rd_ref)
|
||||
if isinstance(gather_ref, torch.Tensor):
|
||||
torch.testing.assert_close(gather_new, gather_ref)
|
||||
torch.testing.assert_close(scatter_new, scatter_ref)
|
||||
return
|
||||
torch.testing.assert_close(gather_new.src_indx, gather_ref.src_indx)
|
||||
torch.testing.assert_close(gather_new.dst_indx, gather_ref.dst_indx)
|
||||
torch.testing.assert_close(scatter_new.src_indx, scatter_ref.src_indx)
|
||||
torch.testing.assert_close(scatter_new.dst_indx, scatter_ref.dst_indx)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_tokens", [1, 33, 4099])
|
||||
@pytest.mark.parametrize("n_experts,topk", [(128, 4), (256, 6)])
|
||||
def test_make_routing_data_tk38_matches_reference(n_tokens, n_experts, topk):
|
||||
from vllm.model_executor.layers.fused_moe.experts import (
|
||||
gpt_oss_triton_kernels_moe as gptoss_moe,
|
||||
)
|
||||
|
||||
if gptoss_moe.triton_kernels_version != "3.8":
|
||||
pytest.skip("requires triton_kernels 3.8")
|
||||
|
||||
torch.manual_seed(0)
|
||||
topk_ids = torch.stack(
|
||||
[torch.randperm(n_experts, device="cuda")[:topk] for _ in range(n_tokens)]
|
||||
).to(torch.int32)
|
||||
topk_ids[torch.rand(topk_ids.shape, device="cuda") < 0.5] = -1
|
||||
topk_weights = torch.rand(topk_ids.shape, device="cuda")
|
||||
|
||||
routing, gather_tok, scatter_tok = gptoss_moe.make_routing_data(
|
||||
topk_ids, topk_weights, n_experts
|
||||
)
|
||||
|
||||
flat = topk_ids.reshape(-1).to(torch.int64)
|
||||
order = torch.argsort(flat.clamp(min=0), stable=True)
|
||||
counts = torch.bincount(flat.clamp(min=0), minlength=n_experts).to(torch.int32)
|
||||
weights = topk_weights.reshape(-1) * (flat >= 0).to(torch.float32)
|
||||
|
||||
assert torch.equal(routing.ragged.slice_sizes, counts)
|
||||
assert torch.equal(gather_tok, (order // topk).to(torch.int32))
|
||||
assert torch.equal(scatter_tok, gather_tok)
|
||||
assert torch.equal(routing.gate_scal, weights[order])
|
||||
assert routing.n_expts_act == topk
|
||||
assert routing.n_valid == flat.numel()
|
||||
|
||||
@@ -847,6 +847,20 @@ def triton_kernel_fused_experts(
|
||||
return output_tensor
|
||||
|
||||
|
||||
def _tk38_routing_data(
|
||||
col_sum: torch.Tensor,
|
||||
order: torch.Tensor,
|
||||
flat_w: torch.Tensor,
|
||||
topk: int,
|
||||
) -> tuple["RoutingData", torch.Tensor, torch.Tensor]:
|
||||
"""Build 3.8 routing from per-expert counts and the stable expert order."""
|
||||
nrows = order.shape[0]
|
||||
ragged = make_ragged_tensor_metadata(col_sum, nrows)
|
||||
gather_tok = (order // topk).to(torch.int32)
|
||||
routing = RoutingData(ragged, flat_w[order], gather_tok, topk, nrows)
|
||||
return routing, gather_tok, gather_tok
|
||||
|
||||
|
||||
def make_routing_data(
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
@@ -854,21 +868,18 @@ def make_routing_data(
|
||||
) -> tuple["RoutingData", torch.Tensor, torch.Tensor]:
|
||||
if triton_kernels_version == "3.8":
|
||||
_, topk = topk_ids.shape
|
||||
flat_e = topk_ids.reshape(-1)
|
||||
valid = flat_e >= 0
|
||||
flat_e = flat_e.to(torch.int64).clamp(min=0)
|
||||
flat_w = topk_weights.reshape(-1).to(torch.float32) * valid.to(torch.float32)
|
||||
nrows = flat_e.shape[0]
|
||||
order = torch.argsort(flat_e, stable=True)
|
||||
col_sum = torch.zeros(
|
||||
num_local_experts, dtype=torch.int32, device=flat_e.device
|
||||
flat_ids = topk_ids.reshape(-1)
|
||||
# -1 (non-local) slots go to expert 0 with zero weight.
|
||||
keys = flat_ids.clamp(min=0)
|
||||
sorted_keys, order = torch.sort(keys, stable=True)
|
||||
experts = torch.arange(
|
||||
num_local_experts + 1, dtype=keys.dtype, device=keys.device
|
||||
)
|
||||
col_sum.scatter_add_(0, flat_e, torch.ones_like(flat_e, dtype=torch.int32))
|
||||
ragged = make_ragged_tensor_metadata(col_sum, nrows)
|
||||
gather_tok = (order // topk).to(torch.int32)
|
||||
gammas = flat_w[order]
|
||||
routing = RoutingData(ragged, gammas, gather_tok, topk, nrows)
|
||||
return routing, gather_tok, gather_tok
|
||||
edges = torch.searchsorted(sorted_keys, experts, out_int32=True)
|
||||
col_sum = edges[1:] - edges[:-1]
|
||||
valid = (flat_ids >= 0).to(torch.float32)
|
||||
flat_w = topk_weights.reshape(-1).to(torch.float32) * valid
|
||||
return _tk38_routing_data(col_sum, order, flat_w, topk)
|
||||
|
||||
topk_ids = topk_ids.to(torch.int16)
|
||||
topk_weights = topk_weights.to(torch.bfloat16)
|
||||
@@ -936,8 +947,14 @@ def routing_data_from_sparse_topk(
|
||||
remapping) that work is redundant.
|
||||
"""
|
||||
if triton_kernels_version == "3.8":
|
||||
# 3.8 routing is rebuilt from raw ids; no SparseMatrix reuse.
|
||||
return make_routing_data(sparse_topk.indx, sparse_topk.vals, n_expts)
|
||||
# topk ids have no -1 slots, so the topk metadata matches make_routing_data.
|
||||
mask_metadata = sparse_topk.mask_metadata
|
||||
return _tk38_routing_data(
|
||||
mask_metadata.col_sum,
|
||||
mask_metadata.col_sorted_indx,
|
||||
sparse_topk.vals.reshape(-1).to(torch.float32),
|
||||
sparse_topk.indx.shape[1],
|
||||
)
|
||||
|
||||
dispatch_indx = sparse_topk.mask_metadata.row_sorted_indx
|
||||
combine_indx = sparse_topk.mask_metadata.col_sorted_indx
|
||||
|
||||
Reference in New Issue
Block a user