[Perf] Reuse topk SparseMatrix routing metadata in GPT-OSS MoE forward (#45457)

Signed-off-by: Shenglei Fu <sfu@confluent.io>
Signed-off-by: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Shenglei Fu
2026-08-29 12:14:39 -04:00
committed by GitHub
co-authored by Claude Fable 5
parent 7f4793eaa3
commit 129087ddab
2 changed files with 102 additions and 15 deletions
@@ -578,13 +578,14 @@ def triton_kernel_moe_forward(
if sm_first:
logits = torch.softmax(logits, dim=-1)
topk_result = topk_fn(logits, topk, apply_softmax=not sm_first)
is_sparse_topk = not isinstance(topk_result, tuple)
# topk may return a tuple (vals, indx, bitmatrix) or a
# SparseMatrix depending on the triton_kernels version.
if isinstance(topk_result, tuple):
topk_weights, topk_ids_raw, _ = topk_result
else:
if is_sparse_topk:
topk_weights = topk_result.vals
topk_ids_raw = topk_result.indx
else:
topk_weights, topk_ids_raw, _ = topk_result
if expert_map is not None:
# topk_ids_raw contains global expert IDs - remap to local.
@@ -597,11 +598,23 @@ def triton_kernel_moe_forward(
effective_expert_map = None
effective_global_num_experts = local_num_experts
else:
topk_ids = topk_ids_raw.to(torch.long)
routing_data, gather_idx, scatter_idx = make_routing_data(
topk_ids, topk_weights, gating_output.shape[-1]
)
effective_expert_map = expert_map
if is_sparse_topk:
# v3.6+ topk returns a SparseMatrix whose construction already
# computed the bitmatrix and its metadata; reuse it instead of
# re-packing the bitmatrix and recomputing the metadata in
# make_routing_data. Only the monolithic path reaches this reuse;
# the modular OAITritonExperts (expert-parallel) path calls
# make_routing_data directly.
routing_data, gather_idx, scatter_idx = routing_data_from_sparse_topk(
topk_result, gating_output.shape[-1]
)
else:
topk_ids = topk_ids_raw.to(torch.long)
routing_data, gather_idx, scatter_idx = make_routing_data(
topk_ids, topk_weights, gating_output.shape[-1]
)
effective_expert_map = None
effective_global_num_experts = global_num_experts
output = torch.empty_like(hidden_states)
@@ -782,18 +795,36 @@ def make_routing_data(
)
sparse_logits = SparseMatrix(indx=topk_ids, vals=topk_weights, mask=bitmatrix)
dispatch_indx = sparse_logits.mask_metadata.row_sorted_indx
combine_indx = sparse_logits.mask_metadata.col_sorted_indx
return routing_data_from_sparse_topk(sparse_logits, num_local_experts)
def routing_data_from_sparse_topk(
sparse_topk, # SparseMatrix
n_expts: int,
) -> tuple["RoutingData", "GatherIndx", "ScatterIndx"]:
"""Build routing structures from the ``SparseMatrix`` returned by
``triton_kernels.topk``, reusing its precomputed bitmatrix metadata.
``make_routing_data`` re-packs a bitmatrix from the topk ids and
constructs a second ``SparseMatrix``, recomputing metadata the topk
call already produced; when the topk ids are used as-is (no expert-map
remapping) that work is redundant.
"""
dispatch_indx = sparse_topk.mask_metadata.row_sorted_indx
combine_indx = sparse_topk.mask_metadata.col_sorted_indx
ragged_batch_metadata = make_ragged_tensor_metadata(
sparse_logits.mask_metadata.col_sum,
sparse_topk.mask_metadata.col_sum,
dispatch_indx.shape[0],
)
gate_scal = sparse_logits.vals.flatten()[combine_indx]
n_expts_act = sparse_topk.indx.shape[1]
# Unlike make_routing_data, no bfloat16 cast: gate_scal inherits the
# gating dtype, which is bfloat16 for all callers of this path.
gate_scal = sparse_topk.vals.flatten()[combine_indx]
routing_data = RoutingData(
gate_scal,
ragged_batch_metadata.block_sizes,
num_local_experts,
num_topk,
n_expts,
n_expts_act,
ragged_batch_metadata,
)
gather_indx = GatherIndx(combine_indx, dispatch_indx)