[Perf] Optimize fused_topk_bias for DSv4, 1.5~2x kernel performance improvement (#47463)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-07-15 23:40:55 +00:00
committed by GitHub
parent 015b0320de
commit 2dab187f75
3 changed files with 78 additions and 69 deletions
@@ -170,13 +170,12 @@ def fused_topk_bias(
hash_indices_table: torch.Tensor | None = None,
routed_scaling_factor: float = 1.0,
):
# The topk kernel dispatches dtype based on topk_ids (set by
# indices_type) and assumes input_tokens/hash_indices_table match.
if indices_type is not None:
if input_tokens is not None and input_tokens.dtype != indices_type:
input_tokens = input_tokens.to(dtype=indices_type)
if hash_indices_table is not None and hash_indices_table.dtype != indices_type:
hash_indices_table = hash_indices_table.to(dtype=indices_type)
if (
input_tokens is not None
and hash_indices_table is not None
and input_tokens.dtype != hash_indices_table.dtype
):
input_tokens = input_tokens.to(dtype=hash_indices_table.dtype)
if not rocm_aiter_ops.is_fused_moe_enabled():
assert hidden_states.size(0) == gating_output.size(0), (
@@ -304,6 +303,7 @@ def fused_topk_bias(
scores_for_choice = scores.view(-1, n_routed_experts)
# For batch invariance, use sorted=True to ensure deterministic expert selection
if hash_indices_table is not None:
assert input_tokens is not None
topk_indices = hash_indices_table[input_tokens]
else:
use_sorted = envs.VLLM_BATCH_INVARIANT