[Perf] Optimize fused_topk_bias for DSv4, 1.5~2x kernel performance improvement (#47463)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -170,13 +170,12 @@ def fused_topk_bias(
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
):
|
||||
# The topk kernel dispatches dtype based on topk_ids (set by
|
||||
# indices_type) and assumes input_tokens/hash_indices_table match.
|
||||
if indices_type is not None:
|
||||
if input_tokens is not None and input_tokens.dtype != indices_type:
|
||||
input_tokens = input_tokens.to(dtype=indices_type)
|
||||
if hash_indices_table is not None and hash_indices_table.dtype != indices_type:
|
||||
hash_indices_table = hash_indices_table.to(dtype=indices_type)
|
||||
if (
|
||||
input_tokens is not None
|
||||
and hash_indices_table is not None
|
||||
and input_tokens.dtype != hash_indices_table.dtype
|
||||
):
|
||||
input_tokens = input_tokens.to(dtype=hash_indices_table.dtype)
|
||||
|
||||
if not rocm_aiter_ops.is_fused_moe_enabled():
|
||||
assert hidden_states.size(0) == gating_output.size(0), (
|
||||
@@ -304,6 +303,7 @@ def fused_topk_bias(
|
||||
scores_for_choice = scores.view(-1, n_routed_experts)
|
||||
# For batch invariance, use sorted=True to ensure deterministic expert selection
|
||||
if hash_indices_table is not None:
|
||||
assert input_tokens is not None
|
||||
topk_indices = hash_indices_table[input_tokens]
|
||||
else:
|
||||
use_sorted = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
Reference in New Issue
Block a user