[FEAT][ROCm] Add AITER grouped topk for DeepSeekV2 (#18825)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
@@ -45,7 +45,7 @@ else:
|
||||
FusedMoEPrepareAndFinalize = None # type: ignore
|
||||
if is_rocm_aiter_moe_enabled():
|
||||
from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( # noqa: E501
|
||||
rocm_aiter_biased_group_topk as grouped_topk)
|
||||
rocm_aiter_grouped_topk as grouped_topk)
|
||||
else:
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import grouped_topk
|
||||
if current_platform.is_tpu():
|
||||
|
||||
@@ -140,6 +140,36 @@ def rocm_aiter_biased_grouped_topk_fake(
|
||||
pass
|
||||
|
||||
|
||||
def rocm_aiter_grouped_topk_impl(
|
||||
gating_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_expert_group: int,
|
||||
topk_group: int,
|
||||
need_renorm: bool,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0 # mul to topk_weights
|
||||
) -> None:
|
||||
|
||||
from aiter import grouped_topk
|
||||
|
||||
grouped_topk(gating_output, topk_weights, topk_ids, num_expert_group,
|
||||
topk_group, need_renorm, scoring_func, routed_scaling_factor)
|
||||
|
||||
|
||||
def rocm_aiter_grouped_topk_fake(
|
||||
gating_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_expert_group: int,
|
||||
topk_group: int,
|
||||
need_renorm: bool,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0 # mul to topk_weights
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def rocm_aiter_fused_moe_impl(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
@@ -218,36 +248,54 @@ if current_platform.is_rocm():
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_grouped_topk",
|
||||
op_func=rocm_aiter_grouped_topk_impl,
|
||||
mutates_args=["topk_weights", "topk_ids"],
|
||||
fake_impl=rocm_aiter_grouped_topk_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
def rocm_aiter_biased_group_topk(
|
||||
|
||||
def rocm_aiter_grouped_topk(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
num_expert_group: int = 0,
|
||||
topk_group: int = 0,
|
||||
scoring_func: str = "sigmoid",
|
||||
scoring_func: str = "softmax",
|
||||
e_score_correction_bias: Optional[torch.Tensor] = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert scoring_func == "sigmoid", (
|
||||
"rocm_aiter_biased_group_topk only supports 'sigmoid' scoring_func.")
|
||||
assert e_score_correction_bias is not None, (
|
||||
"'e_score_correction_bias' must not be None.")
|
||||
token = hidden_states.shape[0]
|
||||
device = hidden_states.device
|
||||
topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
topk_weights = torch.empty((token, topk),
|
||||
dtype=torch.float32,
|
||||
device=device)
|
||||
torch.ops.vllm.rocm_aiter_biased_grouped_topk(
|
||||
gating_output,
|
||||
e_score_correction_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
renormalize,
|
||||
)
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
torch.ops.vllm.rocm_aiter_biased_grouped_topk(
|
||||
gating_output,
|
||||
e_score_correction_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
renormalize,
|
||||
)
|
||||
else:
|
||||
assert (scoring_func == "softmax" or scoring_func == "sigmoid")
|
||||
torch.ops.vllm.rocm_aiter_grouped_topk(
|
||||
gating_output,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
renormalize,
|
||||
scoring_func,
|
||||
)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user