[Kernel] Add fused grouped_topk kernel for MoE (#23274)
Signed-off-by: Xin Yang <xyangx@amazon.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
@@ -1502,6 +1502,17 @@ def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
|
||||
gating_output)
|
||||
|
||||
|
||||
def grouped_topk(scores: torch.Tensor, scores_with_bias: torch.Tensor,
|
||||
num_expert_group: int, topk_group: int, topk: int,
|
||||
renormalize: bool, routed_scaling_factor: float):
|
||||
if not current_platform.is_cuda():
|
||||
raise NotImplementedError("The fused grouped_topk kernel is only "
|
||||
"available on CUDA platforms")
|
||||
return torch.ops._moe_C.grouped_topk(scores, scores_with_bias,
|
||||
num_expert_group, topk_group, topk,
|
||||
renormalize, routed_scaling_factor)
|
||||
|
||||
|
||||
def moe_wna16_marlin_gemm(input: torch.Tensor, output: Optional[torch.Tensor],
|
||||
b_qweight: torch.Tensor,
|
||||
b_bias: Optional[torch.Tensor],
|
||||
|
||||
Reference in New Issue
Block a user