[Kernel] moe wna16 marlin kernel (#14447)
Signed-off-by: Jinzhen Lin <linjinzhen@hotmail.com> Co-authored-by: Michael Goin <michael@neuralmagic.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Michael Goin
mgoin
parent
6b40996ae8
commit
d06ba4ed3f
@@ -1245,6 +1245,29 @@ def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
|
||||
token_expert_indicies, gating_output)
|
||||
|
||||
|
||||
def moe_wna16_marlin_gemm(input: torch.Tensor, output: Optional[torch.Tensor],
|
||||
b_qweight: torch.Tensor, b_scales: torch.Tensor,
|
||||
b_qzeros: Optional[torch.Tensor],
|
||||
g_idx: Optional[torch.Tensor],
|
||||
perm: Optional[torch.Tensor],
|
||||
workspace: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_past_padded: torch.Tensor,
|
||||
topk_weights: torch.Tensor, moe_block_size: int,
|
||||
top_k: int, mul_topk_weights: bool, is_ep: bool,
|
||||
b_q_type: ScalarType, size_m: int, size_n: int,
|
||||
size_k: int, is_k_full: bool, use_atomic_add: bool,
|
||||
use_fp32_reduce: bool,
|
||||
is_zp_float: bool) -> torch.Tensor:
|
||||
return torch.ops._moe_C.moe_wna16_marlin_gemm(
|
||||
input, output, b_qweight, b_scales, b_qzeros, g_idx, perm, workspace,
|
||||
sorted_token_ids, expert_ids, num_tokens_past_padded, topk_weights,
|
||||
moe_block_size, top_k, mul_topk_weights, is_ep, b_q_type.id, size_m,
|
||||
size_n, size_k, is_k_full, use_atomic_add, use_fp32_reduce,
|
||||
is_zp_float)
|
||||
|
||||
|
||||
if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
|
||||
|
||||
@register_fake("_moe_C::marlin_gemm_moe")
|
||||
@@ -1263,6 +1286,29 @@ if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
|
||||
dtype=a.dtype,
|
||||
device=a.device)
|
||||
|
||||
@register_fake("_moe_C::moe_wna16_marlin_gemm")
|
||||
def moe_wna16_marlin_gemm_fake(input: torch.Tensor,
|
||||
output: Optional[torch.Tensor],
|
||||
b_qweight: torch.Tensor,
|
||||
b_scales: torch.Tensor,
|
||||
b_qzeros: Optional[torch.Tensor],
|
||||
g_idx: Optional[torch.Tensor],
|
||||
perm: Optional[torch.Tensor],
|
||||
workspace: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_past_padded: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
moe_block_size: int, top_k: int,
|
||||
mul_topk_weights: bool, is_ep: bool,
|
||||
b_q_type: ScalarType, size_m: int,
|
||||
size_n: int, size_k: int, is_k_full: bool,
|
||||
use_atomic_add: bool, use_fp32_reduce: bool,
|
||||
is_zp_float: bool) -> torch.Tensor:
|
||||
return torch.empty((size_m * top_k, size_n),
|
||||
dtype=input.dtype,
|
||||
device=input.device)
|
||||
|
||||
|
||||
def reshape_and_cache(
|
||||
key: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user