[ModelBash][DSV3] Add TRTLLM DSV3 Router GEMM kernel (6% B1 Speedup) (#34302)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
b1b5e045df
commit
8435b2e049
@@ -2190,6 +2190,21 @@ def moe_wna16_gemm(
|
||||
)
|
||||
|
||||
|
||||
def dsv3_router_gemm(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty(
|
||||
hidden_states.shape[0],
|
||||
router_weight.shape[0],
|
||||
device=hidden_states.device,
|
||||
dtype=output_dtype,
|
||||
)
|
||||
torch.ops._moe_C.dsv3_router_gemm(output, hidden_states, router_weight)
|
||||
return output
|
||||
|
||||
|
||||
def topk_softmax(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user