[Perf] Use torch compile to fuse pack topk in trtllm moe (#37695)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
@@ -323,3 +323,16 @@ def normalize_batched_scales_shape(
|
||||
@functools.cache
|
||||
def disable_inplace() -> bool:
|
||||
return is_torch_equal_or_newer("2.9")
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def trtllm_moe_pack_topk_ids_weights(
|
||||
topk_ids: torch.Tensor, topk_weights: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Pack topk_ids and topk_weights into a single int32 tensor.
|
||||
Format: (expert_id << 16) | weight_bf16.view(int16)
|
||||
"""
|
||||
return (topk_ids.to(torch.int32) << 16) | topk_weights.to(torch.bfloat16).view(
|
||||
torch.int16
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user