[Perf] Use torch compile to fuse pack topk in trtllm moe (#37695)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
This commit is contained in:
Wei Zhao
2026-03-27 17:30:46 -06:00
committed by GitHub
parent 88149b635e
commit b69bf2f0b1
3 changed files with 18 additions and 8 deletions
@@ -323,3 +323,16 @@ def normalize_batched_scales_shape(
@functools.cache
def disable_inplace() -> bool:
return is_torch_equal_or_newer("2.9")
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
def trtllm_moe_pack_topk_ids_weights(
topk_ids: torch.Tensor, topk_weights: torch.Tensor
) -> torch.Tensor:
"""
Pack topk_ids and topk_weights into a single int32 tensor.
Format: (expert_id << 16) | weight_bf16.view(int16)
"""
return (topk_ids.to(torch.int32) << 16) | topk_weights.to(torch.bfloat16).view(
torch.int16
)