[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)
Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
Dakai An
OpenAI Codex
aoshen02
parent
117cf43fb6
commit
78fdf4efb8
@@ -75,6 +75,10 @@ def test_trtllm_bf16_moe_modular_no_graph(
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
|
||||
# The FlashInfer conversion may rewrite unpadded input storage in place.
|
||||
# Preserve the original layout for the independent torch reference.
|
||||
reference_w1 = w1.clone()
|
||||
reference_w2 = w2.clone()
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
scores = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(scores, topk)
|
||||
@@ -133,8 +137,8 @@ def test_trtllm_bf16_moe_modular_no_graph(
|
||||
|
||||
torch_output = torch_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
reference_w1,
|
||||
reference_w2,
|
||||
score,
|
||||
topk,
|
||||
activation=MoEActivation.SILU,
|
||||
|
||||
Reference in New Issue
Block a user