[BugFix] Accuracy fix for llama4 int4 - improperly casted scales (#16801)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
This commit is contained in:
Lucas Wilkinson
2025-04-17 22:13:29 -07:00
committed by GitHub
parent 6a0f547561
commit 7eb4255628
3 changed files with 6 additions and 9 deletions
+2 -2
View File
@@ -51,8 +51,8 @@ class Llama4MoE(nn.Module):
renormalize: bool,
) -> Tuple[torch.Tensor, torch.Tensor]:
router_scores, router_indices = fast_topk(gating_output, topk, dim=-1)
router_scores = torch.sigmoid(router_scores.float()).to(
hidden_states.dtype)
# psuedo-standard is that the router scores are floats
router_scores = torch.sigmoid(router_scores.float())
return (router_scores, router_indices.to(torch.int32))
def __init__(self,