[BugFix] Fix returned logprobs with spec decode + prefill chunking (#29216)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-11-22 22:41:25 +08:00
committed by GitHub
parent 066209a045
commit d44a63c6d6
3 changed files with 22 additions and 15 deletions
+4 -1
View File
@@ -81,7 +81,10 @@ class Sampler(nn.Module):
if logprobs_mode == "raw_logprobs":
raw_logprobs = self.compute_logprobs(logits)
elif logprobs_mode == "raw_logits":
raw_logprobs = logits.clone()
if logits.dtype == torch.float32:
raw_logprobs = logits.clone()
else:
raw_logprobs = logits.to(torch.float32)
# Use float32 for the logits.
logits = logits.to(torch.float32)