[BugFix] Fix returned logprobs with spec decode + prefill chunking (#29216)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -81,7 +81,10 @@ class Sampler(nn.Module):
|
||||
if logprobs_mode == "raw_logprobs":
|
||||
raw_logprobs = self.compute_logprobs(logits)
|
||||
elif logprobs_mode == "raw_logits":
|
||||
raw_logprobs = logits.clone()
|
||||
if logits.dtype == torch.float32:
|
||||
raw_logprobs = logits.clone()
|
||||
else:
|
||||
raw_logprobs = logits.to(torch.float32)
|
||||
|
||||
# Use float32 for the logits.
|
||||
logits = logits.to(torch.float32)
|
||||
|
||||
Reference in New Issue
Block a user