[BugFix] Fix logprobs with spec decode and modified logits (#30846)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-12-18 19:58:28 -08:00
committed by GitHub
parent 7b43db210c
commit 2ac85a4544
2 changed files with 30 additions and 11 deletions
+7 -1
View File
@@ -119,8 +119,14 @@ class RejectionSampler(nn.Module):
raw_target_logits = logits[target_logits_indices]
# Use float32 for the target_logits.
raw_target_logits = raw_target_logits.to(torch.float32)
target_logits = raw_target_logits
if not self.is_processed_logprobs_mode:
# Clone raw_target_logits before applying processors to preserve
# the original raw logits for logprobs computation, since
# apply_logits_processors modifies the tensor in-place.
target_logits = target_logits.clone()
target_logits = self.apply_logits_processors(
raw_target_logits, sampling_metadata, metadata
target_logits, sampling_metadata, metadata
)
# [num_tokens, vocab_size]
# NOTE(woosuk): `target_logits` can be updated in place inside the