[BugFix] Fix bad words with speculative decoding (#31908)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-01-07 15:46:42 -05:00
committed by GitHub
parent 6170d47d22
commit 10ef65eded
2 changed files with 26 additions and 22 deletions
+13 -8
View File
@@ -42,11 +42,16 @@ def apply_bad_words_with_drafts(
num_draft_tokens: list[int],
) -> None:
start_idx = 0
for i, bad_words_ids in bad_words_token_ids.items():
for draft_idx in range(num_draft_tokens[i]):
_apply_bad_words_single_batch(
logits[start_idx + draft_idx],
bad_words_ids,
past_tokens_ids[start_idx + draft_idx],
)
start_idx += num_draft_tokens[i]
remaining = len(bad_words_token_ids)
for i, n in enumerate(num_draft_tokens):
if (bad_words_ids := bad_words_token_ids.get(i)) is not None:
for draft_idx in range(start_idx, start_idx + n):
_apply_bad_words_single_batch(
logits[draft_idx],
bad_words_ids,
past_tokens_ids[draft_idx],
)
remaining -= 1
if not remaining:
break
start_idx += n