[BugFix] Fix bad words with speculative decoding (#31908)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -42,11 +42,16 @@ def apply_bad_words_with_drafts(
|
||||
num_draft_tokens: list[int],
|
||||
) -> None:
|
||||
start_idx = 0
|
||||
for i, bad_words_ids in bad_words_token_ids.items():
|
||||
for draft_idx in range(num_draft_tokens[i]):
|
||||
_apply_bad_words_single_batch(
|
||||
logits[start_idx + draft_idx],
|
||||
bad_words_ids,
|
||||
past_tokens_ids[start_idx + draft_idx],
|
||||
)
|
||||
start_idx += num_draft_tokens[i]
|
||||
remaining = len(bad_words_token_ids)
|
||||
for i, n in enumerate(num_draft_tokens):
|
||||
if (bad_words_ids := bad_words_token_ids.get(i)) is not None:
|
||||
for draft_idx in range(start_idx, start_idx + n):
|
||||
_apply_bad_words_single_batch(
|
||||
logits[draft_idx],
|
||||
bad_words_ids,
|
||||
past_tokens_ids[draft_idx],
|
||||
)
|
||||
remaining -= 1
|
||||
if not remaining:
|
||||
break
|
||||
start_idx += n
|
||||
|
||||
Reference in New Issue
Block a user