[Bugfix][Model Runner V2] Preserve all allowed_token_ids in the logit bias kernel (#46245)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
@@ -189,6 +189,8 @@ def _bias_kernel(
|
||||
logits_ptr + token_idx * logits_stride + allowed_token_ids, mask=mask
|
||||
)
|
||||
|
||||
tl.debug_barrier() # save must read original logits before the -inf overwrite
|
||||
|
||||
# Set logits to -inf for all tokens.
|
||||
for i in range(0, vocab_size, LOGITS_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, LOGITS_BLOCK_SIZE)
|
||||
@@ -198,6 +200,8 @@ def _bias_kernel(
|
||||
mask=offset < vocab_size,
|
||||
)
|
||||
|
||||
tl.debug_barrier() # -inf overwrite must finish before restoring saved logits
|
||||
|
||||
# Restore logits for allowed token IDs.
|
||||
tl.store(
|
||||
logits_ptr + token_idx * logits_stride + allowed_token_ids,
|
||||
|
||||
Reference in New Issue
Block a user