[Bugfix] Reject negative token ids as out-of-vocabulary (#51795)
Signed-off-by: Junhao Shen <junshen@nvidia.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Vadim Gimpelson
parent
827a2af806
commit
b216db3ed0
@@ -483,6 +483,7 @@ class InputProcessor:
|
||||
|
||||
if prompt_ids and tokenizer is not None:
|
||||
max_input_id = max(prompt_ids, default=0)
|
||||
min_input_id = min(prompt_ids, default=0)
|
||||
|
||||
# NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while
|
||||
# self.model_config.get_vocab_size() is the model’s vocab size.
|
||||
@@ -495,6 +496,15 @@ class InputProcessor:
|
||||
# Here we take the max of the two to determine if a token id is
|
||||
# truly out-of-vocabulary.
|
||||
model_vocab_size = model_config.get_vocab_size()
|
||||
# A negative id is out of vocabulary just like an over-large one,
|
||||
# but is not caught by the upper-bound check below. Reject it here
|
||||
# so it is not used as an embedding index downstream. This
|
||||
# validation path is shared by generate, embedding and pooling
|
||||
# requests, so the check covers all three.
|
||||
if min_input_id < 0:
|
||||
raise VLLMValidationError(
|
||||
f"Token id {min_input_id} is out of vocabulary"
|
||||
)
|
||||
if max_input_id > max(tokenizer.max_token_id, model_vocab_size - 1):
|
||||
raise VLLMValidationError(
|
||||
f"Token id {max_input_id} is out of vocabulary"
|
||||
|
||||
Reference in New Issue
Block a user