[Bugfix] Reject empty bad-word tokenizations (#53433)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-31 12:07:17 -07:00
committed by GitHub
parent f9c7c6e090
commit a9dc631429
2 changed files with 45 additions and 0 deletions
+11
View File
@@ -707,6 +707,17 @@ class SamplingParams(
text=prompt, add_special_tokens=False
)
if not prompt_token_ids:
if not add_prefix_space:
raise VLLMValidationError(
"bad_words entries must tokenize to at least one token.",
parameter="bad_words",
value=self.bad_words,
)
# The unprefixed form is still enforceable when only the
# optional space-prefixed form tokenizes to nothing.
continue
# If no space at the beginning
# or if prefix space produces a new word token
if (not add_prefix_space) or (