diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index d75eb3b941..077dfcb980 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -288,6 +288,32 @@ mod tests { StubTokenizer } + struct FixedTokenizer { + token_ids: Vec, + } + + impl Tokenizer for FixedTokenizer { + fn encode( + &self, + _text: &str, + _add_special_tokens: bool, + ) -> vllm_tokenizer::Result> { + Ok(self.token_ids.clone()) + } + + fn decode( + &self, + _token_ids: &[u32], + _skip_special_tokens: bool, + ) -> vllm_tokenizer::Result { + Ok(String::new()) + } + + fn token_to_id(&self, _token: &str) -> Option { + None + } + } + fn sample_request() -> TextRequest { TextRequest { prompt: Prompt::TokenIds(vec![1, 2, 3]), @@ -827,6 +853,33 @@ mod tests { )); } + #[test] + fn lower_sampling_params_rejects_out_of_vocab_bad_words() { + let tokenizer = FixedTokenizer { + token_ids: vec![1999, 2000], + }; + let error = lower_sampling_params( + SamplingParams { + bad_words: Some(vec!["blocked".to_string()]), + ..Default::default() + }, + SamplingHints::default(), + sample_sampling_limits(), + 3, + &tokenizer, + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::OutOfVocab(OutOfVocabError { + parameter: "bad_words", + token_ids, + vocab_size: 2000, + }) if token_ids == vec![2000] + )); + } + #[test] fn lower_sampling_params_rejects_out_of_vocab_logit_bias() { let error = lower_sampling_params_with_limits( diff --git a/rust/src/text/src/lower/token_ids.rs b/rust/src/text/src/lower/token_ids.rs index d24b46d4cc..e434af92f1 100644 --- a/rust/src/text/src/lower/token_ids.rs +++ b/rust/src/text/src/lower/token_ids.rs @@ -85,6 +85,14 @@ pub(crate) fn validate_vocab_range( )?; } + if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() { + validate_param( + "bad_words", + bad_words_token_ids.iter().flatten().copied(), + limits.tokenizer_vocab_size, + )?; + } + Ok(()) }