[Rust Frontend] Validate bad_words limits during request lowering (#61933)
Signed-off-by: Alec Flowers <aflowers@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1338,6 +1338,47 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn lower_sampling_params_enforces_bad_words_limits() {
|
||||||
|
let tokenizer = TestTokenizer::new()
|
||||||
|
.with_regular_token("blocked", 300)
|
||||||
|
.with_regular_token(" blocked", 301);
|
||||||
|
let lower = |bad_words: Option<Vec<String>>, bad_words_token_ids: Vec<Vec<u32>>| {
|
||||||
|
lower_sampling_params(
|
||||||
|
SamplingParams {
|
||||||
|
bad_words,
|
||||||
|
bad_words_token_ids: Some(bad_words_token_ids),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
SamplingHints::default(),
|
||||||
|
sample_sampling_limits(),
|
||||||
|
3,
|
||||||
|
&tokenizer,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
lower(None, vec![vec![7; 8]; 128]).unwrap();
|
||||||
|
for (bad_words, raw, expected) in [
|
||||||
|
(
|
||||||
|
None,
|
||||||
|
vec![vec![7]; 129],
|
||||||
|
"Too many bad words: 129. The max number is 128.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
Some(vec!["blocked".to_string()]),
|
||||||
|
vec![vec![7]; 127],
|
||||||
|
"Too many bad words: 129. The max number is 128.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
None,
|
||||||
|
vec![vec![7; 1025]],
|
||||||
|
"Too many total bad word tokens: 1025. The max is 1024.",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
assert_eq!(lower(bad_words, raw).unwrap_err().to_string(), expected);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn lower_sampling_params_rejects_out_of_vocab_logit_bias() {
|
fn lower_sampling_params_rejects_out_of_vocab_logit_bias() {
|
||||||
let error = lower_sampling_params_with_limits(
|
let error = lower_sampling_params_with_limits(
|
||||||
|
|||||||
@@ -8,12 +8,21 @@ use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
|
|||||||
|
|
||||||
use crate::SamplingLimits;
|
use crate::SamplingLimits;
|
||||||
|
|
||||||
|
const MAX_NUM_BAD_WORDS_ENV: &str = "VLLM_MAX_NUM_BAD_WORDS";
|
||||||
|
const DEFAULT_MAX_NUM_BAD_WORDS: usize = 128;
|
||||||
|
const MAX_BAD_WORDS_TOTAL_TOKENS_ENV: &str = "VLLM_MAX_BAD_WORDS_TOTAL_TOKENS";
|
||||||
|
const DEFAULT_MAX_BAD_WORDS_TOTAL_TOKENS: usize = 1024;
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum TokenIdsError {
|
pub enum TokenIdsError {
|
||||||
#[error("allowed_token_ids should not be empty")]
|
#[error("allowed_token_ids should not be empty")]
|
||||||
EmptyAllowedTokenIds,
|
EmptyAllowedTokenIds,
|
||||||
#[error("bad_words_token_ids must not contain empty sequences")]
|
#[error("bad_words_token_ids must not contain empty sequences")]
|
||||||
EmptyBadWordSequence,
|
EmptyBadWordSequence,
|
||||||
|
#[error("Too many bad words: {count}. The max number is {max}.")]
|
||||||
|
TooManyBadWords { count: usize, max: usize },
|
||||||
|
#[error("Too many total bad word tokens: {count}. The max is {max}.")]
|
||||||
|
TooManyBadWordTokens { count: usize, max: usize },
|
||||||
#[error(
|
#[error(
|
||||||
"token_id(s) {token_ids:?} in {parameter} are out of vocabulary. \
|
"token_id(s) {token_ids:?} in {parameter} are out of vocabulary. \
|
||||||
Vocabulary size: {vocab_size}"
|
Vocabulary size: {vocab_size}"
|
||||||
@@ -45,6 +54,31 @@ fn validate_param(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn env_limit(name: &str, default: usize) -> usize {
|
||||||
|
std::env::var(name).ok().and_then(|value| value.parse().ok()).unwrap_or(default)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Mirrors Model Runner V2's per-request bad-words buffer limits:
|
||||||
|
/// <https://github.com/vllm-project/vllm/blob/a956a8126e6bd184cb0a6b8f7318fe4717675c2f/vllm/v1/worker/gpu/sample/bad_words.py#L40-L66>
|
||||||
|
fn validate_bad_words_limits(bad_words_token_ids: &[Vec<u32>]) -> Result<(), TokenIdsError> {
|
||||||
|
let count = bad_words_token_ids.len();
|
||||||
|
let max = env_limit(MAX_NUM_BAD_WORDS_ENV, DEFAULT_MAX_NUM_BAD_WORDS);
|
||||||
|
if count > max {
|
||||||
|
return Err(TokenIdsError::TooManyBadWords { count, max });
|
||||||
|
}
|
||||||
|
|
||||||
|
let count = bad_words_token_ids.iter().map(Vec::len).sum();
|
||||||
|
let max = env_limit(
|
||||||
|
MAX_BAD_WORDS_TOTAL_TOKENS_ENV,
|
||||||
|
DEFAULT_MAX_BAD_WORDS_TOTAL_TOKENS,
|
||||||
|
);
|
||||||
|
if count > max {
|
||||||
|
return Err(TokenIdsError::TooManyBadWordTokens { count, max });
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Validate that pre-tokenized prompt IDs are within the engine-visible prompt
|
/// Validate that pre-tokenized prompt IDs are within the engine-visible prompt
|
||||||
/// vocabulary range.
|
/// vocabulary range.
|
||||||
pub(crate) fn validate_prompt_token_ids(
|
pub(crate) fn validate_prompt_token_ids(
|
||||||
@@ -101,6 +135,7 @@ pub(crate) fn validate_vocab_range(
|
|||||||
if bad_words_token_ids.iter().any(Vec::is_empty) {
|
if bad_words_token_ids.iter().any(Vec::is_empty) {
|
||||||
return Err(TokenIdsError::EmptyBadWordSequence);
|
return Err(TokenIdsError::EmptyBadWordSequence);
|
||||||
}
|
}
|
||||||
|
validate_bad_words_limits(bad_words_token_ids)?;
|
||||||
validate_param(
|
validate_param(
|
||||||
"bad_words",
|
"bad_words",
|
||||||
bad_words_token_ids.iter().flatten().copied(),
|
bad_words_token_ids.iter().flatten().copied(),
|
||||||
|
|||||||
Reference in New Issue
Block a user