[Rust Frontend] Lower out-of-vocab validation to text layer (#45685)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -29,10 +29,12 @@ pub struct SamplingLimits {
|
||||
///
|
||||
/// `-1` means allowing requests up to the model vocabulary size.
|
||||
pub max_logprobs: i32,
|
||||
/// Model vocabulary size from the model config.
|
||||
|
||||
/// Model vocabulary size from the model config, used to bound
|
||||
/// `logit_bias` keys when available.
|
||||
pub model_vocab_size: Option<usize>,
|
||||
/// Tokenizer vocabulary size, used as a fallback when the model config does
|
||||
/// not expose a vocabulary size.
|
||||
/// Tokenizer vocabulary size, used to bound `allowed_token_ids` and
|
||||
/// token-ID prompts.
|
||||
pub tokenizer_vocab_size: usize,
|
||||
}
|
||||
|
||||
@@ -48,6 +50,16 @@ impl SamplingLimits {
|
||||
pub fn logprobs_vocab_size(&self) -> usize {
|
||||
self.model_vocab_size.unwrap_or(self.tokenizer_vocab_size)
|
||||
}
|
||||
|
||||
/// Return the vocabulary size used to validate generated stop token IDs.
|
||||
pub fn stop_token_vocab_size(&self) -> usize {
|
||||
self.model_vocab_size.unwrap_or(self.tokenizer_vocab_size)
|
||||
}
|
||||
|
||||
/// Return the union bound used to validate token-ID prompts.
|
||||
pub fn prompt_token_vocab_size(&self) -> usize {
|
||||
self.tokenizer_vocab_size.max(self.model_vocab_size.unwrap_or(0))
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal text-processing backend needed by `vllm-text`.
|
||||
|
||||
@@ -3,6 +3,7 @@ use vllm_engine_core_client::Error as EngineCoreError;
|
||||
use vllm_llm::Error as LlmError;
|
||||
|
||||
pub use crate::lower::logprobs::LogprobsError;
|
||||
pub use crate::lower::token_ids::OutOfVocabError;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
@@ -17,6 +18,8 @@ pub enum Error {
|
||||
PromptTooLong { max_model_len: u32, prompt_len: u32 },
|
||||
#[error(transparent)]
|
||||
Logprobs(#[from] LogprobsError),
|
||||
#[error(transparent)]
|
||||
OutOfVocab(#[from] OutOfVocabError),
|
||||
#[error("text request stream `{request_id}` closed before terminal output")]
|
||||
StreamClosedBeforeTerminalOutput { request_id: String },
|
||||
#[error(transparent)]
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
use std::mem::take;
|
||||
|
||||
pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend};
|
||||
pub use error::{Error, LogprobsError, Result};
|
||||
pub use error::{Error, LogprobsError, OutOfVocabError, Result};
|
||||
use futures::Stream;
|
||||
pub use lower::{
|
||||
PreparedTextRequest, lower_sampling_params, lower_text_request, resolve_max_tokens,
|
||||
|
||||
+141
-6
@@ -1,6 +1,7 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
pub(crate) mod logprobs;
|
||||
pub(crate) mod token_ids;
|
||||
|
||||
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
|
||||
use vllm_llm::GenerateRequest;
|
||||
@@ -10,6 +11,7 @@ use crate::backend::{SamplingHints, SamplingLimits};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{SamplingParams, TextRequest};
|
||||
use logprobs::validate_logprobs;
|
||||
use token_ids::{validate_prompt_token_ids, validate_vocab_range};
|
||||
|
||||
/// One text request after it has been lowered into the raw generate boundary.
|
||||
#[derive(Debug)]
|
||||
@@ -31,6 +33,8 @@ pub fn lower_text_request(
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> Result<PreparedTextRequest> {
|
||||
let prompt_len = prompt_token_ids.len() as u32;
|
||||
validate_prompt_token_ids(&prompt_token_ids, &sampling_limits)?;
|
||||
|
||||
let generate_request = GenerateRequest {
|
||||
request_id: request.request_id.clone(),
|
||||
prompt_token_ids,
|
||||
@@ -100,7 +104,6 @@ pub fn lower_sampling_params(
|
||||
vllm_xargs,
|
||||
} = sampling_params;
|
||||
|
||||
// Validate logprobs-related fields first with runtime sampling limits first.
|
||||
validate_logprobs(
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
@@ -139,7 +142,7 @@ pub fn lower_sampling_params(
|
||||
merge_unique_token_ids(&mut stop_token_ids, extra_eos_token_ids.iter().copied());
|
||||
}
|
||||
|
||||
Ok(EngineCoreSamplingParams {
|
||||
let params = EngineCoreSamplingParams {
|
||||
temperature,
|
||||
top_p,
|
||||
top_k,
|
||||
@@ -162,7 +165,9 @@ pub fn lower_sampling_params(
|
||||
logprob_token_ids,
|
||||
skip_reading_prefix_cache,
|
||||
extra_args: vllm_xargs,
|
||||
})
|
||||
};
|
||||
validate_vocab_range(¶ms, &sampling_limits)?;
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
/// Convert bad-word strings into token-ID sequences, following the Python vLLM
|
||||
@@ -243,14 +248,14 @@ fn merge_unique_token_ids(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use serial_test::file_serial;
|
||||
|
||||
use super::*;
|
||||
use crate::backend::hf::HfTextBackend;
|
||||
use crate::backend::{SamplingHints, TextBackend as _};
|
||||
use crate::error::LogprobsError;
|
||||
use crate::error::{LogprobsError, OutOfVocabError};
|
||||
use crate::request::{Prompt, TextRequest};
|
||||
|
||||
/// Stub tokenizer that returns empty token IDs — sufficient for tests that
|
||||
@@ -430,6 +435,57 @@ mod tests {
|
||||
.assert_debug_eq(¶ms);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_uses_union_vocab_for_prompt_token_ids() {
|
||||
lower_text_request(
|
||||
sample_request(),
|
||||
vec![1500],
|
||||
sample_sampling_hints(),
|
||||
SamplingLimits {
|
||||
model_vocab_size: Some(2000),
|
||||
tokenizer_vocab_size: 1000,
|
||||
..sample_sampling_limits()
|
||||
},
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.expect("model vocab extends prompt token range");
|
||||
|
||||
lower_text_request(
|
||||
sample_request(),
|
||||
vec![1500],
|
||||
sample_sampling_hints(),
|
||||
SamplingLimits {
|
||||
model_vocab_size: Some(1000),
|
||||
tokenizer_vocab_size: 2000,
|
||||
..sample_sampling_limits()
|
||||
},
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.expect("tokenizer vocab extends prompt token range");
|
||||
|
||||
let error = lower_text_request(
|
||||
sample_request(),
|
||||
vec![2000],
|
||||
sample_sampling_hints(),
|
||||
SamplingLimits {
|
||||
model_vocab_size: Some(1000),
|
||||
tokenizer_vocab_size: 2000,
|
||||
..sample_sampling_limits()
|
||||
},
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::OutOfVocab(OutOfVocabError {
|
||||
parameter: "prompt",
|
||||
token_ids,
|
||||
vocab_size: 2000,
|
||||
}) if token_ids == vec![2000]
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[file_serial(hf_qwen3)]
|
||||
async fn lower_text_request_uses_real_qwen_generation_defaults() {
|
||||
@@ -747,13 +803,92 @@ mod tests {
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::Logprobs(LogprobsError::InvalidTokenIds {
|
||||
Error::OutOfVocab(OutOfVocabError {
|
||||
parameter: "logprob_token_ids",
|
||||
token_ids,
|
||||
vocab_size: 1000,
|
||||
}) if token_ids == vec![1000]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
stop_token_ids: Some(vec![999, 1000]),
|
||||
..Default::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::OutOfVocab(OutOfVocabError {
|
||||
parameter: "stop_token_ids",
|
||||
token_ids,
|
||||
vocab_size: 1000,
|
||||
}) if token_ids == vec![1000]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_out_of_vocab_allowed_token_ids() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
allowed_token_ids: Some(vec![1999, 2000]),
|
||||
..Default::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::OutOfVocab(OutOfVocabError {
|
||||
parameter: "allowed_token_ids",
|
||||
token_ids,
|
||||
vocab_size: 2000,
|
||||
}) if token_ids == vec![2000]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_out_of_vocab_logit_bias() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
logit_bias: Some(HashMap::from([(1000, 1.0)])),
|
||||
..Default::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::OutOfVocab(OutOfVocabError {
|
||||
parameter: "logit_bias",
|
||||
token_ids,
|
||||
vocab_size: 1000,
|
||||
}) if token_ids == vec![1000]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_skips_logit_bias_range_when_model_vocab_is_unknown() {
|
||||
lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
logit_bias: Some(HashMap::from([(1_000_000, 1.0)])),
|
||||
..Default::default()
|
||||
},
|
||||
SamplingLimits {
|
||||
model_vocab_size: None,
|
||||
..sample_sampling_limits()
|
||||
},
|
||||
)
|
||||
.expect("logit_bias range check is skipped without model vocab size");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_uses_generation_defaults_when_user_omits_values() {
|
||||
let params = lower_sampling_params(
|
||||
|
||||
@@ -26,14 +26,6 @@ pub enum LogprobsError {
|
||||
requested: usize,
|
||||
max_allowed: usize,
|
||||
},
|
||||
#[error(
|
||||
"token_id(s) {token_ids:?} in logprob_token_ids contain out-of-vocab token ids. \
|
||||
Vocabulary size: {vocab_size}"
|
||||
)]
|
||||
InvalidTokenIds {
|
||||
token_ids: Vec<u32>,
|
||||
vocab_size: usize,
|
||||
},
|
||||
#[error(
|
||||
"when both logprobs and logprob_token_ids are set, logprobs must equal \
|
||||
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
|
||||
@@ -41,8 +33,7 @@ pub enum LogprobsError {
|
||||
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
|
||||
}
|
||||
|
||||
/// Validate logprobs-related sampling parameters, returning an error if any
|
||||
/// parameter is out of bounds or if the combination of parameters is invalid.
|
||||
/// Validate logprobs count sampling parameters.
|
||||
pub(super) fn validate_logprobs(
|
||||
logprobs: Option<i32>,
|
||||
prompt_logprobs: Option<i32>,
|
||||
@@ -55,7 +46,7 @@ pub(super) fn validate_logprobs(
|
||||
|
||||
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
|
||||
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
|
||||
validate_logprob_token_ids(logprobs, logprob_token_ids, vocab_size)
|
||||
validate_logprob_token_ids(logprobs, logprob_token_ids)
|
||||
}
|
||||
|
||||
fn validate_logprobs_count(
|
||||
@@ -80,10 +71,9 @@ fn validate_logprobs_count(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_logprob_token_ids(
|
||||
pub(super) fn validate_logprob_token_ids(
|
||||
logprobs: Option<i32>,
|
||||
logprob_token_ids: Option<&[u32]>,
|
||||
vocab_size: usize,
|
||||
) -> Result<(), LogprobsError> {
|
||||
let Some(logprob_token_ids) = logprob_token_ids else {
|
||||
return Ok(());
|
||||
@@ -97,18 +87,6 @@ fn validate_logprob_token_ids(
|
||||
});
|
||||
}
|
||||
|
||||
let invalid_token_ids: Vec<_> = logprob_token_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&token_id| token_id as usize >= vocab_size)
|
||||
.collect();
|
||||
if !invalid_token_ids.is_empty() {
|
||||
return Err(LogprobsError::InvalidTokenIds {
|
||||
token_ids: invalid_token_ids,
|
||||
vocab_size,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(logprobs) = logprobs
|
||||
&& logprobs != n as i32
|
||||
{
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
use std::result::Result;
|
||||
|
||||
use thiserror::Error;
|
||||
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
|
||||
|
||||
use crate::SamplingLimits;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
#[error(
|
||||
"token_id(s) {token_ids:?} in {parameter} contain out-of-vocab token ids. \
|
||||
Vocabulary size: {vocab_size}"
|
||||
)]
|
||||
pub struct OutOfVocabError {
|
||||
pub parameter: &'static str,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub vocab_size: usize,
|
||||
}
|
||||
|
||||
fn validate_param(
|
||||
parameter: &'static str,
|
||||
token_ids: impl IntoIterator<Item = u32>,
|
||||
vocab_size: usize,
|
||||
) -> Result<(), OutOfVocabError> {
|
||||
let invalid_token_ids: Vec<_> = token_ids
|
||||
.into_iter()
|
||||
.filter(|&token_id| token_id as usize >= vocab_size)
|
||||
.collect();
|
||||
if invalid_token_ids.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
Err(OutOfVocabError {
|
||||
parameter,
|
||||
token_ids: invalid_token_ids,
|
||||
vocab_size,
|
||||
})
|
||||
}
|
||||
|
||||
/// Validate that pre-tokenized prompt IDs are within the engine-visible prompt
|
||||
/// vocabulary range.
|
||||
pub(crate) fn validate_prompt_token_ids(
|
||||
prompt_token_ids: &[u32],
|
||||
limits: &SamplingLimits,
|
||||
) -> Result<(), OutOfVocabError> {
|
||||
validate_param(
|
||||
"prompt",
|
||||
prompt_token_ids.iter().copied(),
|
||||
limits.prompt_token_vocab_size(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Validate that token IDs in text sampling parameters are within their
|
||||
/// parameter-specific vocabulary ranges.
|
||||
pub(crate) fn validate_vocab_range(
|
||||
params: &EngineCoreSamplingParams,
|
||||
limits: &SamplingLimits,
|
||||
) -> Result<(), OutOfVocabError> {
|
||||
validate_param(
|
||||
"stop_token_ids",
|
||||
params.stop_token_ids.iter().copied(),
|
||||
limits.stop_token_vocab_size(),
|
||||
)?;
|
||||
|
||||
if let Some(token_ids) = params.allowed_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"allowed_token_ids",
|
||||
token_ids.iter().copied(),
|
||||
limits.tokenizer_vocab_size,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let (Some(logit_bias), Some(vocab_size)) =
|
||||
(params.logit_bias.as_ref(), limits.model_vocab_size)
|
||||
{
|
||||
validate_param("logit_bias", logit_bias.keys().copied(), vocab_size)?;
|
||||
}
|
||||
|
||||
if let Some(token_ids) = params.logprob_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"logprob_token_ids",
|
||||
token_ids.iter().copied(),
|
||||
limits.logprobs_vocab_size(),
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn validate_vocab_range_rejects_out_of_vocab_ids() {
|
||||
let error = validate_param("logprob_token_ids", [5_u32, 1000, 1001], 1000).unwrap_err();
|
||||
|
||||
assert_eq!(error.parameter, "logprob_token_ids");
|
||||
assert_eq!(error.token_ids, vec![1000, 1001]);
|
||||
assert_eq!(error.vocab_size, 1000);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user