[Rust Frontend] Support max_logprobs validation (#45674)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-16 10:57:56 +08:00
committed by GitHub
parent b00e76ff72
commit 3f65e21e32
15 changed files with 484 additions and 98 deletions
+8
View File
@@ -127,6 +127,11 @@ pub struct SharedRuntimeArgs {
/// `config.json`.
#[arg(long)]
pub max_model_len: Option<u32>,
/// Maximum number of log probabilities to return when `logprobs` is
/// specified in sampling parameters. `-1` means no cap.
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
#[serde(default)]
pub max_logprobs: Option<i32>,
/// TCP port for the gRPC Generate service. When not set, no gRPC server is
/// started.
#[arg(long)]
@@ -281,6 +286,7 @@ impl SharedRuntimeArgs {
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
api_server_options,
api_keys: self.api_key,
disable_log_stats: self.disable_log_stats,
@@ -324,6 +330,7 @@ impl SharedRuntimeArgs {
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
api_server_options,
api_keys: self.api_key,
disable_log_stats: self.disable_log_stats,
@@ -467,6 +474,7 @@ impl ServeArgs {
self.managed_engine.clone().into_config(
self.runtime.model.clone(),
self.runtime.max_model_len,
self.runtime.max_logprobs,
self.runtime.language_model_only,
self.runtime.disable_log_stats,
self.runtime.shutdown_timeout,
+32 -1
View File
@@ -38,6 +38,7 @@ fn serve_args_forward_python_flags_with_separator() {
max_model_len: Some(
512,
),
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
chat_template: None,
@@ -134,6 +135,29 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() {
assert_eq!(config.python_args, vec!["--disable-log-stats"]);
}
#[test]
fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--max-logprobs",
"-1",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(args.runtime.max_logprobs, Some(-1));
let frontend_config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
assert_eq!(frontend_config.max_logprobs, Some(-1));
let engine_config = args.to_managed_engine_config(5555);
assert_eq!(engine_config.python_args, vec!["--max-logprobs", "-1"]);
}
#[test]
fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
@@ -388,6 +412,7 @@ fn frontend_args_accept_json() {
renderer: Auto,
language_model_only: false,
max_model_len: None,
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
chat_template: None,
@@ -431,6 +456,7 @@ fn frontend_args_json_applies_defaults() {
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
assert_eq!(args.runtime.max_model_len, None);
assert_eq!(args.runtime.max_logprobs, None);
assert_eq!(args.runtime.shutdown_timeout, 0);
}
@@ -446,7 +472,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"shutdown_timeout":3}"#,
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"max_logprobs":-1,"shutdown_timeout":3}"#,
])
.unwrap();
@@ -465,6 +491,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
assert!(args.runtime.language_model_only);
assert_eq!(args.runtime.max_model_len, Some(8192));
assert_eq!(args.runtime.max_logprobs, Some(-1));
assert_eq!(args.runtime.shutdown_timeout, 3);
}
@@ -792,6 +819,7 @@ fn serve_args_accept_handshake_aliases() {
renderer: Auto,
language_model_only: false,
max_model_len: None,
max_logprobs: None,
grpc_port: None,
shutdown_timeout: 0,
chat_template: None,
@@ -917,6 +945,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -985,6 +1014,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -1068,6 +1098,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
max_logprobs: None,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
-8
View File
@@ -202,14 +202,6 @@ pub struct EngineUnsupportedArgs {
#[arg(long)]
pub tokenizer_revision: Option<Unsupported>,
/// Maximum number of log probabilities to return when `logprobs` is
/// specified in `SamplingParams`. The default value comes the default for
/// the OpenAI Chat Completions API. -1 means no cap, i.e. all
/// (output_length * vocab_size) logprobs are allowed to be returned and
/// it may cause OOM.
#[arg(long)]
pub max_logprobs: Option<Unsupported>,
/// Skip initialization of tokenizer and detokenizer. Expects valid
/// `prompt_token_ids` and `None` for prompt from the input. The generated
/// output will contain token ids.
+5
View File
@@ -71,6 +71,7 @@ impl ManagedEngineArgs {
self,
model: String,
max_model_len: Option<u32>,
max_logprobs: Option<i32>,
language_model_only: bool,
disable_log_stats: bool,
shutdown_timeout: u64,
@@ -82,6 +83,10 @@ impl ManagedEngineArgs {
python_args.push("--max-model-len".to_string());
python_args.push(max_model_len.to_string());
}
if let Some(max_logprobs) = max_logprobs {
python_args.push("--max-logprobs".to_string());
python_args.push(max_logprobs.to_string());
}
if language_model_only {
python_args.push("--language-model-only".to_string());
}
@@ -68,6 +68,7 @@ async fn main() -> Result<()> {
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
max_logprobs: None,
api_server_options: ApiServerOptions::default(),
api_keys: Vec::new(),
disable_log_stats: false,
+12 -1
View File
@@ -2,7 +2,7 @@ use std::collections::HashMap;
use std::fmt;
use std::time::Duration;
use anyhow::Result;
use anyhow::{Result, bail};
use educe::Educe;
use serde::Serialize;
use serde_json::Value;
@@ -77,6 +77,9 @@ pub struct Config {
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
/// How to serialize `message.content` for chat-template rendering.
pub chat_template_content_format: ChatTemplateContentFormatOption,
/// Optional maximum number of top log probabilities accepted by the
/// frontend. `None` delegates to the text layer default.
pub max_logprobs: Option<i32>,
/// HTTP/API-server behavior switches.
pub api_server_options: ApiServerOptions,
/// API keys accepted as bearer tokens for guarded routes.
@@ -98,6 +101,14 @@ impl Config {
/// startup.
pub fn validate(&self) -> Result<()> {
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
if let Some(max_logprobs) = self.max_logprobs
&& max_logprobs < -1
{
bail!(
"max_logprobs must be non-negative or -1, got {}",
max_logprobs
);
}
Ok(())
}
+34 -7
View File
@@ -74,12 +74,11 @@ impl IntoResponse for ApiError {
}
}
/// Classify a text-pipeline submit failure: tokenized-prompt validation
/// failures (the prompt is too long for the model, or empty after
/// tokenization) are the client's fault and map to HTTP 400, mirroring the
/// Python frontend. Everything else stays an internal 500.
/// Classify a text-pipeline submit failure: request validation failures are
/// the client's fault and map to HTTP 400, mirroring the Python frontend.
/// Everything else stays an internal 500.
pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError {
if is_prompt_validation_error(&error) {
if is_request_validation_error(&error) {
return invalid_request!("{error}");
}
server_error!("{}: {}", context, error.to_report_string())
@@ -90,18 +89,19 @@ pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiE
pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError {
match &error {
vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"),
vllm_chat::Error::Text(text_error) if is_prompt_validation_error(text_error) => {
vllm_chat::Error::Text(text_error) if is_request_validation_error(text_error) => {
invalid_request!("{error}")
}
_ => server_error!("{}: {}", context, error.to_report_string()),
}
}
fn is_prompt_validation_error(error: &vllm_text::Error) -> bool {
fn is_request_validation_error(error: &vllm_text::Error) -> bool {
matches!(
error,
vllm_text::Error::PromptTooLong { .. }
| vllm_text::Error::EmptyPromptTokenIds { .. }
| vllm_text::Error::Logprobs(_)
// An empty tokenized prompt detected later, at request prepare
// time, surfaces through the transparent Llm wrapper.
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
@@ -145,6 +145,33 @@ mod tests {
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
}
#[test]
fn logprobs_validation_maps_to_invalid_request() {
let error = vllm_text::Error::Logprobs(vllm_text::LogprobsError::TooManyCount {
parameter: "logprobs",
requested: 1000,
max_allowed: 20,
});
let api_error = text_submit_error("failed to submit completion request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
let response = api_error.to_error_response();
assert_eq!(response.error.error_type, "invalid_request_error");
assert!(response.error.message.contains("logprobs"));
}
#[test]
fn chat_wrapped_logprobs_validation_maps_to_invalid_request() {
let error = vllm_chat::Error::Text(vllm_text::Error::Logprobs(
vllm_text::LogprobsError::TooManyCount {
parameter: "prompt_logprobs",
requested: 1000,
max_allowed: 20,
},
));
let api_error = chat_submit_error("failed to submit chat request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
}
#[test]
fn other_submit_errors_stay_internal() {
let error = vllm_text::Error::Tokenizer("backend exploded".to_string());
+1 -1
View File
@@ -90,7 +90,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.context("failed to connect to engine core")?;
let llm = Llm::new(client).with_log_stats(!config.disable_log_stats);
let text = TextLlm::new(llm, text_backend);
let text = TextLlm::new(llm, text_backend).with_max_logprobs(config.max_logprobs);
let chat = ChatLlm::new(text, chat_backend)
.with_tool_call_parser(config.tool_call_parser.clone())
+8 -6
View File
@@ -245,10 +245,6 @@ impl ModelConfig {
pub(super) fn is_moe(&self) -> bool {
self.num_experts() > 0
}
pub(super) fn max_position_embeddings(&self) -> Option<u32> {
self.effective_text_config().max_position_embeddings
}
}
/// Load the tokenizer-side EOS metadata if a config file is present.
@@ -356,7 +352,10 @@ mod tests {
assert_eq!(config.num_experts(), 8);
assert_eq!(config.model_type(), Some("top_level"));
assert_eq!(config.max_position_embeddings(), Some(4096));
assert_eq!(
config.effective_text_config().max_position_embeddings,
Some(4096)
);
assert!(config.is_moe());
}
@@ -367,7 +366,10 @@ mod tests {
assert_eq!(config.num_experts(), 0);
assert!(!config.is_moe());
assert_eq!(config.max_position_embeddings(), Some(4096));
assert_eq!(
config.effective_text_config().max_position_embeddings,
Some(4096)
);
}
#[test]
-1
View File
@@ -118,7 +118,6 @@ impl TextBackend for HfTextBackend {
default_min_p: self.generation_config.min_p,
default_repetition_penalty: self.generation_config.repetition_penalty,
default_max_tokens: self.generation_config.max_new_tokens,
max_model_len: self.model_config.max_position_embeddings(),
})
}
}
+32 -5
View File
@@ -6,8 +6,8 @@ use vllm_tokenizer::DynTokenizer;
use crate::error::Result;
/// Tokenizer/model-derived hints used to enrich text-generation requests before
/// they are lowered into engine-core.
/// Tokenizer/model-derived defaults used to enrich text-generation requests
/// before they are lowered into engine-core.
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SamplingHints {
pub primary_eos_token_id: Option<u32>,
@@ -18,9 +18,36 @@ pub struct SamplingHints {
pub default_min_p: Option<f32>,
pub default_repetition_penalty: Option<f32>,
pub default_max_tokens: Option<u32>,
/// Model context window size (`max_position_embeddings` from
/// `config.json`).
pub max_model_len: Option<u32>,
}
/// Effective bounds used to validate and lower sampling requests.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SamplingLimits {
/// Runtime context window size reported by the engine startup handshake.
pub max_model_len: u32,
/// Maximum number of top log probabilities accepted by this frontend.
///
/// `-1` means allowing requests up to the model vocabulary size.
pub max_logprobs: i32,
/// Model vocabulary size from the model config.
pub model_vocab_size: Option<usize>,
/// Tokenizer vocabulary size, used as a fallback when the model config does
/// not expose a vocabulary size.
pub tokenizer_vocab_size: usize,
}
impl SamplingLimits {
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/b5adb027ad03c29b46181752ba3b1cb84eff1dd4/vllm/config/model.py#L216-L220>
pub const DEFAULT_MAX_LOGPROBS: i32 = 20;
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/b5adb027ad03c29b46181752ba3b1cb84eff1dd4/vllm/sampling_params.py#L30-L32>
pub const MAX_LOGPROB_TOKEN_IDS: usize = 128;
/// Return the vocabulary size used to expand `logprobs=-1`.
pub fn logprobs_vocab_size(&self) -> usize {
self.model_vocab_size.unwrap_or(self.tokenizer_vocab_size)
}
}
/// Minimal text-processing backend needed by `vllm-text`.
+4
View File
@@ -2,6 +2,8 @@ use thiserror::Error;
use vllm_engine_core_client::Error as EngineCoreError;
use vllm_llm::Error as LlmError;
pub use crate::lower::logprobs::LogprobsError;
#[derive(Debug, Error)]
pub enum Error {
#[error("tokenizer error: {0}")]
@@ -13,6 +15,8 @@ pub enum Error {
but the prompt contains {prompt_len} input tokens"
)]
PromptTooLong { max_model_len: u32, prompt_len: u32 },
#[error(transparent)]
Logprobs(#[from] LogprobsError),
#[error("text request stream `{request_id}` closed before terminal output")]
StreamClosedBeforeTerminalOutput { request_id: String },
#[error(transparent)]
+28 -16
View File
@@ -6,8 +6,8 @@
use std::mem::take;
pub use backend::{DynTextBackend, SamplingHints, TextBackend};
pub use error::{Error, Result};
pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend};
pub use error::{Error, LogprobsError, Result};
use futures::Stream;
pub use lower::{
PreparedTextRequest, lower_sampling_params, lower_text_request, resolve_max_tokens,
@@ -45,33 +45,33 @@ pub struct TextLlm {
/// Tokenizer/model metadata backend responsible for prompt encode/decode
/// and sampling hints.
backend: DynTextBackend,
/// Context window size reported by the engine startup handshake, with
/// optional override from config.
/// Runtime context window size reported by the engine startup handshake.
max_model_len: u32,
/// Maximum number of top log probabilities accepted by this text facade.
max_logprobs: i32,
}
impl TextLlm {
/// Create a new text-generation facade from a shared LLM client plus a text
/// backend.
pub fn new(llm: Llm, backend: DynTextBackend) -> Self {
// Prefer the engine-reported max_model_len because it reflects the
// post-profiling, auto-fitted KV cache limit rather than static
// frontend metadata.
// The engine-reported value reflects the post-profiling, auto-fitted
// KV cache limit used at runtime.
let max_model_len = llm.engine_core_client().max_model_len();
Self {
llm,
backend,
max_model_len,
max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS,
}
}
/// Override the maximum model context length explicitly.
///
/// This takes priority over both the engine-reported default and any
/// tokenizer/model metadata exposed by the backend.
pub fn with_max_model_len(mut self, max_model_len: u32) -> Self {
self.max_model_len = max_model_len;
/// Override the maximum accepted logprobs count.
pub fn with_max_logprobs(mut self, max_logprobs: Option<i32>) -> Self {
if let Some(max_logprobs) = max_logprobs {
self.max_logprobs = max_logprobs;
}
self
}
@@ -140,12 +140,24 @@ impl TextLlm {
Prompt::TokenIds(token_ids) => token_ids,
};
let mut sampling_hints = self.backend.sampling_hints()?;
sampling_hints.max_model_len = Some(self.max_model_len);
let sampling_hints = self.backend.sampling_hints()?;
let sampling_limits = SamplingLimits {
max_model_len: self.max_model_len,
max_logprobs: self.max_logprobs,
model_vocab_size: self.backend.model_vocab_size(),
tokenizer_vocab_size: self.backend.tokenizer_vocab_size(),
};
let PreparedTextRequest {
text_request,
generate_request,
} = lower_text_request(request, prompt_token_ids, sampling_hints, &*tokenizer)?;
} = lower_text_request(
request,
prompt_token_ids,
sampling_hints,
sampling_limits,
&*tokenizer,
)?;
let raw_stream = self.llm.generate(generate_request).await?;
Ok((text_request, raw_stream))
+185 -52
View File
@@ -1,12 +1,15 @@
use std::collections::BTreeSet;
pub(crate) mod logprobs;
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
use vllm_llm::GenerateRequest;
use vllm_tokenizer::Tokenizer;
use crate::backend::SamplingHints;
use crate::backend::{SamplingHints, SamplingLimits};
use crate::error::{Error, Result};
use crate::request::{SamplingParams, TextRequest};
use logprobs::validate_logprobs;
/// One text request after it has been lowered into the raw generate boundary.
#[derive(Debug)]
@@ -24,6 +27,7 @@ pub fn lower_text_request(
request: TextRequest,
prompt_token_ids: Vec<u32>,
sampling_hints: SamplingHints,
sampling_limits: SamplingLimits,
tokenizer: &dyn Tokenizer,
) -> Result<PreparedTextRequest> {
let prompt_len = prompt_token_ids.len() as u32;
@@ -34,6 +38,7 @@ pub fn lower_text_request(
sampling_params: lower_sampling_params(
request.sampling_params.clone(),
sampling_hints,
sampling_limits,
prompt_len,
tokenizer,
)?,
@@ -66,8 +71,8 @@ pub fn lower_sampling_params(
default_min_p,
default_repetition_penalty,
default_max_tokens,
max_model_len,
}: SamplingHints,
sampling_limits: SamplingLimits,
prompt_len: u32,
tokenizer: &dyn Tokenizer,
) -> Result<EngineCoreSamplingParams> {
@@ -95,6 +100,14 @@ pub fn lower_sampling_params(
vllm_xargs,
} = sampling_params;
// Validate logprobs-related fields first with runtime sampling limits first.
validate_logprobs(
logprobs,
prompt_logprobs,
logprob_token_ids.as_deref(),
sampling_limits,
)?;
// Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat
// path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450
// If neither the caller nor the model provides a value, fall back to 1.0 — the
@@ -105,7 +118,12 @@ pub fn lower_sampling_params(
let top_k = top_k.or(default_top_k).unwrap_or(0);
let min_p = min_p.or(default_min_p).unwrap_or(0.0);
let repetition_penalty = repetition_penalty.or(default_repetition_penalty).unwrap_or(1.0);
let max_tokens = resolve_max_tokens(max_tokens, default_max_tokens, max_model_len, prompt_len)?;
let max_tokens = resolve_max_tokens(
max_tokens,
default_max_tokens,
sampling_limits.max_model_len,
prompt_len,
)?;
let min_tokens = min_tokens.unwrap_or(0);
let frequency_penalty = frequency_penalty.unwrap_or(0.0);
let presence_penalty = presence_penalty.unwrap_or(0.0);
@@ -189,33 +207,25 @@ fn tokenize_bad_words(
/// Resolve the effective `max_tokens` for generation, mirroring vLLM Python's
/// `get_max_tokens()` in `vllm/entrypoints/utils.py`.
///
/// Takes the minimum of all available limits (user-specified, generation-config
/// default, and `max_model_len - prompt_len`). When nothing is known, falls
/// back to `u32::MAX` so the engine-core can apply its own context-window
/// limit.
/// Takes the minimum of all available limits: user-specified, generation-config
/// default, and `max_model_len - prompt_len`.
pub fn resolve_max_tokens(
user_max_tokens: Option<u32>,
default_max_tokens: Option<u32>,
max_model_len: Option<u32>,
max_model_len: u32,
prompt_len: u32,
) -> Result<u32> {
let model_max_tokens = match max_model_len {
Some(max_model_len) if prompt_len >= max_model_len => {
return Err(Error::PromptTooLong {
max_model_len,
prompt_len,
});
}
Some(max_model_len) => Some(max_model_len - prompt_len),
None => None,
let model_max_tokens = if prompt_len >= max_model_len {
return Err(Error::PromptTooLong {
max_model_len,
prompt_len,
});
} else {
max_model_len - prompt_len
};
let fallback_max_tokens = user_max_tokens.or(default_max_tokens);
Ok([fallback_max_tokens, model_max_tokens]
.into_iter()
.flatten()
.min()
.unwrap_or(u32::MAX /* TODO: a reasonable fallback? */))
let request_max_tokens = user_max_tokens.or(default_max_tokens);
Ok(request_max_tokens.map_or(model_max_tokens, |n| n.min(model_max_tokens)))
}
fn merge_unique_token_ids(
@@ -240,6 +250,7 @@ mod tests {
use super::*;
use crate::backend::hf::HfTextBackend;
use crate::backend::{SamplingHints, TextBackend as _};
use crate::error::LogprobsError;
use crate::request::{Prompt, TextRequest};
/// Stub tokenizer that returns empty token IDs — sufficient for tests that
@@ -290,16 +301,47 @@ mod tests {
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
max_model_len: None,
}
}
fn sample_sampling_limits() -> SamplingLimits {
SamplingLimits {
max_model_len: 1_000_000,
max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS,
model_vocab_size: Some(1000),
tokenizer_vocab_size: 2000,
}
}
fn lower_sampling_params_with_limits(
sampling_params: SamplingParams,
sampling_limits: SamplingLimits,
) -> Result<EngineCoreSamplingParams> {
lower_sampling_params(
sampling_params,
SamplingHints {
primary_eos_token_id: None,
extra_eos_token_ids: BTreeSet::new(),
default_temperature: None,
default_top_p: None,
default_top_k: None,
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
},
sampling_limits,
3,
&stub_tokenizer(),
)
}
#[test]
fn lower_text_request_applies_python_style_eos_hints() {
let prepared = lower_text_request(
sample_request(),
vec![1, 2, 3],
sample_sampling_hints(),
sample_sampling_limits(),
&stub_tokenizer(),
)
.unwrap();
@@ -311,7 +353,7 @@ mod tests {
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 4294967295,
max_tokens: 999997,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
@@ -350,6 +392,7 @@ mod tests {
request,
vec![1, 2, 3],
sample_sampling_hints(),
sample_sampling_limits(),
&stub_tokenizer(),
)
.unwrap();
@@ -361,7 +404,7 @@ mod tests {
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 4294967295,
max_tokens: 999997,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
@@ -415,16 +458,23 @@ mod tests {
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
max_model_len: Some(
40960,
),
}
"#]]
.assert_debug_eq(&hints);
let prepared =
lower_text_request(sample_request(), vec![1, 2, 3], hints, &stub_tokenizer())
.expect("lower request");
let prepared = lower_text_request(
sample_request(),
vec![1, 2, 3],
hints,
SamplingLimits {
max_model_len: 40960,
max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS,
model_vocab_size: backend.model_vocab_size(),
tokenizer_vocab_size: backend.tokenizer_vocab_size(),
},
&stub_tokenizer(),
)
.expect("lower request");
let params = prepared.generate_request.sampling_params;
expect_test::expect![[r#"
@@ -481,8 +531,8 @@ mod tests {
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
max_model_len: None,
},
sample_sampling_limits(),
3,
&stub_tokenizer(),
)
@@ -494,7 +544,7 @@ mod tests {
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 4294967295,
max_tokens: 999997,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
@@ -550,8 +600,8 @@ mod tests {
default_min_p: Some(0.1),
default_repetition_penalty: Some(1.2),
default_max_tokens: Some(128),
max_model_len: None,
},
sample_sampling_limits(),
3,
&stub_tokenizer(),
)
@@ -605,7 +655,10 @@ mod tests {
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
max_model_len: None,
},
SamplingLimits {
max_logprobs: -1,
..sample_sampling_limits()
},
3,
&stub_tokenizer(),
@@ -616,6 +669,91 @@ mod tests {
assert_eq!(params.prompt_logprobs, Some(-1));
}
#[test]
fn lower_sampling_params_rejects_full_vocab_logprobs_over_default_cap() {
let error = lower_sampling_params_with_limits(
SamplingParams {
logprobs: Some(-1),
..Default::default()
},
sample_sampling_limits(),
)
.unwrap_err();
assert!(matches!(
error,
Error::Logprobs(LogprobsError::TooManyCount {
parameter: "logprobs",
requested: 1000,
max_allowed: 20,
})
));
}
#[test]
fn lower_sampling_params_expands_full_vocab_logprobs_from_model_vocab() {
let params = lower_sampling_params_with_limits(
SamplingParams {
logprobs: Some(-1),
..Default::default()
},
SamplingLimits {
max_logprobs: 1500,
..sample_sampling_limits()
},
)
.unwrap();
assert_eq!(params.logprobs, Some(-1));
}
#[test]
fn lower_sampling_params_uses_tokenizer_vocab_when_model_vocab_is_unknown() {
let error = lower_sampling_params_with_limits(
SamplingParams {
logprobs: Some(-1),
..Default::default()
},
SamplingLimits {
max_logprobs: 1500,
model_vocab_size: None,
tokenizer_vocab_size: 2000,
..sample_sampling_limits()
},
)
.unwrap_err();
assert!(matches!(
error,
Error::Logprobs(LogprobsError::TooManyCount {
parameter: "logprobs",
requested: 2000,
max_allowed: 1500,
})
));
}
#[test]
fn lower_sampling_params_rejects_invalid_logprob_token_ids() {
let error = lower_sampling_params_with_limits(
SamplingParams {
logprobs: Some(1),
logprob_token_ids: Some(vec![1000]),
..Default::default()
},
sample_sampling_limits(),
)
.unwrap_err();
assert!(matches!(
error,
Error::Logprobs(LogprobsError::InvalidTokenIds {
token_ids,
vocab_size: 1000,
}) if token_ids == vec![1000]
));
}
#[test]
fn lower_sampling_params_uses_generation_defaults_when_user_omits_values() {
let params = lower_sampling_params(
@@ -629,8 +767,8 @@ mod tests {
default_min_p: Some(0.1),
default_repetition_penalty: Some(1.2),
default_max_tokens: Some(128),
max_model_len: None,
},
sample_sampling_limits(),
3,
&stub_tokenizer(),
)
@@ -667,7 +805,7 @@ mod tests {
#[test]
fn resolve_max_tokens_caps_by_model_len() {
let result = resolve_max_tokens(Some(150), None, Some(200), 100);
let result = resolve_max_tokens(Some(150), None, 200, 100);
assert_eq!(result.unwrap(), 100);
}
@@ -680,6 +818,7 @@ mod tests {
request,
vec![1, 2, 3],
sample_sampling_hints(),
sample_sampling_limits(),
&stub_tokenizer(),
)
.unwrap();
@@ -690,37 +829,31 @@ mod tests {
#[test]
fn resolve_max_tokens_user_smaller_than_model_limit() {
let result = resolve_max_tokens(Some(50), None, Some(200), 100);
let result = resolve_max_tokens(Some(50), None, 200, 100);
assert_eq!(result.unwrap(), 50);
}
#[test]
fn resolve_max_tokens_uses_default_when_user_omits() {
let result = resolve_max_tokens(None, Some(64), Some(200), 100);
let result = resolve_max_tokens(None, Some(64), 200, 100);
assert_eq!(result.unwrap(), 64);
}
#[test]
fn resolve_max_tokens_default_capped_by_model_len() {
let result = resolve_max_tokens(None, Some(256), Some(200), 100);
let result = resolve_max_tokens(None, Some(256), 200, 100);
assert_eq!(result.unwrap(), 100);
}
#[test]
fn resolve_max_tokens_no_model_len_falls_back() {
let result = resolve_max_tokens(Some(9999), None, None, 100);
assert_eq!(result.unwrap(), 9999);
}
#[test]
fn resolve_max_tokens_no_limits_known_falls_back_to_u32_max() {
let result = resolve_max_tokens(None, None, None, 100);
assert_eq!(result.unwrap(), u32::MAX);
fn resolve_max_tokens_uses_model_limit_when_user_omits() {
let result = resolve_max_tokens(None, None, 200, 100);
assert_eq!(result.unwrap(), 100);
}
#[test]
fn resolve_max_tokens_prompt_too_long() {
let result = resolve_max_tokens(Some(10), None, Some(100), 100);
let result = resolve_max_tokens(Some(10), None, 100, 100);
assert!(matches!(
result,
Err(Error::PromptTooLong {
@@ -732,7 +865,7 @@ mod tests {
#[test]
fn resolve_max_tokens_prompt_exceeds_model_len() {
let result = resolve_max_tokens(Some(10), None, Some(100), 200);
let result = resolve_max_tokens(Some(10), None, 100, 200);
assert!(matches!(
result,
Err(Error::PromptTooLong {
+134
View File
@@ -0,0 +1,134 @@
//! Python-compatible validation for logprobs sampling params.
//!
//! `-1` is expanded only for bounds checks. The original request values are
//! passed through to engine-core.
use crate::backend::SamplingLimits;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum LogprobsError {
#[error("{parameter} must be non-negative or -1, got {value}")]
InvalidCount { parameter: &'static str, value: i32 },
#[error(
"requested {parameter} of {requested}, which is greater than max allowed: {max_allowed}"
)]
TooManyCount {
parameter: &'static str,
requested: usize,
max_allowed: usize,
},
#[error(
"requested logprob_token_ids of length {requested}, \
which is greater than max allowed: {max_allowed}"
)]
TooManyTokenIds {
requested: usize,
max_allowed: usize,
},
#[error(
"token_id(s) {token_ids:?} in logprob_token_ids contain out-of-vocab token ids. \
Vocabulary size: {vocab_size}"
)]
InvalidTokenIds {
token_ids: Vec<u32>,
vocab_size: usize,
},
#[error(
"when both logprobs and logprob_token_ids are set, logprobs must equal \
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
)]
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
}
/// Validate logprobs-related sampling parameters, returning an error if any
/// parameter is out of bounds or if the combination of parameters is invalid.
pub(super) fn validate_logprobs(
logprobs: Option<i32>,
prompt_logprobs: Option<i32>,
logprob_token_ids: Option<&[u32]>,
sampling_limits: SamplingLimits,
) -> Result<(), LogprobsError> {
let vocab_size = sampling_limits.logprobs_vocab_size();
let max_logprobs =
normalize_logprobs_count(sampling_limits.max_logprobs, vocab_size, "max_logprobs")?;
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
validate_logprob_token_ids(logprobs, logprob_token_ids, vocab_size)
}
fn validate_logprobs_count(
requested: Option<i32>,
max_logprobs: usize,
vocab_size: usize,
parameter: &'static str,
) -> Result<(), LogprobsError> {
let Some(requested) = requested else {
return Ok(());
};
let requested = normalize_logprobs_count(requested, vocab_size, parameter)?;
if requested > max_logprobs {
return Err(LogprobsError::TooManyCount {
parameter,
requested,
max_allowed: max_logprobs,
});
}
Ok(())
}
fn validate_logprob_token_ids(
logprobs: Option<i32>,
logprob_token_ids: Option<&[u32]>,
vocab_size: usize,
) -> Result<(), LogprobsError> {
let Some(logprob_token_ids) = logprob_token_ids else {
return Ok(());
};
let n = logprob_token_ids.len();
if n > SamplingLimits::MAX_LOGPROB_TOKEN_IDS {
return Err(LogprobsError::TooManyTokenIds {
requested: n,
max_allowed: SamplingLimits::MAX_LOGPROB_TOKEN_IDS,
});
}
let invalid_token_ids: Vec<_> = logprob_token_ids
.iter()
.copied()
.filter(|&token_id| token_id as usize >= vocab_size)
.collect();
if !invalid_token_ids.is_empty() {
return Err(LogprobsError::InvalidTokenIds {
token_ids: invalid_token_ids,
vocab_size,
});
}
if let Some(logprobs) = logprobs
&& logprobs != n as i32
{
return Err(LogprobsError::TokenIdsMismatch {
logprobs,
num_token_ids: n,
});
}
Ok(())
}
fn normalize_logprobs_count(
value: i32,
vocab_size: usize,
parameter: &'static str,
) -> Result<usize, LogprobsError> {
match value {
-1 => Ok(vocab_size),
value if value < 0 => Err(LogprobsError::InvalidCount { parameter, value }),
value => Ok(value as usize),
}
}