From 3f65e21e3200038e1f4524144b739ae207c8560d Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 16 Jun 2026 10:57:56 +0800 Subject: [PATCH] [Rust Frontend] Support `max_logprobs` validation (#45674) Signed-off-by: Bugen Zhao --- rust/src/cmd/src/cli.rs | 8 + rust/src/cmd/src/cli/tests.rs | 33 ++- rust/src/cmd/src/cli/unsupported.rs | 8 - rust/src/managed-engine/src/cli.rs | 5 + .../examples/external_engine_openai_qwen.rs | 1 + rust/src/server/src/config.rs | 13 +- rust/src/server/src/error.rs | 41 ++- rust/src/server/src/lib.rs | 2 +- rust/src/text/src/backend/hf/config.rs | 14 +- rust/src/text/src/backend/hf/mod.rs | 1 - rust/src/text/src/backend/mod.rs | 37 ++- rust/src/text/src/error.rs | 4 + rust/src/text/src/lib.rs | 44 ++-- rust/src/text/src/lower.rs | 237 ++++++++++++++---- rust/src/text/src/lower/logprobs.rs | 134 ++++++++++ 15 files changed, 484 insertions(+), 98 deletions(-) create mode 100644 rust/src/text/src/lower/logprobs.rs diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index b49d100da6..47e8ee5a93 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -127,6 +127,11 @@ pub struct SharedRuntimeArgs { /// `config.json`. #[arg(long)] pub max_model_len: Option, + /// Maximum number of log probabilities to return when `logprobs` is + /// specified in sampling parameters. `-1` means no cap. + #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)] + #[serde(default)] + pub max_logprobs: Option, /// TCP port for the gRPC Generate service. When not set, no gRPC server is /// started. #[arg(long)] @@ -281,6 +286,7 @@ impl SharedRuntimeArgs { chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs, chat_template_content_format: self.chat_template_content_format, + max_logprobs: self.max_logprobs, api_server_options, api_keys: self.api_key, disable_log_stats: self.disable_log_stats, @@ -324,6 +330,7 @@ impl SharedRuntimeArgs { chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs, chat_template_content_format: self.chat_template_content_format, + max_logprobs: self.max_logprobs, api_server_options, api_keys: self.api_key, disable_log_stats: self.disable_log_stats, @@ -467,6 +474,7 @@ impl ServeArgs { self.managed_engine.clone().into_config( self.runtime.model.clone(), self.runtime.max_model_len, + self.runtime.max_logprobs, self.runtime.language_model_only, self.runtime.disable_log_stats, self.runtime.shutdown_timeout, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index c6bd7c2b12..8e793075c7 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -38,6 +38,7 @@ fn serve_args_forward_python_flags_with_separator() { max_model_len: Some( 512, ), + max_logprobs: None, grpc_port: None, shutdown_timeout: 0, chat_template: None, @@ -134,6 +135,29 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() { assert_eq!(config.python_args, vec!["--disable-log-stats"]); } +#[test] +fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--max-logprobs", + "-1", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + assert_eq!(args.runtime.max_logprobs, Some(-1)); + + let frontend_config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string()); + assert_eq!(frontend_config.max_logprobs, Some(-1)); + + let engine_config = args.to_managed_engine_config(5555); + assert_eq!(engine_config.python_args, vec!["--max-logprobs", "-1"]); +} + #[test] fn serve_args_auto_forward_python_multi_char_alias_without_separator() { let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap(); @@ -388,6 +412,7 @@ fn frontend_args_accept_json() { renderer: Auto, language_model_only: false, max_model_len: None, + max_logprobs: None, grpc_port: None, shutdown_timeout: 0, chat_template: None, @@ -431,6 +456,7 @@ fn frontend_args_json_applies_defaults() { assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto); assert_eq!(args.runtime.renderer, RendererSelection::Auto); assert_eq!(args.runtime.max_model_len, None); + assert_eq!(args.runtime.max_logprobs, None); assert_eq!(args.runtime.shutdown_timeout, 0); } @@ -446,7 +472,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() { "--output-address", "ipc:///tmp/output.sock", "--args-json", - r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"shutdown_timeout":3}"#, + r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"max_logprobs":-1,"shutdown_timeout":3}"#, ]) .unwrap(); @@ -465,6 +491,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() { assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32); assert!(args.runtime.language_model_only); assert_eq!(args.runtime.max_model_len, Some(8192)); + assert_eq!(args.runtime.max_logprobs, Some(-1)); assert_eq!(args.runtime.shutdown_timeout, 3); } @@ -792,6 +819,7 @@ fn serve_args_accept_handshake_aliases() { renderer: Auto, language_model_only: false, max_model_len: None, + max_logprobs: None, grpc_port: None, shutdown_timeout: 0, chat_template: None, @@ -917,6 +945,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { chat_template: None, default_chat_template_kwargs: None, chat_template_content_format: Auto, + max_logprobs: None, api_server_options: ApiServerOptions { enable_log_requests: false, enable_prompt_tokens_details: false, @@ -985,6 +1014,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { chat_template: None, default_chat_template_kwargs: None, chat_template_content_format: Auto, + max_logprobs: None, api_server_options: ApiServerOptions { enable_log_requests: false, enable_prompt_tokens_details: false, @@ -1068,6 +1098,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present chat_template: None, default_chat_template_kwargs: None, chat_template_content_format: Auto, + max_logprobs: None, api_server_options: ApiServerOptions { enable_log_requests: false, enable_prompt_tokens_details: false, diff --git a/rust/src/cmd/src/cli/unsupported.rs b/rust/src/cmd/src/cli/unsupported.rs index e9dd5285e5..8fe8208a2b 100644 --- a/rust/src/cmd/src/cli/unsupported.rs +++ b/rust/src/cmd/src/cli/unsupported.rs @@ -202,14 +202,6 @@ pub struct EngineUnsupportedArgs { #[arg(long)] pub tokenizer_revision: Option, - /// Maximum number of log probabilities to return when `logprobs` is - /// specified in `SamplingParams`. The default value comes the default for - /// the OpenAI Chat Completions API. -1 means no cap, i.e. all - /// (output_length * vocab_size) logprobs are allowed to be returned and - /// it may cause OOM. - #[arg(long)] - pub max_logprobs: Option, - /// Skip initialization of tokenizer and detokenizer. Expects valid /// `prompt_token_ids` and `None` for prompt from the input. The generated /// output will contain token ids. diff --git a/rust/src/managed-engine/src/cli.rs b/rust/src/managed-engine/src/cli.rs index b6619b7a49..bbd8e70f90 100644 --- a/rust/src/managed-engine/src/cli.rs +++ b/rust/src/managed-engine/src/cli.rs @@ -71,6 +71,7 @@ impl ManagedEngineArgs { self, model: String, max_model_len: Option, + max_logprobs: Option, language_model_only: bool, disable_log_stats: bool, shutdown_timeout: u64, @@ -82,6 +83,10 @@ impl ManagedEngineArgs { python_args.push("--max-model-len".to_string()); python_args.push(max_model_len.to_string()); } + if let Some(max_logprobs) = max_logprobs { + python_args.push("--max-logprobs".to_string()); + python_args.push(max_logprobs.to_string()); + } if language_model_only { python_args.push("--language-model-only".to_string()); } diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 510149deea..c8d609f19b 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -68,6 +68,7 @@ async fn main() -> Result<()> { chat_template: None, default_chat_template_kwargs: None, chat_template_content_format: ChatTemplateContentFormatOption::Auto, + max_logprobs: None, api_server_options: ApiServerOptions::default(), api_keys: Vec::new(), disable_log_stats: false, diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index aa65dc03c2..e0d8a44300 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::fmt; use std::time::Duration; -use anyhow::Result; +use anyhow::{Result, bail}; use educe::Educe; use serde::Serialize; use serde_json::Value; @@ -77,6 +77,9 @@ pub struct Config { pub default_chat_template_kwargs: Option>, /// How to serialize `message.content` for chat-template rendering. pub chat_template_content_format: ChatTemplateContentFormatOption, + /// Optional maximum number of top log probabilities accepted by the + /// frontend. `None` delegates to the text layer default. + pub max_logprobs: Option, /// HTTP/API-server behavior switches. pub api_server_options: ApiServerOptions, /// API keys accepted as bearer tokens for guarded routes. @@ -98,6 +101,14 @@ impl Config { /// startup. pub fn validate(&self) -> Result<()> { vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?; + if let Some(max_logprobs) = self.max_logprobs + && max_logprobs < -1 + { + bail!( + "max_logprobs must be non-negative or -1, got {}", + max_logprobs + ); + } Ok(()) } diff --git a/rust/src/server/src/error.rs b/rust/src/server/src/error.rs index ce716bb65f..2096f4876d 100644 --- a/rust/src/server/src/error.rs +++ b/rust/src/server/src/error.rs @@ -74,12 +74,11 @@ impl IntoResponse for ApiError { } } -/// Classify a text-pipeline submit failure: tokenized-prompt validation -/// failures (the prompt is too long for the model, or empty after -/// tokenization) are the client's fault and map to HTTP 400, mirroring the -/// Python frontend. Everything else stays an internal 500. +/// Classify a text-pipeline submit failure: request validation failures are +/// the client's fault and map to HTTP 400, mirroring the Python frontend. +/// Everything else stays an internal 500. pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError { - if is_prompt_validation_error(&error) { + if is_request_validation_error(&error) { return invalid_request!("{error}"); } server_error!("{}: {}", context, error.to_report_string()) @@ -90,18 +89,19 @@ pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiE pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError { match &error { vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"), - vllm_chat::Error::Text(text_error) if is_prompt_validation_error(text_error) => { + vllm_chat::Error::Text(text_error) if is_request_validation_error(text_error) => { invalid_request!("{error}") } _ => server_error!("{}: {}", context, error.to_report_string()), } } -fn is_prompt_validation_error(error: &vllm_text::Error) -> bool { +fn is_request_validation_error(error: &vllm_text::Error) -> bool { matches!( error, vllm_text::Error::PromptTooLong { .. } | vllm_text::Error::EmptyPromptTokenIds { .. } + | vllm_text::Error::Logprobs(_) // An empty tokenized prompt detected later, at request prepare // time, surfaces through the transparent Llm wrapper. | vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. }) @@ -145,6 +145,33 @@ mod tests { assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); } + #[test] + fn logprobs_validation_maps_to_invalid_request() { + let error = vllm_text::Error::Logprobs(vllm_text::LogprobsError::TooManyCount { + parameter: "logprobs", + requested: 1000, + max_allowed: 20, + }); + let api_error = text_submit_error("failed to submit completion request", error); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + let response = api_error.to_error_response(); + assert_eq!(response.error.error_type, "invalid_request_error"); + assert!(response.error.message.contains("logprobs")); + } + + #[test] + fn chat_wrapped_logprobs_validation_maps_to_invalid_request() { + let error = vllm_chat::Error::Text(vllm_text::Error::Logprobs( + vllm_text::LogprobsError::TooManyCount { + parameter: "prompt_logprobs", + requested: 1000, + max_allowed: 20, + }, + )); + let api_error = chat_submit_error("failed to submit chat request", error); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + } + #[test] fn other_submit_errors_stay_internal() { let error = vllm_text::Error::Tokenizer("backend exploded".to_string()); diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 8cbb3e4d9f..ddf270e6c1 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -90,7 +90,7 @@ async fn build_state(config: &Config) -> Result> { .context("failed to connect to engine core")?; let llm = Llm::new(client).with_log_stats(!config.disable_log_stats); - let text = TextLlm::new(llm, text_backend); + let text = TextLlm::new(llm, text_backend).with_max_logprobs(config.max_logprobs); let chat = ChatLlm::new(text, chat_backend) .with_tool_call_parser(config.tool_call_parser.clone()) diff --git a/rust/src/text/src/backend/hf/config.rs b/rust/src/text/src/backend/hf/config.rs index 1efb31618d..65055722be 100644 --- a/rust/src/text/src/backend/hf/config.rs +++ b/rust/src/text/src/backend/hf/config.rs @@ -245,10 +245,6 @@ impl ModelConfig { pub(super) fn is_moe(&self) -> bool { self.num_experts() > 0 } - - pub(super) fn max_position_embeddings(&self) -> Option { - self.effective_text_config().max_position_embeddings - } } /// Load the tokenizer-side EOS metadata if a config file is present. @@ -356,7 +352,10 @@ mod tests { assert_eq!(config.num_experts(), 8); assert_eq!(config.model_type(), Some("top_level")); - assert_eq!(config.max_position_embeddings(), Some(4096)); + assert_eq!( + config.effective_text_config().max_position_embeddings, + Some(4096) + ); assert!(config.is_moe()); } @@ -367,7 +366,10 @@ mod tests { assert_eq!(config.num_experts(), 0); assert!(!config.is_moe()); - assert_eq!(config.max_position_embeddings(), Some(4096)); + assert_eq!( + config.effective_text_config().max_position_embeddings, + Some(4096) + ); } #[test] diff --git a/rust/src/text/src/backend/hf/mod.rs b/rust/src/text/src/backend/hf/mod.rs index b6b79d9914..94241ea74d 100644 --- a/rust/src/text/src/backend/hf/mod.rs +++ b/rust/src/text/src/backend/hf/mod.rs @@ -118,7 +118,6 @@ impl TextBackend for HfTextBackend { default_min_p: self.generation_config.min_p, default_repetition_penalty: self.generation_config.repetition_penalty, default_max_tokens: self.generation_config.max_new_tokens, - max_model_len: self.model_config.max_position_embeddings(), }) } } diff --git a/rust/src/text/src/backend/mod.rs b/rust/src/text/src/backend/mod.rs index 680d454da9..2b11d81d96 100644 --- a/rust/src/text/src/backend/mod.rs +++ b/rust/src/text/src/backend/mod.rs @@ -6,8 +6,8 @@ use vllm_tokenizer::DynTokenizer; use crate::error::Result; -/// Tokenizer/model-derived hints used to enrich text-generation requests before -/// they are lowered into engine-core. +/// Tokenizer/model-derived defaults used to enrich text-generation requests +/// before they are lowered into engine-core. #[derive(Debug, Clone, Default, PartialEq)] pub struct SamplingHints { pub primary_eos_token_id: Option, @@ -18,9 +18,36 @@ pub struct SamplingHints { pub default_min_p: Option, pub default_repetition_penalty: Option, pub default_max_tokens: Option, - /// Model context window size (`max_position_embeddings` from - /// `config.json`). - pub max_model_len: Option, +} + +/// Effective bounds used to validate and lower sampling requests. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SamplingLimits { + /// Runtime context window size reported by the engine startup handshake. + pub max_model_len: u32, + /// Maximum number of top log probabilities accepted by this frontend. + /// + /// `-1` means allowing requests up to the model vocabulary size. + pub max_logprobs: i32, + /// Model vocabulary size from the model config. + pub model_vocab_size: Option, + /// Tokenizer vocabulary size, used as a fallback when the model config does + /// not expose a vocabulary size. + pub tokenizer_vocab_size: usize, +} + +impl SamplingLimits { + /// Original Python definition: + /// + pub const DEFAULT_MAX_LOGPROBS: i32 = 20; + /// Original Python definition: + /// + pub const MAX_LOGPROB_TOKEN_IDS: usize = 128; + + /// Return the vocabulary size used to expand `logprobs=-1`. + pub fn logprobs_vocab_size(&self) -> usize { + self.model_vocab_size.unwrap_or(self.tokenizer_vocab_size) + } } /// Minimal text-processing backend needed by `vllm-text`. diff --git a/rust/src/text/src/error.rs b/rust/src/text/src/error.rs index 62e8e2ae98..a6b2fe5af1 100644 --- a/rust/src/text/src/error.rs +++ b/rust/src/text/src/error.rs @@ -2,6 +2,8 @@ use thiserror::Error; use vllm_engine_core_client::Error as EngineCoreError; use vllm_llm::Error as LlmError; +pub use crate::lower::logprobs::LogprobsError; + #[derive(Debug, Error)] pub enum Error { #[error("tokenizer error: {0}")] @@ -13,6 +15,8 @@ pub enum Error { but the prompt contains {prompt_len} input tokens" )] PromptTooLong { max_model_len: u32, prompt_len: u32 }, + #[error(transparent)] + Logprobs(#[from] LogprobsError), #[error("text request stream `{request_id}` closed before terminal output")] StreamClosedBeforeTerminalOutput { request_id: String }, #[error(transparent)] diff --git a/rust/src/text/src/lib.rs b/rust/src/text/src/lib.rs index a8ab4191ef..b23e33135c 100644 --- a/rust/src/text/src/lib.rs +++ b/rust/src/text/src/lib.rs @@ -6,8 +6,8 @@ use std::mem::take; -pub use backend::{DynTextBackend, SamplingHints, TextBackend}; -pub use error::{Error, Result}; +pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend}; +pub use error::{Error, LogprobsError, Result}; use futures::Stream; pub use lower::{ PreparedTextRequest, lower_sampling_params, lower_text_request, resolve_max_tokens, @@ -45,33 +45,33 @@ pub struct TextLlm { /// Tokenizer/model metadata backend responsible for prompt encode/decode /// and sampling hints. backend: DynTextBackend, - /// Context window size reported by the engine startup handshake, with - /// optional override from config. + /// Runtime context window size reported by the engine startup handshake. max_model_len: u32, + /// Maximum number of top log probabilities accepted by this text facade. + max_logprobs: i32, } impl TextLlm { /// Create a new text-generation facade from a shared LLM client plus a text /// backend. pub fn new(llm: Llm, backend: DynTextBackend) -> Self { - // Prefer the engine-reported max_model_len because it reflects the - // post-profiling, auto-fitted KV cache limit rather than static - // frontend metadata. + // The engine-reported value reflects the post-profiling, auto-fitted + // KV cache limit used at runtime. let max_model_len = llm.engine_core_client().max_model_len(); Self { llm, backend, max_model_len, + max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS, } } - /// Override the maximum model context length explicitly. - /// - /// This takes priority over both the engine-reported default and any - /// tokenizer/model metadata exposed by the backend. - pub fn with_max_model_len(mut self, max_model_len: u32) -> Self { - self.max_model_len = max_model_len; + /// Override the maximum accepted logprobs count. + pub fn with_max_logprobs(mut self, max_logprobs: Option) -> Self { + if let Some(max_logprobs) = max_logprobs { + self.max_logprobs = max_logprobs; + } self } @@ -140,12 +140,24 @@ impl TextLlm { Prompt::TokenIds(token_ids) => token_ids, }; - let mut sampling_hints = self.backend.sampling_hints()?; - sampling_hints.max_model_len = Some(self.max_model_len); + let sampling_hints = self.backend.sampling_hints()?; + let sampling_limits = SamplingLimits { + max_model_len: self.max_model_len, + max_logprobs: self.max_logprobs, + model_vocab_size: self.backend.model_vocab_size(), + tokenizer_vocab_size: self.backend.tokenizer_vocab_size(), + }; + let PreparedTextRequest { text_request, generate_request, - } = lower_text_request(request, prompt_token_ids, sampling_hints, &*tokenizer)?; + } = lower_text_request( + request, + prompt_token_ids, + sampling_hints, + sampling_limits, + &*tokenizer, + )?; let raw_stream = self.llm.generate(generate_request).await?; Ok((text_request, raw_stream)) diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index d661c99606..d482f8cbf1 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -1,12 +1,15 @@ use std::collections::BTreeSet; +pub(crate) mod logprobs; + use vllm_engine_core_client::protocol::EngineCoreSamplingParams; use vllm_llm::GenerateRequest; use vllm_tokenizer::Tokenizer; -use crate::backend::SamplingHints; +use crate::backend::{SamplingHints, SamplingLimits}; use crate::error::{Error, Result}; use crate::request::{SamplingParams, TextRequest}; +use logprobs::validate_logprobs; /// One text request after it has been lowered into the raw generate boundary. #[derive(Debug)] @@ -24,6 +27,7 @@ pub fn lower_text_request( request: TextRequest, prompt_token_ids: Vec, sampling_hints: SamplingHints, + sampling_limits: SamplingLimits, tokenizer: &dyn Tokenizer, ) -> Result { let prompt_len = prompt_token_ids.len() as u32; @@ -34,6 +38,7 @@ pub fn lower_text_request( sampling_params: lower_sampling_params( request.sampling_params.clone(), sampling_hints, + sampling_limits, prompt_len, tokenizer, )?, @@ -66,8 +71,8 @@ pub fn lower_sampling_params( default_min_p, default_repetition_penalty, default_max_tokens, - max_model_len, }: SamplingHints, + sampling_limits: SamplingLimits, prompt_len: u32, tokenizer: &dyn Tokenizer, ) -> Result { @@ -95,6 +100,14 @@ pub fn lower_sampling_params( vllm_xargs, } = sampling_params; + // Validate logprobs-related fields first with runtime sampling limits first. + validate_logprobs( + logprobs, + prompt_logprobs, + logprob_token_ids.as_deref(), + sampling_limits, + )?; + // Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat // path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450 // If neither the caller nor the model provides a value, fall back to 1.0 — the @@ -105,7 +118,12 @@ pub fn lower_sampling_params( let top_k = top_k.or(default_top_k).unwrap_or(0); let min_p = min_p.or(default_min_p).unwrap_or(0.0); let repetition_penalty = repetition_penalty.or(default_repetition_penalty).unwrap_or(1.0); - let max_tokens = resolve_max_tokens(max_tokens, default_max_tokens, max_model_len, prompt_len)?; + let max_tokens = resolve_max_tokens( + max_tokens, + default_max_tokens, + sampling_limits.max_model_len, + prompt_len, + )?; let min_tokens = min_tokens.unwrap_or(0); let frequency_penalty = frequency_penalty.unwrap_or(0.0); let presence_penalty = presence_penalty.unwrap_or(0.0); @@ -189,33 +207,25 @@ fn tokenize_bad_words( /// Resolve the effective `max_tokens` for generation, mirroring vLLM Python's /// `get_max_tokens()` in `vllm/entrypoints/utils.py`. /// -/// Takes the minimum of all available limits (user-specified, generation-config -/// default, and `max_model_len - prompt_len`). When nothing is known, falls -/// back to `u32::MAX` so the engine-core can apply its own context-window -/// limit. +/// Takes the minimum of all available limits: user-specified, generation-config +/// default, and `max_model_len - prompt_len`. pub fn resolve_max_tokens( user_max_tokens: Option, default_max_tokens: Option, - max_model_len: Option, + max_model_len: u32, prompt_len: u32, ) -> Result { - let model_max_tokens = match max_model_len { - Some(max_model_len) if prompt_len >= max_model_len => { - return Err(Error::PromptTooLong { - max_model_len, - prompt_len, - }); - } - Some(max_model_len) => Some(max_model_len - prompt_len), - None => None, + let model_max_tokens = if prompt_len >= max_model_len { + return Err(Error::PromptTooLong { + max_model_len, + prompt_len, + }); + } else { + max_model_len - prompt_len }; - let fallback_max_tokens = user_max_tokens.or(default_max_tokens); - Ok([fallback_max_tokens, model_max_tokens] - .into_iter() - .flatten() - .min() - .unwrap_or(u32::MAX /* TODO: a reasonable fallback? */)) + let request_max_tokens = user_max_tokens.or(default_max_tokens); + Ok(request_max_tokens.map_or(model_max_tokens, |n| n.min(model_max_tokens))) } fn merge_unique_token_ids( @@ -240,6 +250,7 @@ mod tests { use super::*; use crate::backend::hf::HfTextBackend; use crate::backend::{SamplingHints, TextBackend as _}; + use crate::error::LogprobsError; use crate::request::{Prompt, TextRequest}; /// Stub tokenizer that returns empty token IDs — sufficient for tests that @@ -290,16 +301,47 @@ mod tests { default_min_p: None, default_repetition_penalty: None, default_max_tokens: None, - max_model_len: None, } } + fn sample_sampling_limits() -> SamplingLimits { + SamplingLimits { + max_model_len: 1_000_000, + max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS, + model_vocab_size: Some(1000), + tokenizer_vocab_size: 2000, + } + } + + fn lower_sampling_params_with_limits( + sampling_params: SamplingParams, + sampling_limits: SamplingLimits, + ) -> Result { + lower_sampling_params( + sampling_params, + SamplingHints { + primary_eos_token_id: None, + extra_eos_token_ids: BTreeSet::new(), + default_temperature: None, + default_top_p: None, + default_top_k: None, + default_min_p: None, + default_repetition_penalty: None, + default_max_tokens: None, + }, + sampling_limits, + 3, + &stub_tokenizer(), + ) + } + #[test] fn lower_text_request_applies_python_style_eos_hints() { let prepared = lower_text_request( sample_request(), vec![1, 2, 3], sample_sampling_hints(), + sample_sampling_limits(), &stub_tokenizer(), ) .unwrap(); @@ -311,7 +353,7 @@ mod tests { top_p: 1.0, top_k: 0, seed: None, - max_tokens: 4294967295, + max_tokens: 999997, min_tokens: 0, logprobs: None, prompt_logprobs: None, @@ -350,6 +392,7 @@ mod tests { request, vec![1, 2, 3], sample_sampling_hints(), + sample_sampling_limits(), &stub_tokenizer(), ) .unwrap(); @@ -361,7 +404,7 @@ mod tests { top_p: 1.0, top_k: 0, seed: None, - max_tokens: 4294967295, + max_tokens: 999997, min_tokens: 0, logprobs: None, prompt_logprobs: None, @@ -415,16 +458,23 @@ mod tests { default_min_p: None, default_repetition_penalty: None, default_max_tokens: None, - max_model_len: Some( - 40960, - ), } "#]] .assert_debug_eq(&hints); - let prepared = - lower_text_request(sample_request(), vec![1, 2, 3], hints, &stub_tokenizer()) - .expect("lower request"); + let prepared = lower_text_request( + sample_request(), + vec![1, 2, 3], + hints, + SamplingLimits { + max_model_len: 40960, + max_logprobs: SamplingLimits::DEFAULT_MAX_LOGPROBS, + model_vocab_size: backend.model_vocab_size(), + tokenizer_vocab_size: backend.tokenizer_vocab_size(), + }, + &stub_tokenizer(), + ) + .expect("lower request"); let params = prepared.generate_request.sampling_params; expect_test::expect![[r#" @@ -481,8 +531,8 @@ mod tests { default_min_p: None, default_repetition_penalty: None, default_max_tokens: None, - max_model_len: None, }, + sample_sampling_limits(), 3, &stub_tokenizer(), ) @@ -494,7 +544,7 @@ mod tests { top_p: 1.0, top_k: 0, seed: None, - max_tokens: 4294967295, + max_tokens: 999997, min_tokens: 0, logprobs: None, prompt_logprobs: None, @@ -550,8 +600,8 @@ mod tests { default_min_p: Some(0.1), default_repetition_penalty: Some(1.2), default_max_tokens: Some(128), - max_model_len: None, }, + sample_sampling_limits(), 3, &stub_tokenizer(), ) @@ -605,7 +655,10 @@ mod tests { default_min_p: None, default_repetition_penalty: None, default_max_tokens: None, - max_model_len: None, + }, + SamplingLimits { + max_logprobs: -1, + ..sample_sampling_limits() }, 3, &stub_tokenizer(), @@ -616,6 +669,91 @@ mod tests { assert_eq!(params.prompt_logprobs, Some(-1)); } + #[test] + fn lower_sampling_params_rejects_full_vocab_logprobs_over_default_cap() { + let error = lower_sampling_params_with_limits( + SamplingParams { + logprobs: Some(-1), + ..Default::default() + }, + sample_sampling_limits(), + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::Logprobs(LogprobsError::TooManyCount { + parameter: "logprobs", + requested: 1000, + max_allowed: 20, + }) + )); + } + + #[test] + fn lower_sampling_params_expands_full_vocab_logprobs_from_model_vocab() { + let params = lower_sampling_params_with_limits( + SamplingParams { + logprobs: Some(-1), + ..Default::default() + }, + SamplingLimits { + max_logprobs: 1500, + ..sample_sampling_limits() + }, + ) + .unwrap(); + + assert_eq!(params.logprobs, Some(-1)); + } + + #[test] + fn lower_sampling_params_uses_tokenizer_vocab_when_model_vocab_is_unknown() { + let error = lower_sampling_params_with_limits( + SamplingParams { + logprobs: Some(-1), + ..Default::default() + }, + SamplingLimits { + max_logprobs: 1500, + model_vocab_size: None, + tokenizer_vocab_size: 2000, + ..sample_sampling_limits() + }, + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::Logprobs(LogprobsError::TooManyCount { + parameter: "logprobs", + requested: 2000, + max_allowed: 1500, + }) + )); + } + + #[test] + fn lower_sampling_params_rejects_invalid_logprob_token_ids() { + let error = lower_sampling_params_with_limits( + SamplingParams { + logprobs: Some(1), + logprob_token_ids: Some(vec![1000]), + ..Default::default() + }, + sample_sampling_limits(), + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::Logprobs(LogprobsError::InvalidTokenIds { + token_ids, + vocab_size: 1000, + }) if token_ids == vec![1000] + )); + } + #[test] fn lower_sampling_params_uses_generation_defaults_when_user_omits_values() { let params = lower_sampling_params( @@ -629,8 +767,8 @@ mod tests { default_min_p: Some(0.1), default_repetition_penalty: Some(1.2), default_max_tokens: Some(128), - max_model_len: None, }, + sample_sampling_limits(), 3, &stub_tokenizer(), ) @@ -667,7 +805,7 @@ mod tests { #[test] fn resolve_max_tokens_caps_by_model_len() { - let result = resolve_max_tokens(Some(150), None, Some(200), 100); + let result = resolve_max_tokens(Some(150), None, 200, 100); assert_eq!(result.unwrap(), 100); } @@ -680,6 +818,7 @@ mod tests { request, vec![1, 2, 3], sample_sampling_hints(), + sample_sampling_limits(), &stub_tokenizer(), ) .unwrap(); @@ -690,37 +829,31 @@ mod tests { #[test] fn resolve_max_tokens_user_smaller_than_model_limit() { - let result = resolve_max_tokens(Some(50), None, Some(200), 100); + let result = resolve_max_tokens(Some(50), None, 200, 100); assert_eq!(result.unwrap(), 50); } #[test] fn resolve_max_tokens_uses_default_when_user_omits() { - let result = resolve_max_tokens(None, Some(64), Some(200), 100); + let result = resolve_max_tokens(None, Some(64), 200, 100); assert_eq!(result.unwrap(), 64); } #[test] fn resolve_max_tokens_default_capped_by_model_len() { - let result = resolve_max_tokens(None, Some(256), Some(200), 100); + let result = resolve_max_tokens(None, Some(256), 200, 100); assert_eq!(result.unwrap(), 100); } #[test] - fn resolve_max_tokens_no_model_len_falls_back() { - let result = resolve_max_tokens(Some(9999), None, None, 100); - assert_eq!(result.unwrap(), 9999); - } - - #[test] - fn resolve_max_tokens_no_limits_known_falls_back_to_u32_max() { - let result = resolve_max_tokens(None, None, None, 100); - assert_eq!(result.unwrap(), u32::MAX); + fn resolve_max_tokens_uses_model_limit_when_user_omits() { + let result = resolve_max_tokens(None, None, 200, 100); + assert_eq!(result.unwrap(), 100); } #[test] fn resolve_max_tokens_prompt_too_long() { - let result = resolve_max_tokens(Some(10), None, Some(100), 100); + let result = resolve_max_tokens(Some(10), None, 100, 100); assert!(matches!( result, Err(Error::PromptTooLong { @@ -732,7 +865,7 @@ mod tests { #[test] fn resolve_max_tokens_prompt_exceeds_model_len() { - let result = resolve_max_tokens(Some(10), None, Some(100), 200); + let result = resolve_max_tokens(Some(10), None, 100, 200); assert!(matches!( result, Err(Error::PromptTooLong { diff --git a/rust/src/text/src/lower/logprobs.rs b/rust/src/text/src/lower/logprobs.rs new file mode 100644 index 0000000000..0372b20c9b --- /dev/null +++ b/rust/src/text/src/lower/logprobs.rs @@ -0,0 +1,134 @@ +//! Python-compatible validation for logprobs sampling params. +//! +//! `-1` is expanded only for bounds checks. The original request values are +//! passed through to engine-core. + +use crate::backend::SamplingLimits; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum LogprobsError { + #[error("{parameter} must be non-negative or -1, got {value}")] + InvalidCount { parameter: &'static str, value: i32 }, + #[error( + "requested {parameter} of {requested}, which is greater than max allowed: {max_allowed}" + )] + TooManyCount { + parameter: &'static str, + requested: usize, + max_allowed: usize, + }, + #[error( + "requested logprob_token_ids of length {requested}, \ + which is greater than max allowed: {max_allowed}" + )] + TooManyTokenIds { + requested: usize, + max_allowed: usize, + }, + #[error( + "token_id(s) {token_ids:?} in logprob_token_ids contain out-of-vocab token ids. \ + Vocabulary size: {vocab_size}" + )] + InvalidTokenIds { + token_ids: Vec, + vocab_size: usize, + }, + #[error( + "when both logprobs and logprob_token_ids are set, logprobs must equal \ + len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}." + )] + TokenIdsMismatch { logprobs: i32, num_token_ids: usize }, +} + +/// Validate logprobs-related sampling parameters, returning an error if any +/// parameter is out of bounds or if the combination of parameters is invalid. +pub(super) fn validate_logprobs( + logprobs: Option, + prompt_logprobs: Option, + logprob_token_ids: Option<&[u32]>, + sampling_limits: SamplingLimits, +) -> Result<(), LogprobsError> { + let vocab_size = sampling_limits.logprobs_vocab_size(); + let max_logprobs = + normalize_logprobs_count(sampling_limits.max_logprobs, vocab_size, "max_logprobs")?; + + validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?; + validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?; + validate_logprob_token_ids(logprobs, logprob_token_ids, vocab_size) +} + +fn validate_logprobs_count( + requested: Option, + max_logprobs: usize, + vocab_size: usize, + parameter: &'static str, +) -> Result<(), LogprobsError> { + let Some(requested) = requested else { + return Ok(()); + }; + + let requested = normalize_logprobs_count(requested, vocab_size, parameter)?; + if requested > max_logprobs { + return Err(LogprobsError::TooManyCount { + parameter, + requested, + max_allowed: max_logprobs, + }); + } + + Ok(()) +} + +fn validate_logprob_token_ids( + logprobs: Option, + logprob_token_ids: Option<&[u32]>, + vocab_size: usize, +) -> Result<(), LogprobsError> { + let Some(logprob_token_ids) = logprob_token_ids else { + return Ok(()); + }; + + let n = logprob_token_ids.len(); + if n > SamplingLimits::MAX_LOGPROB_TOKEN_IDS { + return Err(LogprobsError::TooManyTokenIds { + requested: n, + max_allowed: SamplingLimits::MAX_LOGPROB_TOKEN_IDS, + }); + } + + let invalid_token_ids: Vec<_> = logprob_token_ids + .iter() + .copied() + .filter(|&token_id| token_id as usize >= vocab_size) + .collect(); + if !invalid_token_ids.is_empty() { + return Err(LogprobsError::InvalidTokenIds { + token_ids: invalid_token_ids, + vocab_size, + }); + } + + if let Some(logprobs) = logprobs + && logprobs != n as i32 + { + return Err(LogprobsError::TokenIdsMismatch { + logprobs, + num_token_ids: n, + }); + } + + Ok(()) +} + +fn normalize_logprobs_count( + value: i32, + vocab_size: usize, + parameter: &'static str, +) -> Result { + match value { + -1 => Ok(vocab_size), + value if value < 0 => Err(LogprobsError::InvalidCount { parameter, value }), + value => Ok(value as usize), + } +}