[Rust Frontend] Support max_logprobs validation (#45674)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-16 10:57:56 +08:00
committed by GitHub
parent b00e76ff72
commit 3f65e21e32
15 changed files with 484 additions and 98 deletions
@@ -68,6 +68,7 @@ async fn main() -> Result<()> {
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
max_logprobs: None,
api_server_options: ApiServerOptions::default(),
api_keys: Vec::new(),
disable_log_stats: false,
+12 -1
View File
@@ -2,7 +2,7 @@ use std::collections::HashMap;
use std::fmt;
use std::time::Duration;
use anyhow::Result;
use anyhow::{Result, bail};
use educe::Educe;
use serde::Serialize;
use serde_json::Value;
@@ -77,6 +77,9 @@ pub struct Config {
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
/// How to serialize `message.content` for chat-template rendering.
pub chat_template_content_format: ChatTemplateContentFormatOption,
/// Optional maximum number of top log probabilities accepted by the
/// frontend. `None` delegates to the text layer default.
pub max_logprobs: Option<i32>,
/// HTTP/API-server behavior switches.
pub api_server_options: ApiServerOptions,
/// API keys accepted as bearer tokens for guarded routes.
@@ -98,6 +101,14 @@ impl Config {
/// startup.
pub fn validate(&self) -> Result<()> {
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
if let Some(max_logprobs) = self.max_logprobs
&& max_logprobs < -1
{
bail!(
"max_logprobs must be non-negative or -1, got {}",
max_logprobs
);
}
Ok(())
}
+34 -7
View File
@@ -74,12 +74,11 @@ impl IntoResponse for ApiError {
}
}
/// Classify a text-pipeline submit failure: tokenized-prompt validation
/// failures (the prompt is too long for the model, or empty after
/// tokenization) are the client's fault and map to HTTP 400, mirroring the
/// Python frontend. Everything else stays an internal 500.
/// Classify a text-pipeline submit failure: request validation failures are
/// the client's fault and map to HTTP 400, mirroring the Python frontend.
/// Everything else stays an internal 500.
pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError {
if is_prompt_validation_error(&error) {
if is_request_validation_error(&error) {
return invalid_request!("{error}");
}
server_error!("{}: {}", context, error.to_report_string())
@@ -90,18 +89,19 @@ pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiE
pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError {
match &error {
vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"),
vllm_chat::Error::Text(text_error) if is_prompt_validation_error(text_error) => {
vllm_chat::Error::Text(text_error) if is_request_validation_error(text_error) => {
invalid_request!("{error}")
}
_ => server_error!("{}: {}", context, error.to_report_string()),
}
}
fn is_prompt_validation_error(error: &vllm_text::Error) -> bool {
fn is_request_validation_error(error: &vllm_text::Error) -> bool {
matches!(
error,
vllm_text::Error::PromptTooLong { .. }
| vllm_text::Error::EmptyPromptTokenIds { .. }
| vllm_text::Error::Logprobs(_)
// An empty tokenized prompt detected later, at request prepare
// time, surfaces through the transparent Llm wrapper.
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
@@ -145,6 +145,33 @@ mod tests {
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
}
#[test]
fn logprobs_validation_maps_to_invalid_request() {
let error = vllm_text::Error::Logprobs(vllm_text::LogprobsError::TooManyCount {
parameter: "logprobs",
requested: 1000,
max_allowed: 20,
});
let api_error = text_submit_error("failed to submit completion request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
let response = api_error.to_error_response();
assert_eq!(response.error.error_type, "invalid_request_error");
assert!(response.error.message.contains("logprobs"));
}
#[test]
fn chat_wrapped_logprobs_validation_maps_to_invalid_request() {
let error = vllm_chat::Error::Text(vllm_text::Error::Logprobs(
vllm_text::LogprobsError::TooManyCount {
parameter: "prompt_logprobs",
requested: 1000,
max_allowed: 20,
},
));
let api_error = chat_submit_error("failed to submit chat request", error);
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
}
#[test]
fn other_submit_errors_stay_internal() {
let error = vllm_text::Error::Tokenizer("backend exploded".to_string());
+1 -1
View File
@@ -90,7 +90,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.context("failed to connect to engine core")?;
let llm = Llm::new(client).with_log_stats(!config.disable_log_stats);
let text = TextLlm::new(llm, text_backend);
let text = TextLlm::new(llm, text_backend).with_max_logprobs(config.max_logprobs);
let chat = ChatLlm::new(text, chat_backend)
.with_tool_call_parser(config.tool_call_parser.clone())