[Rust Frontend] Support max_logprobs validation (#45674)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -68,6 +68,7 @@ async fn main() -> Result<()> {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
api_keys: Vec::new(),
|
||||
disable_log_stats: false,
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Result, bail};
|
||||
use educe::Educe;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
@@ -77,6 +77,9 @@ pub struct Config {
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
/// How to serialize `message.content` for chat-template rendering.
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
/// Optional maximum number of top log probabilities accepted by the
|
||||
/// frontend. `None` delegates to the text layer default.
|
||||
pub max_logprobs: Option<i32>,
|
||||
/// HTTP/API-server behavior switches.
|
||||
pub api_server_options: ApiServerOptions,
|
||||
/// API keys accepted as bearer tokens for guarded routes.
|
||||
@@ -98,6 +101,14 @@ impl Config {
|
||||
/// startup.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
|
||||
if let Some(max_logprobs) = self.max_logprobs
|
||||
&& max_logprobs < -1
|
||||
{
|
||||
bail!(
|
||||
"max_logprobs must be non-negative or -1, got {}",
|
||||
max_logprobs
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -74,12 +74,11 @@ impl IntoResponse for ApiError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify a text-pipeline submit failure: tokenized-prompt validation
|
||||
/// failures (the prompt is too long for the model, or empty after
|
||||
/// tokenization) are the client's fault and map to HTTP 400, mirroring the
|
||||
/// Python frontend. Everything else stays an internal 500.
|
||||
/// Classify a text-pipeline submit failure: request validation failures are
|
||||
/// the client's fault and map to HTTP 400, mirroring the Python frontend.
|
||||
/// Everything else stays an internal 500.
|
||||
pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError {
|
||||
if is_prompt_validation_error(&error) {
|
||||
if is_request_validation_error(&error) {
|
||||
return invalid_request!("{error}");
|
||||
}
|
||||
server_error!("{}: {}", context, error.to_report_string())
|
||||
@@ -90,18 +89,19 @@ pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiE
|
||||
pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError {
|
||||
match &error {
|
||||
vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"),
|
||||
vllm_chat::Error::Text(text_error) if is_prompt_validation_error(text_error) => {
|
||||
vllm_chat::Error::Text(text_error) if is_request_validation_error(text_error) => {
|
||||
invalid_request!("{error}")
|
||||
}
|
||||
_ => server_error!("{}: {}", context, error.to_report_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_prompt_validation_error(error: &vllm_text::Error) -> bool {
|
||||
fn is_request_validation_error(error: &vllm_text::Error) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
vllm_text::Error::PromptTooLong { .. }
|
||||
| vllm_text::Error::EmptyPromptTokenIds { .. }
|
||||
| vllm_text::Error::Logprobs(_)
|
||||
// An empty tokenized prompt detected later, at request prepare
|
||||
// time, surfaces through the transparent Llm wrapper.
|
||||
| vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
|
||||
@@ -145,6 +145,33 @@ mod tests {
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logprobs_validation_maps_to_invalid_request() {
|
||||
let error = vllm_text::Error::Logprobs(vllm_text::LogprobsError::TooManyCount {
|
||||
parameter: "logprobs",
|
||||
requested: 1000,
|
||||
max_allowed: 20,
|
||||
});
|
||||
let api_error = text_submit_error("failed to submit completion request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
let response = api_error.to_error_response();
|
||||
assert_eq!(response.error.error_type, "invalid_request_error");
|
||||
assert!(response.error.message.contains("logprobs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_wrapped_logprobs_validation_maps_to_invalid_request() {
|
||||
let error = vllm_chat::Error::Text(vllm_text::Error::Logprobs(
|
||||
vllm_text::LogprobsError::TooManyCount {
|
||||
parameter: "prompt_logprobs",
|
||||
requested: 1000,
|
||||
max_allowed: 20,
|
||||
},
|
||||
));
|
||||
let api_error = chat_submit_error("failed to submit chat request", error);
|
||||
assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_submit_errors_stay_internal() {
|
||||
let error = vllm_text::Error::Tokenizer("backend exploded".to_string());
|
||||
|
||||
@@ -90,7 +90,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
.context("failed to connect to engine core")?;
|
||||
|
||||
let llm = Llm::new(client).with_log_stats(!config.disable_log_stats);
|
||||
let text = TextLlm::new(llm, text_backend);
|
||||
let text = TextLlm::new(llm, text_backend).with_max_logprobs(config.max_logprobs);
|
||||
|
||||
let chat = ChatLlm::new(text, chat_backend)
|
||||
.with_tool_call_parser(config.tool_call_parser.clone())
|
||||
|
||||
Reference in New Issue
Block a user