[Frontend][Rust] Reject empty structured-output values (#54821)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
parent
658c8131c7
commit
22f6e4eccb
@@ -4,6 +4,7 @@
|
||||
//! Conversion between gRPC protobuf types and internal `vllm-text`
|
||||
//! request/response types.
|
||||
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tonic::Status;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
@@ -259,6 +260,9 @@ fn convert_structured_output(
|
||||
StructuredOutputsParams::structural_tag(tag.clone())
|
||||
}
|
||||
};
|
||||
params
|
||||
.validate()
|
||||
.map_err(|error| Status::invalid_argument(error.to_report_string()))?;
|
||||
Ok(Some(params))
|
||||
}
|
||||
|
||||
@@ -546,6 +550,20 @@ mod tests {
|
||||
assert_eq!(text.sampling_params.temperature, Some(0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grpc_rejects_empty_grammar_before_engine() {
|
||||
use super::pb::decoding_parameters::StructuredOutput;
|
||||
let req = pb::GenerateRequest {
|
||||
decoding: Some(pb::DecodingParameters {
|
||||
structured_output: Some(StructuredOutput::Grammar(" ".to_string())),
|
||||
..Default::default()
|
||||
}),
|
||||
..base_request()
|
||||
};
|
||||
let err = to_text_request(req, false, &["test-model".to_string()]).unwrap_err();
|
||||
assert!(err.message().contains("grammar cannot be an empty string"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_seed_is_none() {
|
||||
let req = pb::GenerateRequest {
|
||||
|
||||
@@ -437,7 +437,7 @@ mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use expect_test::expect;
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde_json::json;
|
||||
@@ -583,6 +583,61 @@ mod tests {
|
||||
assert_eq!(prepared.chat_request.decode_options.min_tokens, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_empty_json_schema() {
|
||||
let mut request = base_request();
|
||||
request.response_format = Some(ResponseFormat::JsonSchema {
|
||||
json_schema: JsonSchemaFormat {
|
||||
name: "answer".to_string(),
|
||||
description: None,
|
||||
schema: json!(" "),
|
||||
strict: None,
|
||||
},
|
||||
});
|
||||
|
||||
let error = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(error.status_code(), StatusCode::BAD_REQUEST);
|
||||
assert!(
|
||||
error
|
||||
.to_error_response()
|
||||
.error
|
||||
.message
|
||||
.contains("json cannot be an empty string")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_rejects_whitespace_only_structured_outputs_grammar() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"structured_outputs": {"grammar": " \t\n"},
|
||||
}))
|
||||
.expect("parse structured_outputs");
|
||||
|
||||
let error = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(error.status_code(), StatusCode::BAD_REQUEST);
|
||||
assert!(
|
||||
error
|
||||
.to_error_response()
|
||||
.error
|
||||
.message
|
||||
.contains("grammar cannot be an empty string")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_defaults_function_tool_fields() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
|
||||
use crate::error::ApiError;
|
||||
@@ -80,12 +81,12 @@ pub fn convert_from_response_format(
|
||||
let Some(fmt) = response_format else {
|
||||
return Ok(None);
|
||||
};
|
||||
match fmt {
|
||||
ResponseFormat::Text => Ok(None),
|
||||
ResponseFormat::JsonObject => Ok(Some(StructuredOutputsParams::json_object())),
|
||||
ResponseFormat::JsonSchema { json_schema } => Ok(Some(StructuredOutputsParams::json(
|
||||
json_schema.schema.clone(),
|
||||
))),
|
||||
let params = match fmt {
|
||||
ResponseFormat::Text => return Ok(None),
|
||||
ResponseFormat::JsonObject => StructuredOutputsParams::json_object(),
|
||||
ResponseFormat::JsonSchema { json_schema } => {
|
||||
StructuredOutputsParams::json(json_schema.schema.clone())
|
||||
}
|
||||
ResponseFormat::StructuralTag { .. } => {
|
||||
// The Python frontend dumps the entire response_format object (including the
|
||||
// `type` field) as a JSON string for the engine-core backend.
|
||||
@@ -95,9 +96,13 @@ pub fn convert_from_response_format(
|
||||
Some("response_format"),
|
||||
)
|
||||
})?;
|
||||
Ok(Some(StructuredOutputsParams::structural_tag(tag_json)))
|
||||
StructuredOutputsParams::structural_tag(tag_json)
|
||||
}
|
||||
}
|
||||
};
|
||||
params.validate().map_err(|error| {
|
||||
ApiError::invalid_request(error.to_report_string(), Some("response_format"))
|
||||
})?;
|
||||
Ok(Some(params))
|
||||
}
|
||||
|
||||
/// Convert raw `response_format` and/or `structured_outputs` JSON blobs into
|
||||
|
||||
Reference in New Issue
Block a user