[Rust Frontend] Align OpenAI request and response edge cases (#53218)
Co-authored-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -63,7 +63,8 @@ pub async fn chat_completions(
|
||||
) -> Response {
|
||||
let stream = body.stream;
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
|
||||
let requested_model = body.model.as_deref().filter(|model| !model.is_empty());
|
||||
let lora_resolution = state.resolve_model_with_loras(requested_model).await;
|
||||
|
||||
let prepared = match prepare_chat_request(body, &lora_resolution, request_context) {
|
||||
Ok(prepared) => prepared,
|
||||
@@ -196,7 +197,6 @@ async fn collect_chat_completion(
|
||||
Some(prompt_logprobs_to_maps(
|
||||
prompt_logprobs.as_ref(),
|
||||
&prompt_token_ids,
|
||||
return_tokens_as_token_ids,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
|
||||
@@ -433,6 +433,7 @@ mod tests {
|
||||
use expect_test::expect;
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde_json::json;
|
||||
use validator::Validate;
|
||||
use vllm_chat::{
|
||||
AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
|
||||
ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
|
||||
@@ -466,7 +467,7 @@ mod tests {
|
||||
|
||||
fn base_request() -> ChatCompletionRequest {
|
||||
ChatCompletionRequest {
|
||||
model: "Qwen/Qwen1.5-0.5B-Chat".to_string(),
|
||||
model: Some("Qwen/Qwen1.5-0.5B-Chat".to_string()),
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Text("hello".to_string()),
|
||||
name: None,
|
||||
@@ -476,6 +477,120 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_http_request_defaults_missing_or_null_model() {
|
||||
for model in [None, Some(serde_json::Value::Null)] {
|
||||
let mut value = json!({
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"stream": true,
|
||||
});
|
||||
if let Some(model) = model {
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("request object")
|
||||
.insert("model".to_string(), model);
|
||||
}
|
||||
|
||||
let request: ChatCompletionRequest =
|
||||
serde_json::from_value(value).expect("parse request without model");
|
||||
assert!(request.model.is_none());
|
||||
}
|
||||
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"model": "",
|
||||
}))
|
||||
.expect("parse empty model");
|
||||
assert_eq!(request.model.as_deref(), Some(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_normalizes_top_k_sentinels() {
|
||||
for (top_k, expected) in [
|
||||
(serde_json::Value::Null, None),
|
||||
(json!(-1), Some(0)),
|
||||
(json!(0), Some(0)),
|
||||
(json!(20), Some(20)),
|
||||
] {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"stream": true,
|
||||
"top_k": top_k,
|
||||
}))
|
||||
.expect("parse top_k");
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare top_k");
|
||||
|
||||
assert_eq!(prepared.chat_request.sampling_params.top_k, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_zero_min_tokens() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"stream": true,
|
||||
"min_tokens": 0,
|
||||
}))
|
||||
.expect("parse zero min_tokens");
|
||||
request.validate().expect("validate zero min_tokens");
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare zero min_tokens");
|
||||
|
||||
assert_eq!(prepared.chat_request.sampling_params.min_tokens, Some(0));
|
||||
assert_eq!(prepared.chat_request.decode_options.min_tokens, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_defaults_function_tool_fields() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"tools": [{"function": {"name": "lookup"}}],
|
||||
}))
|
||||
.expect("parse tool defaults");
|
||||
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare tool defaults");
|
||||
|
||||
assert_eq!(
|
||||
prepared.chat_request.tools(),
|
||||
&[VllmChatTool {
|
||||
name: "lookup".to_string(),
|
||||
description: None,
|
||||
parameters: serde_json::Value::Null,
|
||||
strict: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_http_request_rejects_empty_cache_salt() {
|
||||
let request: ChatCompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"cache_salt": "",
|
||||
}))
|
||||
.expect("parse cache_salt");
|
||||
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_maps_parallel_tool_calls() {
|
||||
let mut request = base_request();
|
||||
@@ -506,7 +621,7 @@ mod tests {
|
||||
#[test]
|
||||
fn prepare_chat_request_passes_response_format_to_kimi_k3_renderer() {
|
||||
let mut request = base_request();
|
||||
request.model = "moonshotai/Kimi-K3".to_string();
|
||||
request.model = Some("moonshotai/Kimi-K3".to_string());
|
||||
let response_format = ResponseFormat::JsonSchema {
|
||||
json_schema: JsonSchemaFormat {
|
||||
name: "answer".to_string(),
|
||||
@@ -913,7 +1028,7 @@ mod tests {
|
||||
#[test]
|
||||
fn prepare_chat_request_accepts_audio_content_parts() {
|
||||
let request = ChatCompletionRequest {
|
||||
model: "Qwen/Qwen3-ASR-1.7B".to_string(),
|
||||
model: Some("Qwen/Qwen3-ASR-1.7B".to_string()),
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Parts(vec![
|
||||
ContentPart::InputAudio {
|
||||
|
||||
@@ -13,9 +13,9 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
|
||||
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall,
|
||||
ToolCallDelta, ToolChoice, UNKNOWN_MODEL_ID, Usage, default_true, validate_messages,
|
||||
validate_stop, validate_top_p_value,
|
||||
ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Tool,
|
||||
ToolCall, ToolCallDelta, ToolChoice, Usage, default_true, deserialize_request_top_k,
|
||||
validate_messages, validate_stop, validate_top_p_value,
|
||||
};
|
||||
|
||||
/// vLLM-compatible request type for the Chat Completions API.
|
||||
@@ -33,8 +33,7 @@ pub struct ChatCompletionRequest {
|
||||
pub messages: Vec<ChatMessage>,
|
||||
|
||||
/// ID of the model to use
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
pub model: Option<String>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based
|
||||
/// on their existing frequency in the text so far
|
||||
@@ -120,6 +119,7 @@ pub struct ChatCompletionRequest {
|
||||
pub use_beam_search: bool,
|
||||
|
||||
/// Top-k sampling parameter
|
||||
#[serde(default, deserialize_with = "deserialize_request_top_k")]
|
||||
pub top_k: Option<u32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
@@ -145,7 +145,6 @@ pub struct ChatCompletionRequest {
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[validate(range(min = 1))]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
@@ -233,6 +232,7 @@ pub struct ChatCompletionRequest {
|
||||
pub return_token_ids: Option<bool>,
|
||||
|
||||
/// Salt for prefix cache isolation in multi-user environments
|
||||
#[validate(length(min = 1))]
|
||||
pub cache_salt: Option<String>,
|
||||
|
||||
/// KV transfer parameters for disaggregated serving
|
||||
@@ -254,7 +254,7 @@ impl Default for ChatCompletionRequest {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
messages: Vec::new(),
|
||||
model: default_model(),
|
||||
model: None,
|
||||
frequency_penalty: None,
|
||||
logit_bias: None,
|
||||
logprobs: false,
|
||||
@@ -341,7 +341,7 @@ pub(super) struct ChatCompletionResponse {
|
||||
pub choices: Vec<ChatCompletionChoice>,
|
||||
pub usage: Option<Usage>,
|
||||
pub system_fingerprint: Option<String>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,
|
||||
pub prompt_logprobs: Option<PromptLogprobs>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
pub ec_transfer_params: Option<Value>,
|
||||
@@ -429,10 +429,6 @@ pub(super) struct ChatMessageDelta {
|
||||
pub reasoning: Option<String>,
|
||||
}
|
||||
|
||||
fn default_model() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_chat_cross_parameters(
|
||||
req: &ChatCompletionRequest,
|
||||
|
||||
@@ -10,8 +10,10 @@ pub(super) fn validate_request_compat(
|
||||
request: &ChatCompletionRequest,
|
||||
served_model_names: &[String],
|
||||
) -> Result<(), ApiError> {
|
||||
if !served_model_names.iter().any(|n| n == &request.model) {
|
||||
return Err(ApiError::model_not_found(request.model.clone()));
|
||||
if let Some(model) = request.model.as_ref().filter(|model| !model.is_empty())
|
||||
&& !served_model_names.iter().any(|name| name == model)
|
||||
{
|
||||
return Err(ApiError::model_not_found(model.clone()));
|
||||
}
|
||||
|
||||
if request.stream_options.is_some() && !request.stream {
|
||||
@@ -142,7 +144,7 @@ mod tests {
|
||||
|
||||
fn base_request() -> ChatCompletionRequest {
|
||||
ChatCompletionRequest {
|
||||
model: "Qwen/Qwen1.5-0.5B-Chat".to_string(),
|
||||
model: Some("Qwen/Qwen1.5-0.5B-Chat".to_string()),
|
||||
messages: vec![ChatMessage::User {
|
||||
content: MessageContent::Text("hello".to_string()),
|
||||
name: None,
|
||||
@@ -163,6 +165,18 @@ mod tests {
|
||||
.expect("stop strings should be accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_default_model_inputs() {
|
||||
for model in [None, Some(String::new())] {
|
||||
let request = ChatCompletionRequest {
|
||||
model,
|
||||
..base_request()
|
||||
};
|
||||
validate_request_compat(&request, &served(&["served-model"]))
|
||||
.expect("default model input should select the primary served model");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_non_zero_penalties_and_function_tools() {
|
||||
let request = ChatCompletionRequest {
|
||||
|
||||
@@ -66,7 +66,8 @@ pub async fn completions(
|
||||
) -> Response {
|
||||
let stream = body.stream;
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
|
||||
let requested_model = body.model.as_deref().filter(|model| !model.is_empty());
|
||||
let lora_resolution = state.resolve_model_with_loras(requested_model).await;
|
||||
|
||||
let tokenizer = state.chat.text().tokenizer();
|
||||
let prepared = match prepare_completion_request(
|
||||
@@ -186,7 +187,6 @@ async fn collect_completion(
|
||||
Some(prompt_logprobs_to_maps(
|
||||
collected.prompt_logprobs.as_ref(),
|
||||
collected.prompt_token_ids.as_ref(),
|
||||
return_tokens_as_token_ids,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
|
||||
@@ -40,7 +40,7 @@ pub(super) struct ResponseOptions {
|
||||
/// Prompt text that should be echoed back northbound when `echo=true`.
|
||||
pub echo: Option<String>,
|
||||
/// Whether the caller requested output logprobs on completion choices.
|
||||
pub requested_logprobs: Option<u32>,
|
||||
pub requested_logprobs: Option<i32>,
|
||||
/// Whether the caller requested choice-level prompt logprobs.
|
||||
pub include_prompt_logprobs: bool,
|
||||
/// Whether to include token IDs alongside generated text.
|
||||
@@ -69,15 +69,7 @@ pub(super) fn prepare_completion_request(
|
||||
.map(|request| request.lora_name.clone())
|
||||
.unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default());
|
||||
|
||||
let logprobs = match request.logprobs {
|
||||
Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| {
|
||||
ApiError::invalid_request(
|
||||
"`logprobs` must fit within a signed 32-bit integer.".to_string(),
|
||||
Some("logprobs"),
|
||||
)
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
let logprobs = request.logprobs;
|
||||
let prompt_only = request.echo && request.max_tokens == Some(0);
|
||||
let prompt_logprobs =
|
||||
request.prompt_logprobs.or(if request.echo && (!request.stream || prompt_only) {
|
||||
@@ -207,6 +199,7 @@ fn completion_echo_text(
|
||||
mod tests {
|
||||
use axum::http::HeaderMap;
|
||||
use serde_json::json;
|
||||
use validator::Validate;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
@@ -245,7 +238,35 @@ mod tests {
|
||||
serde_json::from_value(base_request_json()).expect("parse request");
|
||||
|
||||
assert_eq!(request.prompt, Prompt::Text("hello".to_string()));
|
||||
assert_eq!(request.model, "Qwen/Qwen1.5-0.5B-Chat");
|
||||
assert_eq!(request.model.as_deref(), Some("Qwen/Qwen1.5-0.5B-Chat"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_http_request_defaults_missing_or_null_model() {
|
||||
for model in [None, Some(serde_json::Value::Null)] {
|
||||
let mut value = base_request_json();
|
||||
let object = value.as_object_mut().expect("request object");
|
||||
match model {
|
||||
Some(model) => {
|
||||
object.insert("model".to_string(), model);
|
||||
}
|
||||
None => {
|
||||
object.remove("model");
|
||||
}
|
||||
}
|
||||
|
||||
let request: CompletionRequest =
|
||||
serde_json::from_value(value).expect("parse request without model");
|
||||
assert!(request.model.is_none());
|
||||
}
|
||||
|
||||
let mut value = base_request_json();
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("request object")
|
||||
.insert("model".to_string(), json!(""));
|
||||
let request: CompletionRequest = serde_json::from_value(value).expect("parse empty model");
|
||||
assert_eq!(request.model.as_deref(), Some(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -264,6 +285,44 @@ mod tests {
|
||||
assert!(request.ignore_eos);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_http_request_rejects_empty_cache_salt() {
|
||||
let mut value = base_request_json();
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("request object")
|
||||
.insert("cache_salt".to_string(), json!(""));
|
||||
let request: CompletionRequest = serde_json::from_value(value).expect("parse cache_salt");
|
||||
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_normalizes_top_k_sentinels() {
|
||||
for (top_k, expected) in [
|
||||
(serde_json::Value::Null, None),
|
||||
(json!(-1), Some(0)),
|
||||
(json!(0), Some(0)),
|
||||
(json!(20), Some(20)),
|
||||
] {
|
||||
let mut value = base_request_json();
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("request object")
|
||||
.insert("top_k".to_string(), top_k);
|
||||
let request: CompletionRequest = serde_json::from_value(value).expect("parse top_k");
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare top_k");
|
||||
|
||||
assert_eq!(prepared.text_request.sampling_params.top_k, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_coerces_null_max_tokens_to_default() {
|
||||
// An absent `max_tokens` already gets the serde default.
|
||||
@@ -572,6 +631,28 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_accepts_full_vocab_logprobs() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"logprobs": -1
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.expect("prepare full-vocabulary logprobs");
|
||||
|
||||
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(-1));
|
||||
assert_eq!(prepared.options.requested_logprobs, Some(-1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_threads_data_parallel_rank() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -10,7 +10,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use crate::routes::openai::utils::types::{
|
||||
LogProbs, Normalizable, StreamOptions, StringOrArray, Usage, default_true, validate_stop,
|
||||
LogProbs, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Usage, default_true,
|
||||
deserialize_request_top_k, validate_stop,
|
||||
};
|
||||
|
||||
/// Serde default for `CompletionRequest::max_tokens`, matching the Python vLLM
|
||||
@@ -30,7 +31,7 @@ fn default_completion_max_tokens() -> Option<u32> {
|
||||
pub struct CompletionRequest {
|
||||
// -------- Standard OpenAI API Parameters --------
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
pub model: Option<String>,
|
||||
|
||||
/// The prompt(s) to generate completions for.
|
||||
///
|
||||
@@ -49,7 +50,7 @@ pub struct CompletionRequest {
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// Include the log probabilities on the logprobs most likely tokens
|
||||
pub logprobs: Option<u32>,
|
||||
pub logprobs: Option<i32>,
|
||||
|
||||
/// The maximum number of tokens to generate (defaults to 16 when absent,
|
||||
/// matching the Python vLLM / OpenAI API convention)
|
||||
@@ -96,6 +97,7 @@ pub struct CompletionRequest {
|
||||
pub use_beam_search: bool,
|
||||
|
||||
/// Top-k sampling parameter
|
||||
#[serde(default, deserialize_with = "deserialize_request_top_k")]
|
||||
pub top_k: Option<u32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
@@ -175,6 +177,7 @@ pub struct CompletionRequest {
|
||||
pub return_token_ids: Option<bool>,
|
||||
|
||||
/// Salt for prefix cache isolation in multi-user environments
|
||||
#[validate(length(min = 1))]
|
||||
pub cache_salt: Option<String>,
|
||||
|
||||
/// KV transfer parameters for disaggregated serving
|
||||
@@ -229,7 +232,7 @@ pub(super) struct CompletionChoice {
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub stop_reason: Option<Value>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<String, f32>>>>,
|
||||
pub prompt_logprobs: Option<PromptLogprobs>,
|
||||
pub token_ids: Option<Vec<u32>>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
}
|
||||
|
||||
@@ -12,8 +12,10 @@ pub(super) fn validate_request_compat(
|
||||
// This path is intentionally scoped to the minimum surface needed by
|
||||
// `vllm-bench` random workload compatibility, so unsupported legacy
|
||||
// completions features fail early here.
|
||||
if !served_model_names.iter().any(|n| n == &request.model) {
|
||||
return Err(ApiError::model_not_found(request.model.clone()));
|
||||
if let Some(model) = request.model.as_ref().filter(|model| !model.is_empty())
|
||||
&& !served_model_names.iter().any(|name| name == model)
|
||||
{
|
||||
return Err(ApiError::model_not_found(model.clone()));
|
||||
}
|
||||
|
||||
if request.stream_options.is_some() && !request.stream {
|
||||
@@ -39,11 +41,12 @@ pub(super) fn validate_request_compat(
|
||||
}
|
||||
|
||||
if let Some(logprobs) = request.logprobs
|
||||
&& logprobs > i32::MAX as u32
|
||||
&& logprobs < 0
|
||||
&& logprobs != -1
|
||||
{
|
||||
bail_invalid_request!(
|
||||
param = "logprobs",
|
||||
"`logprobs` must fit within a signed 32-bit integer."
|
||||
"`logprobs` must be a non-negative value or -1."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -123,6 +126,28 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_full_vocab_logprobs() {
|
||||
let request = CompletionRequest {
|
||||
logprobs: Some(-1),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_other_negative_logprobs() {
|
||||
let request = CompletionRequest {
|
||||
logprobs: Some(-2),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_any_served_name() {
|
||||
let request = base_request();
|
||||
@@ -135,6 +160,17 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_default_model_inputs() {
|
||||
for model in [None, Some(String::new())] {
|
||||
let request = CompletionRequest {
|
||||
model,
|
||||
..base_request()
|
||||
};
|
||||
assert!(validate_request_compat(&request, &served_names(&["served-model"])).is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_unknown_model() {
|
||||
let request = base_request();
|
||||
|
||||
@@ -9,7 +9,9 @@ use vllm_text::{
|
||||
DecodedTokenLogprob,
|
||||
};
|
||||
|
||||
use super::types::{ChatLogProbs, ChatLogProbsContent, LogProbs, TopLogProb};
|
||||
use super::types::{
|
||||
ChatLogProbs, ChatLogProbsContent, LogProbs, PromptLogprob, PromptLogprobs, TopLogProb,
|
||||
};
|
||||
use crate::error::{ApiError, server_error};
|
||||
|
||||
/// Convert decoded token-position logprobs into the OpenAI completions
|
||||
@@ -105,15 +107,26 @@ pub fn decoded_prompt_logprobs_to_openai(
|
||||
pub fn prompt_logprobs_to_maps(
|
||||
prompt_logprobs: Option<&DecodedPromptLogprobs>,
|
||||
prompt_token_ids: &[u32],
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<Vec<Option<HashMap<String, f32>>>, ApiError> {
|
||||
) -> Result<PromptLogprobs, ApiError> {
|
||||
if let Some(prompt_logprobs) = prompt_logprobs {
|
||||
return Ok(std::iter::once(None)
|
||||
.chain(prompt_logprobs.scored_positions.iter().map(|position| {
|
||||
Some(position_top_logprobs_map(
|
||||
position,
|
||||
return_tokens_as_token_ids,
|
||||
))
|
||||
Some(
|
||||
position
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
(
|
||||
entry.token_id,
|
||||
PromptLogprob {
|
||||
logprob: clamp_logprob(entry.logprob),
|
||||
rank: entry.rank,
|
||||
decoded_token: entry.token.clone(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}))
|
||||
.collect());
|
||||
}
|
||||
@@ -290,10 +303,32 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn prompt_logprobs_maps_reject_missing_multi_token_payload() {
|
||||
prompt_logprobs_to_maps(None, &[9707, 11], false)
|
||||
prompt_logprobs_to_maps(None, &[9707, 11])
|
||||
.expect_err("multi-token prompt without payload is an engine failure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_logprobs_maps_preserve_token_ids_and_metadata() {
|
||||
let prompt_logprobs = vllm_text::DecodedPromptLogprobs {
|
||||
first_token_id: 10,
|
||||
first_token: "first".to_string(),
|
||||
scored_positions: sample_logprobs().positions,
|
||||
};
|
||||
|
||||
let maps = prompt_logprobs_to_maps(Some(&prompt_logprobs), &[10, 1]).expect("maps");
|
||||
assert_eq!(
|
||||
serde_json::to_value(maps).expect("serialize prompt logprobs"),
|
||||
serde_json::json!([
|
||||
null,
|
||||
{
|
||||
"1": {"logprob": -0.10000000149011612_f64, "rank": 1, "decoded_token": "A"},
|
||||
"2": {"logprob": -1.0, "rank": 2, "decoded_token": "B"},
|
||||
"3": {"logprob": -2.0, "rank": 3, "decoded_token": "C"},
|
||||
}
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
fn sample_logprobs() -> DecodedLogprobs {
|
||||
DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::collections::HashMap;
|
||||
use std::slice;
|
||||
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_llm::TokenUsage;
|
||||
|
||||
@@ -13,9 +13,6 @@ use vllm_llm::TokenUsage;
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
/// Default model identifier used when no model is specified.
|
||||
pub const UNKNOWN_MODEL_ID: &str = "unknown";
|
||||
|
||||
// ============================================================================
|
||||
// Default value helpers
|
||||
// ============================================================================
|
||||
@@ -25,6 +22,23 @@ pub fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Deserialize an OpenAI request `top_k` while preserving explicit disable.
|
||||
///
|
||||
/// Null remains `None` so model generation defaults apply. Explicit `-1` and
|
||||
/// `0` become `Some(0)` so the request overrides those defaults and disables
|
||||
/// top-k sampling. Positive limits are preserved.
|
||||
pub fn deserialize_request_top_k<'de, D>(deserializer: D) -> Result<Option<u32>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
match Option::<i64>::deserialize(deserializer)? {
|
||||
None => Ok(None),
|
||||
Some(value) => vllm_text::normalize_top_k(value)
|
||||
.map(|value| Some(value.unwrap_or(0)))
|
||||
.map_err(serde::de::Error::custom),
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// String/Array Utilities
|
||||
// ============================================================================
|
||||
@@ -201,15 +215,21 @@ pub struct StreamOptions {
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Tool {
|
||||
#[serde(rename = "type")]
|
||||
#[serde(default = "default_function_tool_type")]
|
||||
pub tool_type: String,
|
||||
pub function: Function,
|
||||
}
|
||||
|
||||
fn default_function_tool_type() -> String {
|
||||
"function".to_string()
|
||||
}
|
||||
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Function {
|
||||
pub name: String,
|
||||
pub description: Option<String>,
|
||||
#[serde(default)]
|
||||
pub parameters: Value,
|
||||
/// Whether to enable strict schema adherence (OpenAI structured outputs).
|
||||
pub strict: Option<bool>,
|
||||
@@ -480,6 +500,16 @@ pub struct LogProbs {
|
||||
pub text_offset: Vec<u32>,
|
||||
}
|
||||
|
||||
/// vLLM prompt-logprob metadata keyed by vocabulary token ID.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize)]
|
||||
pub struct PromptLogprob {
|
||||
pub logprob: f32,
|
||||
pub rank: u32,
|
||||
pub decoded_token: String,
|
||||
}
|
||||
|
||||
pub type PromptLogprobs = Vec<Option<HashMap<u32, PromptLogprob>>>;
|
||||
|
||||
/// Mirrors the Python vLLM `ChatCompletionLogProbs` class.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ChatLogProbs {
|
||||
|
||||
@@ -68,6 +68,14 @@ fn model_resolution(state: &RenderState) -> LoraModelResolution {
|
||||
}
|
||||
}
|
||||
|
||||
fn response_model(state: &RenderState, requested_model: Option<&str>) -> String {
|
||||
requested_model
|
||||
.filter(|model| !model.is_empty())
|
||||
.or_else(|| state.served_model_names.first().map(String::as_str))
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn lower_render_request(
|
||||
state: &RenderState,
|
||||
text_request: TextRequest,
|
||||
@@ -108,7 +116,7 @@ async fn render_chat(
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<ChatCompletionRequest>,
|
||||
) -> Result<Json<GenerateRequest>, ApiError> {
|
||||
let model = body.model.clone();
|
||||
let model = response_model(&state, body.model.as_deref());
|
||||
let stream = body.stream;
|
||||
let stream_options = body.stream_options.clone();
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
@@ -132,7 +140,7 @@ async fn render_completion(
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<CompletionRequest>,
|
||||
) -> Result<Json<Vec<GenerateRequest>>, ApiError> {
|
||||
let model = body.model.clone();
|
||||
let model = response_model(&state, body.model.as_deref());
|
||||
let stream = body.stream;
|
||||
let stream_options = body.stream_options.clone();
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
|
||||
@@ -3654,7 +3654,10 @@ async fn non_stream_completions_include_prompt_logprobs() {
|
||||
);
|
||||
assert_eq!(
|
||||
json["choices"][0]["prompt_logprobs"][1],
|
||||
json!({"a": -0.5, "e": -0.3})
|
||||
json!({
|
||||
"97": {"logprob": -0.5, "rank": 1, "decoded_token": "a"},
|
||||
"101": {"logprob": -0.3, "rank": 1, "decoded_token": "e"},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::normalize_top_k;
|
||||
|
||||
/// Minimal subset of `tokenizer_config.json` needed by chat/EOS handling.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
@@ -118,23 +119,19 @@ pub(super) struct GenerationConfig {
|
||||
pub max_new_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// Deserialize vLLM-compatible `top_k` values from generation configs.
|
||||
/// Deserialize a generation-config `top_k` into the text sampling model.
|
||||
///
|
||||
/// Both `-1` and `0` disable top-k sampling and become `None`; positive values
|
||||
/// are preserved.
|
||||
/// Null, `-1`, and `0` all disable top-k sampling; positive limits are
|
||||
/// preserved.
|
||||
fn deserialize_top_k<'de, D>(deserializer: D) -> std::result::Result<Option<u32>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
match Option::<i64>::deserialize(deserializer)? {
|
||||
None | Some(-1) | Some(0) => Ok(None),
|
||||
Some(value) if value > 0 => {
|
||||
u32::try_from(value).map(Some).map_err(serde::de::Error::custom)
|
||||
}
|
||||
Some(value) => Err(serde::de::Error::custom(format!(
|
||||
"top_k must be -1, 0, or a positive integer, got {value}"
|
||||
))),
|
||||
}
|
||||
Option::<i64>::deserialize(deserializer)?
|
||||
.map(normalize_top_k)
|
||||
.transpose()
|
||||
.map(Option::flatten)
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
/// HF generation configs allow either one EOS id or a list of EOS ids.
|
||||
|
||||
@@ -21,7 +21,7 @@ pub use output::{
|
||||
CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs,
|
||||
DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt,
|
||||
};
|
||||
pub use request::{Prompt, SamplingParams, TextRequest};
|
||||
pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k};
|
||||
use trait_set::trait_set;
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
pub use vllm_llm::FinishReason;
|
||||
|
||||
@@ -434,6 +434,20 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_preserves_zero_min_tokens() {
|
||||
let params = lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
min_tokens: Some(0),
|
||||
..SamplingParams::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
.expect("lower zero min_tokens");
|
||||
|
||||
assert_eq!(params.min_tokens, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_validates_repetition_detection() {
|
||||
let lower = |repetition_detection| {
|
||||
|
||||
@@ -15,6 +15,24 @@ use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsPara
|
||||
use crate::error::{Error, Result};
|
||||
use crate::output::TextDecodeOptions;
|
||||
|
||||
/// Normalize vLLM's signed `top_k` domain into [`SamplingParams::top_k`].
|
||||
///
|
||||
/// vLLM uses both `-1` and `0` to disable top-k sampling. The text sampling
|
||||
/// model represents that state as `None` and preserves positive limits as
|
||||
/// `Some(k)`. Values below `-1` and values above `u32::MAX` are rejected.
|
||||
///
|
||||
/// Callers that need to distinguish an omitted value from an explicit disable
|
||||
/// should preserve that request-level distinction around this normalization.
|
||||
pub fn normalize_top_k(value: i64) -> std::result::Result<Option<u32>, String> {
|
||||
match value {
|
||||
-1 | 0 => Ok(None),
|
||||
value if value > 0 => u32::try_from(value).map(Some).map_err(|error| error.to_string()),
|
||||
value => Err(format!(
|
||||
"top_k must be -1, 0, or a positive integer, got {value}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// One raw text-generation prompt.
|
||||
///
|
||||
/// This supports either ordinary text that still needs tokenization or
|
||||
|
||||
Reference in New Issue
Block a user