diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 22f0d1bb4d..0485192191 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -63,7 +63,8 @@ pub async fn chat_completions( ) -> Response { let stream = body.stream; let request_context = resolve_request_context(&headers, body.request_id.as_deref()); - let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await; + let requested_model = body.model.as_deref().filter(|model| !model.is_empty()); + let lora_resolution = state.resolve_model_with_loras(requested_model).await; let prepared = match prepare_chat_request(body, &lora_resolution, request_context) { Ok(prepared) => prepared, @@ -196,7 +197,6 @@ async fn collect_chat_completion( Some(prompt_logprobs_to_maps( prompt_logprobs.as_ref(), &prompt_token_ids, - return_tokens_as_token_ids, )?) } else { None diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 2c563f27bc..233d881449 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -433,6 +433,7 @@ mod tests { use expect_test::expect; use llm_multimodal::ImageDetail; use serde_json::json; + use validator::Validate; use vllm_chat::{ AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage, ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode, @@ -466,7 +467,7 @@ mod tests { fn base_request() -> ChatCompletionRequest { ChatCompletionRequest { - model: "Qwen/Qwen1.5-0.5B-Chat".to_string(), + model: Some("Qwen/Qwen1.5-0.5B-Chat".to_string()), messages: vec![ChatMessage::User { content: MessageContent::Text("hello".to_string()), name: None, @@ -476,6 +477,120 @@ mod tests { } } + #[test] + fn chat_http_request_defaults_missing_or_null_model() { + for model in [None, Some(serde_json::Value::Null)] { + let mut value = json!({ + "messages": [{"role": "user", "content": "hello"}], + "stream": true, + }); + if let Some(model) = model { + value + .as_object_mut() + .expect("request object") + .insert("model".to_string(), model); + } + + let request: ChatCompletionRequest = + serde_json::from_value(value).expect("parse request without model"); + assert!(request.model.is_none()); + } + + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "messages": [{"role": "user", "content": "hello"}], + "model": "", + })) + .expect("parse empty model"); + assert_eq!(request.model.as_deref(), Some("")); + } + + #[test] + fn prepare_chat_request_normalizes_top_k_sentinels() { + for (top_k, expected) in [ + (serde_json::Value::Null, None), + (json!(-1), Some(0)), + (json!(0), Some(0)), + (json!(20), Some(20)), + ] { + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "messages": [{"role": "user", "content": "hello"}], + "stream": true, + "top_k": top_k, + })) + .expect("parse top_k"); + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare top_k"); + + assert_eq!(prepared.chat_request.sampling_params.top_k, expected); + } + } + + #[test] + fn prepare_chat_request_accepts_zero_min_tokens() { + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "messages": [{"role": "user", "content": "hello"}], + "stream": true, + "min_tokens": 0, + })) + .expect("parse zero min_tokens"); + request.validate().expect("validate zero min_tokens"); + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare zero min_tokens"); + + assert_eq!(prepared.chat_request.sampling_params.min_tokens, Some(0)); + assert_eq!(prepared.chat_request.decode_options.min_tokens, 0); + } + + #[test] + fn prepare_chat_request_defaults_function_tool_fields() { + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "messages": [{"role": "user", "content": "hello"}], + "tools": [{"function": {"name": "lookup"}}], + })) + .expect("parse tool defaults"); + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare tool defaults"); + + assert_eq!( + prepared.chat_request.tools(), + &[VllmChatTool { + name: "lookup".to_string(), + description: None, + parameters: serde_json::Value::Null, + strict: None, + }] + ); + } + + #[test] + fn chat_http_request_rejects_empty_cache_salt() { + let request: ChatCompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "messages": [{"role": "user", "content": "hello"}], + "cache_salt": "", + })) + .expect("parse cache_salt"); + + assert!(request.validate().is_err()); + } + #[test] fn prepare_chat_request_maps_parallel_tool_calls() { let mut request = base_request(); @@ -506,7 +621,7 @@ mod tests { #[test] fn prepare_chat_request_passes_response_format_to_kimi_k3_renderer() { let mut request = base_request(); - request.model = "moonshotai/Kimi-K3".to_string(); + request.model = Some("moonshotai/Kimi-K3".to_string()); let response_format = ResponseFormat::JsonSchema { json_schema: JsonSchemaFormat { name: "answer".to_string(), @@ -913,7 +1028,7 @@ mod tests { #[test] fn prepare_chat_request_accepts_audio_content_parts() { let request = ChatCompletionRequest { - model: "Qwen/Qwen3-ASR-1.7B".to_string(), + model: Some("Qwen/Qwen3-ASR-1.7B".to_string()), messages: vec![ChatMessage::User { content: MessageContent::Parts(vec![ ContentPart::InputAudio { diff --git a/rust/src/server/src/routes/openai/chat_completions/types.rs b/rust/src/server/src/routes/openai/chat_completions/types.rs index 27911bc3f3..5b4e733d63 100644 --- a/rust/src/server/src/routes/openai/chat_completions/types.rs +++ b/rust/src/server/src/routes/openai/chat_completions/types.rs @@ -13,9 +13,9 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams; use crate::routes::openai::utils::structured_outputs::ResponseFormat; use crate::routes::openai::utils::types::{ - ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall, - ToolCallDelta, ToolChoice, UNKNOWN_MODEL_ID, Usage, default_true, validate_messages, - validate_stop, validate_top_p_value, + ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Tool, + ToolCall, ToolCallDelta, ToolChoice, Usage, default_true, deserialize_request_top_k, + validate_messages, validate_stop, validate_top_p_value, }; /// vLLM-compatible request type for the Chat Completions API. @@ -33,8 +33,7 @@ pub struct ChatCompletionRequest { pub messages: Vec, /// ID of the model to use - #[serde(default = "default_model")] - pub model: String, + pub model: Option, /// Number between -2.0 and 2.0. Positive values penalize new tokens based /// on their existing frequency in the text so far @@ -120,6 +119,7 @@ pub struct ChatCompletionRequest { pub use_beam_search: bool, /// Top-k sampling parameter + #[serde(default, deserialize_with = "deserialize_request_top_k")] pub top_k: Option, /// Min-p nucleus sampling parameter @@ -145,7 +145,6 @@ pub struct ChatCompletionRequest { pub ignore_eos: bool, /// Minimum number of tokens to generate - #[validate(range(min = 1))] pub min_tokens: Option, /// Skip special tokens during detokenization @@ -233,6 +232,7 @@ pub struct ChatCompletionRequest { pub return_token_ids: Option, /// Salt for prefix cache isolation in multi-user environments + #[validate(length(min = 1))] pub cache_salt: Option, /// KV transfer parameters for disaggregated serving @@ -254,7 +254,7 @@ impl Default for ChatCompletionRequest { fn default() -> Self { Self { messages: Vec::new(), - model: default_model(), + model: None, frequency_penalty: None, logit_bias: None, logprobs: false, @@ -341,7 +341,7 @@ pub(super) struct ChatCompletionResponse { pub choices: Vec, pub usage: Option, pub system_fingerprint: Option, - pub prompt_logprobs: Option>>>, + pub prompt_logprobs: Option, pub prompt_token_ids: Option>, pub kv_transfer_params: Option, pub ec_transfer_params: Option, @@ -429,10 +429,6 @@ pub(super) struct ChatMessageDelta { pub reasoning: Option, } -fn default_model() -> String { - UNKNOWN_MODEL_ID.to_string() -} - /// Schema-level validation for cross-field dependencies fn validate_chat_cross_parameters( req: &ChatCompletionRequest, diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index 3869b969ac..b78ca46ee5 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -10,8 +10,10 @@ pub(super) fn validate_request_compat( request: &ChatCompletionRequest, served_model_names: &[String], ) -> Result<(), ApiError> { - if !served_model_names.iter().any(|n| n == &request.model) { - return Err(ApiError::model_not_found(request.model.clone())); + if let Some(model) = request.model.as_ref().filter(|model| !model.is_empty()) + && !served_model_names.iter().any(|name| name == model) + { + return Err(ApiError::model_not_found(model.clone())); } if request.stream_options.is_some() && !request.stream { @@ -142,7 +144,7 @@ mod tests { fn base_request() -> ChatCompletionRequest { ChatCompletionRequest { - model: "Qwen/Qwen1.5-0.5B-Chat".to_string(), + model: Some("Qwen/Qwen1.5-0.5B-Chat".to_string()), messages: vec![ChatMessage::User { content: MessageContent::Text("hello".to_string()), name: None, @@ -163,6 +165,18 @@ mod tests { .expect("stop strings should be accepted"); } + #[test] + fn validate_request_compat_accepts_default_model_inputs() { + for model in [None, Some(String::new())] { + let request = ChatCompletionRequest { + model, + ..base_request() + }; + validate_request_compat(&request, &served(&["served-model"])) + .expect("default model input should select the primary served model"); + } + } + #[test] fn validate_request_compat_accepts_non_zero_penalties_and_function_tools() { let request = ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index 44c4bf9ae0..d8943b69f3 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -66,7 +66,8 @@ pub async fn completions( ) -> Response { let stream = body.stream; let request_context = resolve_request_context(&headers, body.request_id.as_deref()); - let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await; + let requested_model = body.model.as_deref().filter(|model| !model.is_empty()); + let lora_resolution = state.resolve_model_with_loras(requested_model).await; let tokenizer = state.chat.text().tokenizer(); let prepared = match prepare_completion_request( @@ -186,7 +187,6 @@ async fn collect_completion( Some(prompt_logprobs_to_maps( collected.prompt_logprobs.as_ref(), collected.prompt_token_ids.as_ref(), - return_tokens_as_token_ids, )?) } else { None diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index c2651064eb..813e5c21b2 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -40,7 +40,7 @@ pub(super) struct ResponseOptions { /// Prompt text that should be echoed back northbound when `echo=true`. pub echo: Option, /// Whether the caller requested output logprobs on completion choices. - pub requested_logprobs: Option, + pub requested_logprobs: Option, /// Whether the caller requested choice-level prompt logprobs. pub include_prompt_logprobs: bool, /// Whether to include token IDs alongside generated text. @@ -69,15 +69,7 @@ pub(super) fn prepare_completion_request( .map(|request| request.lora_name.clone()) .unwrap_or_else(|| lora_resolution.model_names.first().cloned().unwrap_or_default()); - let logprobs = match request.logprobs { - Some(logprobs) => Some(i32::try_from(logprobs).map_err(|_| { - ApiError::invalid_request( - "`logprobs` must fit within a signed 32-bit integer.".to_string(), - Some("logprobs"), - ) - })?), - None => None, - }; + let logprobs = request.logprobs; let prompt_only = request.echo && request.max_tokens == Some(0); let prompt_logprobs = request.prompt_logprobs.or(if request.echo && (!request.stream || prompt_only) { @@ -207,6 +199,7 @@ fn completion_echo_text( mod tests { use axum::http::HeaderMap; use serde_json::json; + use validator::Validate; use vllm_text::Prompt; use vllm_tokenizer::test_utils::TestTokenizer; @@ -245,7 +238,35 @@ mod tests { serde_json::from_value(base_request_json()).expect("parse request"); assert_eq!(request.prompt, Prompt::Text("hello".to_string())); - assert_eq!(request.model, "Qwen/Qwen1.5-0.5B-Chat"); + assert_eq!(request.model.as_deref(), Some("Qwen/Qwen1.5-0.5B-Chat")); + } + + #[test] + fn completion_http_request_defaults_missing_or_null_model() { + for model in [None, Some(serde_json::Value::Null)] { + let mut value = base_request_json(); + let object = value.as_object_mut().expect("request object"); + match model { + Some(model) => { + object.insert("model".to_string(), model); + } + None => { + object.remove("model"); + } + } + + let request: CompletionRequest = + serde_json::from_value(value).expect("parse request without model"); + assert!(request.model.is_none()); + } + + let mut value = base_request_json(); + value + .as_object_mut() + .expect("request object") + .insert("model".to_string(), json!("")); + let request: CompletionRequest = serde_json::from_value(value).expect("parse empty model"); + assert_eq!(request.model.as_deref(), Some("")); } #[test] @@ -264,6 +285,44 @@ mod tests { assert!(request.ignore_eos); } + #[test] + fn completion_http_request_rejects_empty_cache_salt() { + let mut value = base_request_json(); + value + .as_object_mut() + .expect("request object") + .insert("cache_salt".to_string(), json!("")); + let request: CompletionRequest = serde_json::from_value(value).expect("parse cache_salt"); + + assert!(request.validate().is_err()); + } + + #[test] + fn prepare_completion_request_normalizes_top_k_sentinels() { + for (top_k, expected) in [ + (serde_json::Value::Null, None), + (json!(-1), Some(0)), + (json!(0), Some(0)), + (json!(20), Some(20)), + ] { + let mut value = base_request_json(); + value + .as_object_mut() + .expect("request object") + .insert("top_k".to_string(), top_k); + let request: CompletionRequest = serde_json::from_value(value).expect("parse top_k"); + let prepared = prepare_completion_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + &test_tokenizer(), + ) + .expect("prepare top_k"); + + assert_eq!(prepared.text_request.sampling_params.top_k, expected); + } + } + #[test] fn normalize_coerces_null_max_tokens_to_default() { // An absent `max_tokens` already gets the serde default. @@ -572,6 +631,28 @@ mod tests { ); } + #[test] + fn prepare_completion_request_accepts_full_vocab_logprobs() { + let request: CompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "stream": false, + "logprobs": -1 + })) + .expect("parse request"); + + let prepared = prepare_completion_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + &test_tokenizer(), + ) + .expect("prepare full-vocabulary logprobs"); + + assert_eq!(prepared.text_request.sampling_params.logprobs, Some(-1)); + assert_eq!(prepared.options.requested_logprobs, Some(-1)); + } + #[test] fn prepare_completion_request_threads_data_parallel_rank() { let request: CompletionRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/openai/completions/types.rs b/rust/src/server/src/routes/openai/completions/types.rs index 504c3e3b2c..b83cf06717 100644 --- a/rust/src/server/src/routes/openai/completions/types.rs +++ b/rust/src/server/src/routes/openai/completions/types.rs @@ -10,7 +10,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams; use vllm_text::Prompt; use crate::routes::openai::utils::types::{ - LogProbs, Normalizable, StreamOptions, StringOrArray, Usage, default_true, validate_stop, + LogProbs, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Usage, default_true, + deserialize_request_top_k, validate_stop, }; /// Serde default for `CompletionRequest::max_tokens`, matching the Python vLLM @@ -30,7 +31,7 @@ fn default_completion_max_tokens() -> Option { pub struct CompletionRequest { // -------- Standard OpenAI API Parameters -------- /// ID of the model to use - pub model: String, + pub model: Option, /// The prompt(s) to generate completions for. /// @@ -49,7 +50,7 @@ pub struct CompletionRequest { pub logit_bias: Option>, /// Include the log probabilities on the logprobs most likely tokens - pub logprobs: Option, + pub logprobs: Option, /// The maximum number of tokens to generate (defaults to 16 when absent, /// matching the Python vLLM / OpenAI API convention) @@ -96,6 +97,7 @@ pub struct CompletionRequest { pub use_beam_search: bool, /// Top-k sampling parameter + #[serde(default, deserialize_with = "deserialize_request_top_k")] pub top_k: Option, /// Min-p nucleus sampling parameter @@ -175,6 +177,7 @@ pub struct CompletionRequest { pub return_token_ids: Option, /// Salt for prefix cache isolation in multi-user environments + #[validate(length(min = 1))] pub cache_salt: Option, /// KV transfer parameters for disaggregated serving @@ -229,7 +232,7 @@ pub(super) struct CompletionChoice { pub logprobs: Option, pub finish_reason: Option, pub stop_reason: Option, - pub prompt_logprobs: Option>>>, + pub prompt_logprobs: Option, pub token_ids: Option>, pub prompt_token_ids: Option>, } diff --git a/rust/src/server/src/routes/openai/completions/validate.rs b/rust/src/server/src/routes/openai/completions/validate.rs index 2261b05114..76fe6a8eab 100644 --- a/rust/src/server/src/routes/openai/completions/validate.rs +++ b/rust/src/server/src/routes/openai/completions/validate.rs @@ -12,8 +12,10 @@ pub(super) fn validate_request_compat( // This path is intentionally scoped to the minimum surface needed by // `vllm-bench` random workload compatibility, so unsupported legacy // completions features fail early here. - if !served_model_names.iter().any(|n| n == &request.model) { - return Err(ApiError::model_not_found(request.model.clone())); + if let Some(model) = request.model.as_ref().filter(|model| !model.is_empty()) + && !served_model_names.iter().any(|name| name == model) + { + return Err(ApiError::model_not_found(model.clone())); } if request.stream_options.is_some() && !request.stream { @@ -39,11 +41,12 @@ pub(super) fn validate_request_compat( } if let Some(logprobs) = request.logprobs - && logprobs > i32::MAX as u32 + && logprobs < 0 + && logprobs != -1 { bail_invalid_request!( param = "logprobs", - "`logprobs` must fit within a signed 32-bit integer." + "`logprobs` must be a non-negative value or -1." ); } @@ -123,6 +126,28 @@ mod tests { ); } + #[test] + fn validate_request_compat_accepts_full_vocab_logprobs() { + let request = CompletionRequest { + logprobs: Some(-1), + ..base_request() + }; + assert!( + validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok() + ); + } + + #[test] + fn validate_request_compat_rejects_other_negative_logprobs() { + let request = CompletionRequest { + logprobs: Some(-2), + ..base_request() + }; + assert!( + validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err() + ); + } + #[test] fn validate_request_compat_accepts_any_served_name() { let request = base_request(); @@ -135,6 +160,17 @@ mod tests { ); } + #[test] + fn validate_request_compat_accepts_default_model_inputs() { + for model in [None, Some(String::new())] { + let request = CompletionRequest { + model, + ..base_request() + }; + assert!(validate_request_compat(&request, &served_names(&["served-model"])).is_ok()); + } + } + #[test] fn validate_request_compat_rejects_unknown_model() { let request = base_request(); diff --git a/rust/src/server/src/routes/openai/utils/logprobs.rs b/rust/src/server/src/routes/openai/utils/logprobs.rs index e300e124b1..66c7a8fad9 100644 --- a/rust/src/server/src/routes/openai/utils/logprobs.rs +++ b/rust/src/server/src/routes/openai/utils/logprobs.rs @@ -9,7 +9,9 @@ use vllm_text::{ DecodedTokenLogprob, }; -use super::types::{ChatLogProbs, ChatLogProbsContent, LogProbs, TopLogProb}; +use super::types::{ + ChatLogProbs, ChatLogProbsContent, LogProbs, PromptLogprob, PromptLogprobs, TopLogProb, +}; use crate::error::{ApiError, server_error}; /// Convert decoded token-position logprobs into the OpenAI completions @@ -105,15 +107,26 @@ pub fn decoded_prompt_logprobs_to_openai( pub fn prompt_logprobs_to_maps( prompt_logprobs: Option<&DecodedPromptLogprobs>, prompt_token_ids: &[u32], - return_tokens_as_token_ids: bool, -) -> Result>>, ApiError> { +) -> Result { if let Some(prompt_logprobs) = prompt_logprobs { return Ok(std::iter::once(None) .chain(prompt_logprobs.scored_positions.iter().map(|position| { - Some(position_top_logprobs_map( - position, - return_tokens_as_token_ids, - )) + Some( + position + .entries + .iter() + .map(|entry| { + ( + entry.token_id, + PromptLogprob { + logprob: clamp_logprob(entry.logprob), + rank: entry.rank, + decoded_token: entry.token.clone(), + }, + ) + }) + .collect(), + ) })) .collect()); } @@ -290,10 +303,32 @@ mod tests { #[test] fn prompt_logprobs_maps_reject_missing_multi_token_payload() { - prompt_logprobs_to_maps(None, &[9707, 11], false) + prompt_logprobs_to_maps(None, &[9707, 11]) .expect_err("multi-token prompt without payload is an engine failure"); } + #[test] + fn prompt_logprobs_maps_preserve_token_ids_and_metadata() { + let prompt_logprobs = vllm_text::DecodedPromptLogprobs { + first_token_id: 10, + first_token: "first".to_string(), + scored_positions: sample_logprobs().positions, + }; + + let maps = prompt_logprobs_to_maps(Some(&prompt_logprobs), &[10, 1]).expect("maps"); + assert_eq!( + serde_json::to_value(maps).expect("serialize prompt logprobs"), + serde_json::json!([ + null, + { + "1": {"logprob": -0.10000000149011612_f64, "rank": 1, "decoded_token": "A"}, + "2": {"logprob": -1.0, "rank": 2, "decoded_token": "B"}, + "3": {"logprob": -2.0, "rank": 3, "decoded_token": "C"}, + } + ]) + ); + } + fn sample_logprobs() -> DecodedLogprobs { DecodedLogprobs { positions: vec![DecodedPositionLogprobs { diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index d87dcd676d..b079bf4857 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use std::slice; use llm_multimodal::ImageDetail; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; use vllm_llm::TokenUsage; @@ -13,9 +13,6 @@ use vllm_llm::TokenUsage; // Constants // ============================================================================ -/// Default model identifier used when no model is specified. -pub const UNKNOWN_MODEL_ID: &str = "unknown"; - // ============================================================================ // Default value helpers // ============================================================================ @@ -25,6 +22,23 @@ pub fn default_true() -> bool { true } +/// Deserialize an OpenAI request `top_k` while preserving explicit disable. +/// +/// Null remains `None` so model generation defaults apply. Explicit `-1` and +/// `0` become `Some(0)` so the request overrides those defaults and disables +/// top-k sampling. Positive limits are preserved. +pub fn deserialize_request_top_k<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + match Option::::deserialize(deserializer)? { + None => Ok(None), + Some(value) => vllm_text::normalize_top_k(value) + .map(|value| Some(value.unwrap_or(0))) + .map_err(serde::de::Error::custom), + } +} + // ============================================================================ // String/Array Utilities // ============================================================================ @@ -201,15 +215,21 @@ pub struct StreamOptions { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Tool { #[serde(rename = "type")] + #[serde(default = "default_function_tool_type")] pub tool_type: String, pub function: Function, } +fn default_function_tool_type() -> String { + "function".to_string() +} + #[serde_with::skip_serializing_none] #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Function { pub name: String, pub description: Option, + #[serde(default)] pub parameters: Value, /// Whether to enable strict schema adherence (OpenAI structured outputs). pub strict: Option, @@ -480,6 +500,16 @@ pub struct LogProbs { pub text_offset: Vec, } +/// vLLM prompt-logprob metadata keyed by vocabulary token ID. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PromptLogprob { + pub logprob: f32, + pub rank: u32, + pub decoded_token: String, +} + +pub type PromptLogprobs = Vec>>; + /// Mirrors the Python vLLM `ChatCompletionLogProbs` class. #[derive(Debug, Clone, Serialize)] pub struct ChatLogProbs { diff --git a/rust/src/server/src/routes/render.rs b/rust/src/server/src/routes/render.rs index b2d7601ddc..8833a2d94e 100644 --- a/rust/src/server/src/routes/render.rs +++ b/rust/src/server/src/routes/render.rs @@ -68,6 +68,14 @@ fn model_resolution(state: &RenderState) -> LoraModelResolution { } } +fn response_model(state: &RenderState, requested_model: Option<&str>) -> String { + requested_model + .filter(|model| !model.is_empty()) + .or_else(|| state.served_model_names.first().map(String::as_str)) + .unwrap_or_default() + .to_string() +} + fn lower_render_request( state: &RenderState, text_request: TextRequest, @@ -108,7 +116,7 @@ async fn render_chat( headers: HeaderMap, ValidatedJson(body): ValidatedJson, ) -> Result, ApiError> { - let model = body.model.clone(); + let model = response_model(&state, body.model.as_deref()); let stream = body.stream; let stream_options = body.stream_options.clone(); let request_context = resolve_request_context(&headers, body.request_id.as_deref()); @@ -132,7 +140,7 @@ async fn render_completion( headers: HeaderMap, ValidatedJson(body): ValidatedJson, ) -> Result>, ApiError> { - let model = body.model.clone(); + let model = response_model(&state, body.model.as_deref()); let stream = body.stream; let stream_options = body.stream_options.clone(); let request_context = resolve_request_context(&headers, body.request_id.as_deref()); diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index fb10e7cb47..95bae1939e 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -3654,7 +3654,10 @@ async fn non_stream_completions_include_prompt_logprobs() { ); assert_eq!( json["choices"][0]["prompt_logprobs"][1], - json!({"a": -0.5, "e": -0.3}) + json!({ + "97": {"logprob": -0.5, "rank": 1, "decoded_token": "a"}, + "101": {"logprob": -0.3, "rank": 1, "decoded_token": "e"}, + }) ); } diff --git a/rust/src/text/src/backend/hf/config.rs b/rust/src/text/src/backend/hf/config.rs index 7f393014b7..849874bede 100644 --- a/rust/src/text/src/backend/hf/config.rs +++ b/rust/src/text/src/backend/hf/config.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use thiserror_ext::AsReport as _; use crate::error::{Error, Result}; +use crate::normalize_top_k; /// Minimal subset of `tokenizer_config.json` needed by chat/EOS handling. #[derive(Debug, Default, Deserialize)] @@ -118,23 +119,19 @@ pub(super) struct GenerationConfig { pub max_new_tokens: Option, } -/// Deserialize vLLM-compatible `top_k` values from generation configs. +/// Deserialize a generation-config `top_k` into the text sampling model. /// -/// Both `-1` and `0` disable top-k sampling and become `None`; positive values -/// are preserved. +/// Null, `-1`, and `0` all disable top-k sampling; positive limits are +/// preserved. fn deserialize_top_k<'de, D>(deserializer: D) -> std::result::Result, D::Error> where D: serde::Deserializer<'de>, { - match Option::::deserialize(deserializer)? { - None | Some(-1) | Some(0) => Ok(None), - Some(value) if value > 0 => { - u32::try_from(value).map(Some).map_err(serde::de::Error::custom) - } - Some(value) => Err(serde::de::Error::custom(format!( - "top_k must be -1, 0, or a positive integer, got {value}" - ))), - } + Option::::deserialize(deserializer)? + .map(normalize_top_k) + .transpose() + .map(Option::flatten) + .map_err(serde::de::Error::custom) } /// HF generation configs allow either one EOS id or a list of EOS ids. diff --git a/rust/src/text/src/lib.rs b/rust/src/text/src/lib.rs index f49038b27f..f337009735 100644 --- a/rust/src/text/src/lib.rs +++ b/rust/src/text/src/lib.rs @@ -21,7 +21,7 @@ pub use output::{ CollectedTextOutput, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent, DecodedTokenLogprob, Finished, TextDecodeOptions, TextOutputStreamExt, }; -pub use request::{Prompt, SamplingParams, TextRequest}; +pub use request::{Prompt, SamplingParams, TextRequest, normalize_top_k}; use trait_set::trait_set; use vllm_engine_core_client::EngineCoreClient; pub use vllm_llm::FinishReason; diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 0e1fbd26bc..8e516921e9 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -434,6 +434,20 @@ mod tests { )); } + #[test] + fn lower_sampling_params_preserves_zero_min_tokens() { + let params = lower_sampling_params_with_limits( + SamplingParams { + min_tokens: Some(0), + ..SamplingParams::default() + }, + sample_sampling_limits(), + ) + .expect("lower zero min_tokens"); + + assert_eq!(params.min_tokens, 0); + } + #[test] fn lower_sampling_params_validates_repetition_detection() { let lower = |repetition_detection| { diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index dfe7dc742c..4e429201c5 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -15,6 +15,24 @@ use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsPara use crate::error::{Error, Result}; use crate::output::TextDecodeOptions; +/// Normalize vLLM's signed `top_k` domain into [`SamplingParams::top_k`]. +/// +/// vLLM uses both `-1` and `0` to disable top-k sampling. The text sampling +/// model represents that state as `None` and preserves positive limits as +/// `Some(k)`. Values below `-1` and values above `u32::MAX` are rejected. +/// +/// Callers that need to distinguish an omitted value from an explicit disable +/// should preserve that request-level distinction around this normalization. +pub fn normalize_top_k(value: i64) -> std::result::Result, String> { + match value { + -1 | 0 => Ok(None), + value if value > 0 => u32::try_from(value).map(Some).map_err(|error| error.to_string()), + value => Err(format!( + "top_k must be -1, 0, or a positive integer, got {value}" + )), + } +} + /// One raw text-generation prompt. /// /// This supports either ordinary text that still needs tokenization or