diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index de21dc3a1c..95fb4db9a6 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -2,6 +2,7 @@ mod convert; mod types; mod validate; +use std::collections::HashMap; use std::convert::Infallible; use std::result::Result; use std::sync::Arc; @@ -16,12 +17,15 @@ use futures::{Stream, StreamExt as _, pin_mut}; use thiserror_ext::AsReport as _; use tracing::{debug, error, info, trace}; use tracing_futures::Instrument as _; -use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _}; +use vllm_text::{ + DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream, + TextOutputStreamExt as _, +}; use self::convert::{ResponseOptions, prepare_completion_request}; use super::utils::logprobs::{ collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps, - text_len, + decoded_prompt_logprobs_to_openai, text_len, }; use super::utils::types::Usage; use crate::config::ApiServerOptions; @@ -119,6 +123,7 @@ async fn collect_completion( include_usage: _, // Ignored: non-streaming responses are collected before usage is attached. include_continuous_usage: _, + prompt_only, echo, requested_logprobs, include_prompt_logprobs, @@ -136,17 +141,17 @@ async fn collect_completion( .map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON")); let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default(); - let prompt_logprobs = if include_prompt_logprobs { - let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| { - server_error!( - "completion response requested prompt_logprobs but generation returned none" - ) + let logprobs = if requested_logprobs.is_some() && prompt_only { + let prompt = echo.as_deref().ok_or_else(|| { + server_error!("prompt-only completion response missing echoed prompt") })?; - Some(prompt_logprobs) - } else { - None - }; - let logprobs = if requested_logprobs.is_some() { + Some(prompt_only_logprobs_to_openai( + collected.prompt_logprobs.as_ref(), + prompt, + collected.prompt_token_ids.as_ref(), + return_tokens_as_token_ids, + )?) + } else if requested_logprobs.is_some() { Some(collected_logprobs_to_openai( &collected, echo.is_some(), @@ -156,10 +161,18 @@ async fn collect_completion( } else { None }; - let prompt_logprobs = - prompt_logprobs.map(|lp| decoded_prompt_logprobs_to_maps(lp, return_tokens_as_token_ids)); + let prompt_logprobs = if include_prompt_logprobs { + Some(prompt_logprobs_to_maps( + collected.prompt_logprobs.as_ref(), + collected.prompt_token_ids.as_ref(), + return_tokens_as_token_ids, + )?) + } else { + None + }; let text = match &echo { None => collected.text, + Some(prompt) if prompt_only => prompt.clone(), Some(prompt) => format!("{prompt}{}", collected.text), }; let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string(); @@ -211,6 +224,7 @@ async fn completion_chunk_stream( ResponseOptions { include_usage, include_continuous_usage, + prompt_only, echo, requested_logprobs, // Ignored: streaming prompt logprobs are rejected for Python parity. @@ -239,14 +253,30 @@ async fn completion_chunk_stream( while let Some(next) = stream.next().await { match next { Ok(DecodedTextEvent::Start { - prompt_token_ids, .. + prompt_token_ids, + prompt_logprobs, }) => { debug!("completion stream started"); continuous_usage.set_prompt_tokens(prompt_token_ids.len()); if let Some(prompt) = echo.as_ref() { visible_text_len = text_len(prompt); - let mut chunk = - delta_chunk(&request_id, &response_model, created, prompt.clone(), None); + let logprobs = if prompt_only && requested_logprobs.is_some() { + Some(prompt_only_logprobs_to_openai( + prompt_logprobs.as_ref(), + prompt, + prompt_token_ids.as_ref(), + return_tokens_as_token_ids, + )?) + } else { + None + }; + let mut chunk = delta_chunk( + &request_id, + &response_model, + created, + prompt.clone(), + logprobs, + ); if return_token_ids && first_chunk { if let Some(choice) = chunk.choices.first_mut() { choice.prompt_token_ids = Some(prompt_token_ids.to_vec()); @@ -271,6 +301,48 @@ async fn completion_chunk_stream( logprobs, finished, }) => { + // Prompt-only streaming already emitted the echoed prompt in the Start chunk. + // The one generated token is only used to drive the engine to a finished event, + // so hide its delta and forward only the terminal finish/usage metadata. + if prompt_only { + if let Some(finished) = finished { + if enable_log_requests { + info!( + stream = true, + model = %response_model, + prompt_tokens = finished.usage.prompt_token_count, + output_tokens = finished.usage.output_token_count, + finish_reason = finished.finish_reason.as_str(), + "completion finished" + ); + } + continuous_usage.set_final_counts( + finished.usage.prompt_token_count, + finished.usage.output_token_count, + ); + let final_chunk = final_chunk( + &request_id, + &response_model, + created, + finished.finish_reason, + )?; + yield_chunk!(final_chunk); + + if include_usage { + y.yield_ok(CompletionSseChunk::Usage(usage_chunk( + &request_id, + &response_model, + created, + Usage::from_token_usage( + finished.usage, + enable_prompt_tokens_details, + ), + ))) + .await; + } + } + continue; + } let delta_text_len = text_len(&delta); let logprobs = if requested_logprobs.is_some() { let decoded_logprobs = logprobs.as_ref().ok_or_else(|| { @@ -386,6 +458,57 @@ fn completion_finish_reason_to_openai( } } +fn prompt_only_logprobs_to_openai( + prompt_logprobs: Option<&DecodedPromptLogprobs>, + prompt: &str, + prompt_token_ids: &[u32], + return_tokens_as_token_ids: bool, +) -> Result { + if let Some(prompt_logprobs) = prompt_logprobs { + return decoded_prompt_logprobs_to_openai(prompt_logprobs, 0, return_tokens_as_token_ids); + } + + if let [token_id] = prompt_token_ids { + let token = if return_tokens_as_token_ids { + format!("token_id:{token_id}") + } else { + prompt.to_string() + }; + + return Ok(LogProbs { + tokens: vec![token], + token_logprobs: vec![None], + top_logprobs: vec![None], + text_offset: vec![0], + }); + } + + Err(server_error!( + "prompt-only completion requested logprobs but generation returned none" + )) +} + +fn prompt_logprobs_to_maps( + prompt_logprobs: Option<&DecodedPromptLogprobs>, + prompt_token_ids: &[u32], + return_tokens_as_token_ids: bool, +) -> Result>>, ApiError> { + if let Some(prompt_logprobs) = prompt_logprobs { + return Ok(decoded_prompt_logprobs_to_maps( + prompt_logprobs, + return_tokens_as_token_ids, + )); + } + + if let [_token_id] = prompt_token_ids { + return Ok(vec![None]); + } + + Err(server_error!( + "completion response requested prompt_logprobs but generation returned none" + )) +} + fn usage_chunk( request_id: &str, response_model: &str, @@ -449,8 +572,8 @@ mod tests { use futures::{StreamExt as _, stream}; use itertools::Itertools as _; use vllm_text::{ - DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob, - FinishReason, Finished, + DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent, + DecodedTokenLogprob, FinishReason, Finished, }; use super::{ @@ -613,4 +736,314 @@ mod tests { CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"), } } + + #[tokio::test] + async fn collect_completion_hides_internal_prompt_only_token() { + let stream = stream::iter(vec![ + Ok(DecodedTextEvent::Start { + prompt_token_ids: vec![1, 2].into(), + prompt_logprobs: None, + }), + Ok(DecodedTextEvent::TextDelta { + delta: " leaked".to_string(), + token_ids: vec![3], + logprobs: None, + finished: Some(Finished { + usage: vllm_llm::TokenUsage { + prompt_token_count: 2, + output_token_count: 1, + cached_token_count: 0, + }, + finish_reason: FinishReason::Length, + kv_transfer_params: None, + }), + }), + ]); + + let response = super::collect_completion( + stream, + "cmpl-1".to_string(), + "model".to_string(), + 1, + ApiServerOptions::default(), + ResponseOptions { + prompt_only: true, + echo: Some("hello".to_string()), + return_token_ids: true, + ..Default::default() + }, + ) + .await + .expect("collect completion"); + + assert_eq!(response.choices[0].text, "hello"); + assert_eq!(response.choices[0].token_ids.as_deref(), Some(&[3][..])); + assert_eq!( + response.choices[0].prompt_token_ids.as_deref(), + Some(&[1, 2][..]) + ); + let usage = response.usage.expect("usage"); + assert_eq!(usage.prompt_tokens, 2); + assert_eq!(usage.completion_tokens, Some(1)); + assert_eq!(usage.total_tokens, 3); + } + + #[tokio::test] + async fn collect_completion_maps_prompt_logprobs_for_single_token_prompt() { + let stream = stream::iter(vec![ + Ok(DecodedTextEvent::Start { + prompt_token_ids: vec![9707].into(), + prompt_logprobs: None, + }), + Ok(DecodedTextEvent::TextDelta { + delta: " leaked".to_string(), + token_ids: vec![3], + logprobs: None, + finished: Some(Finished { + usage: vllm_llm::TokenUsage { + prompt_token_count: 1, + output_token_count: 1, + cached_token_count: 0, + }, + finish_reason: FinishReason::Length, + kv_transfer_params: None, + }), + }), + ]); + + let response = super::collect_completion( + stream, + "cmpl-1".to_string(), + "model".to_string(), + 1, + ApiServerOptions::default(), + ResponseOptions { + prompt_only: true, + echo: Some("Hello".to_string()), + requested_logprobs: Some(1), + include_prompt_logprobs: true, + ..Default::default() + }, + ) + .await + .expect("collect completion"); + + let choice = &response.choices[0]; + assert_eq!(choice.text, "Hello"); + assert_eq!(choice.prompt_logprobs, Some(vec![None])); + let logprobs = choice.logprobs.as_ref().expect("logprobs"); + assert_eq!(logprobs.tokens, vec!["Hello".to_string()]); + assert_eq!(logprobs.token_logprobs, vec![None]); + assert_eq!(logprobs.top_logprobs, vec![None]); + assert_eq!(logprobs.text_offset, vec![0]); + let usage = response.usage.expect("usage"); + assert_eq!(usage.prompt_tokens, 1); + assert_eq!(usage.completion_tokens, Some(1)); + assert_eq!(usage.total_tokens, 2); + } + + #[tokio::test] + async fn completion_chunk_stream_hides_internal_prompt_only_token() { + let stream = stream::iter(vec![ + Ok(DecodedTextEvent::Start { + prompt_token_ids: vec![1, 2].into(), + prompt_logprobs: None, + }), + Ok(DecodedTextEvent::TextDelta { + delta: " leaked".to_string(), + token_ids: vec![3], + logprobs: None, + finished: Some(Finished { + usage: vllm_llm::TokenUsage { + prompt_token_count: 2, + output_token_count: 1, + cached_token_count: 0, + }, + finish_reason: FinishReason::Length, + kv_transfer_params: None, + }), + }), + ]); + + let chunks = completion_chunk_stream( + stream, + "cmpl-1".to_string(), + "model".to_string(), + 1, + ApiServerOptions::default(), + ResponseOptions { + include_usage: true, + prompt_only: true, + echo: Some("hello".to_string()), + return_token_ids: true, + ..Default::default() + }, + ) + .collect::>() + .await; + + let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed"); + assert_eq!(chunks.len(), 3); + + match &chunks[0] { + CompletionSseChunk::Chunk(chunk) => { + assert_eq!(chunk.choices[0].text, "hello"); + assert_eq!( + chunk.choices[0].prompt_token_ids.as_deref(), + Some(&[1, 2][..]) + ); + } + CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"), + } + match &chunks[1] { + CompletionSseChunk::Chunk(chunk) => { + assert_eq!(chunk.choices[0].text, ""); + assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length")); + } + CompletionSseChunk::Usage(_) => panic!("expected final chunk"), + } + match &chunks[2] { + CompletionSseChunk::Usage(chunk) => { + let usage = chunk.usage.as_ref().expect("usage"); + assert_eq!(usage.prompt_tokens, 2); + assert_eq!(usage.completion_tokens, Some(1)); + assert_eq!(usage.total_tokens, 3); + } + CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"), + } + } + + #[tokio::test] + async fn completion_chunk_stream_maps_prompt_logprobs_for_single_token_prompt() { + let stream = stream::iter(vec![ + Ok(DecodedTextEvent::Start { + prompt_token_ids: vec![9707].into(), + prompt_logprobs: None, + }), + Ok(DecodedTextEvent::TextDelta { + delta: " leaked".to_string(), + token_ids: vec![3], + logprobs: None, + finished: Some(Finished { + usage: vllm_llm::TokenUsage { + prompt_token_count: 1, + output_token_count: 1, + cached_token_count: 0, + }, + finish_reason: FinishReason::Length, + kv_transfer_params: None, + }), + }), + ]); + + let chunks = completion_chunk_stream( + stream, + "cmpl-1".to_string(), + "model".to_string(), + 1, + ApiServerOptions::default(), + ResponseOptions { + prompt_only: true, + echo: Some("Hello".to_string()), + requested_logprobs: Some(1), + ..Default::default() + }, + ) + .collect::>() + .await; + + let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed"); + assert_eq!(chunks.len(), 2); + + match &chunks[0] { + CompletionSseChunk::Chunk(chunk) => { + assert_eq!(chunk.choices[0].text, "Hello"); + let logprobs = chunk.choices[0].logprobs.as_ref().expect("logprobs"); + assert_eq!(logprobs.tokens, vec!["Hello".to_string()]); + assert_eq!(logprobs.token_logprobs, vec![None]); + assert_eq!(logprobs.top_logprobs, vec![None]); + assert_eq!(logprobs.text_offset, vec![0]); + } + CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"), + } + match &chunks[1] { + CompletionSseChunk::Chunk(chunk) => { + assert_eq!(chunk.choices[0].text, ""); + assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length")); + } + CompletionSseChunk::Usage(_) => panic!("expected final chunk"), + } + } + + #[tokio::test] + async fn completion_chunk_stream_maps_prompt_only_logprobs() { + let stream = stream::iter(vec![ + Ok(DecodedTextEvent::Start { + prompt_token_ids: vec![1, 2].into(), + prompt_logprobs: Some(DecodedPromptLogprobs { + first_token_id: 1, + first_token: "he".to_string(), + scored_positions: vec![DecodedPositionLogprobs { + entries: vec![DecodedTokenLogprob { + token_id: 2, + token: "llo".to_string(), + logprob: -0.2, + rank: 1, + }], + }], + }), + }), + Ok(DecodedTextEvent::TextDelta { + delta: " leaked".to_string(), + token_ids: vec![3], + logprobs: None, + finished: Some(Finished { + usage: vllm_llm::TokenUsage { + prompt_token_count: 2, + output_token_count: 1, + cached_token_count: 0, + }, + finish_reason: FinishReason::Length, + kv_transfer_params: None, + }), + }), + ]); + + let chunks = completion_chunk_stream( + stream, + "cmpl-1".to_string(), + "model".to_string(), + 1, + ApiServerOptions::default(), + ResponseOptions { + prompt_only: true, + echo: Some("hello".to_string()), + requested_logprobs: Some(1), + ..Default::default() + }, + ) + .collect::>() + .await; + + let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed"); + assert_eq!(chunks.len(), 2); + + match &chunks[0] { + CompletionSseChunk::Chunk(chunk) => { + assert_eq!(chunk.choices[0].text, "hello"); + let logprobs = chunk.choices[0].logprobs.as_ref().expect("logprobs"); + assert_eq!(logprobs.tokens, vec!["he".to_string(), "llo".to_string()]); + assert_eq!(logprobs.token_logprobs, vec![None, Some(-0.2)]); + assert_eq!(logprobs.text_offset, vec![0, 2]); + } + CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"), + } + match &chunks[1] { + CompletionSseChunk::Chunk(chunk) => { + assert_eq!(chunk.choices[0].text, ""); + assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length")); + } + CompletionSseChunk::Usage(_) => panic!("expected final chunk"), + } + } } diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 2f6c760a99..9c30692859 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -27,6 +27,8 @@ pub(super) struct ResponseOptions { pub include_usage: bool, /// Whether every streamed chunk should carry cumulative usage. pub include_continuous_usage: bool, + /// Whether the caller requested prompt-only echo via `max_tokens=0`. + pub prompt_only: bool, /// Original text prompt that should be echoed back northbound when /// `echo=true`. pub echo: Option, @@ -68,11 +70,13 @@ pub(super) fn prepare_completion_request( })?), None => None, }; - let prompt_logprobs = request.prompt_logprobs.or(if request.echo && !request.stream { - logprobs - } else { - None - }); + let prompt_only = request.echo && request.max_tokens == Some(0); + let prompt_logprobs = + request.prompt_logprobs.or(if request.echo && (!request.stream || prompt_only) { + logprobs + } else { + None + }); let include_usage = (request.stream_options.as_ref()) .and_then(|options| options.include_usage) .unwrap_or(false); @@ -83,6 +87,11 @@ pub(super) fn prepare_completion_request( .and_then(|options| options.continuous_usage_stats) .unwrap_or(false); let include_prompt_logprobs = prompt_logprobs.is_some(); + let max_tokens = if prompt_only { + Some(1) + } else { + request.max_tokens + }; let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten(); let structured_outputs = @@ -97,7 +106,7 @@ pub(super) fn prepare_completion_request( top_p: request.top_p, top_k: request.top_k, seed: request.seed, - max_tokens: request.max_tokens, + max_tokens, min_tokens: request.min_tokens, logprobs, prompt_logprobs, @@ -138,6 +147,7 @@ pub(super) fn prepare_completion_request( options: ResponseOptions { include_usage, include_continuous_usage, + prompt_only, echo, requested_logprobs: request.logprobs, include_prompt_logprobs, @@ -325,6 +335,57 @@ mod tests { assert_eq!(prepared.options.echo, Some("hello".to_string())); assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7)); + assert!(!prepared.options.prompt_only); + } + + #[test] + fn prepare_completion_request_lowers_prompt_only_echo_as_one_internal_token() { + let request: CompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "stream": false, + "echo": true, + "max_tokens": 0 + })) + .expect("parse request"); + + let prepared = prepare_completion_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare"); + + assert!(prepared.options.prompt_only); + assert_eq!(prepared.options.echo, Some("hello".to_string())); + assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(1)); + } + + #[test] + fn prepare_completion_request_enables_prompt_logprobs_for_stream_prompt_only_echo() { + let request: CompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "echo": true, + "stream": true, + "max_tokens": 0, + "logprobs": 3 + })) + .expect("parse request"); + + let prepared = prepare_completion_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare"); + + assert!(prepared.options.prompt_only); + assert_eq!(prepared.text_request.sampling_params.logprobs, Some(3)); + assert_eq!( + prepared.text_request.sampling_params.prompt_logprobs, + Some(3) + ); } #[test] diff --git a/rust/src/server/src/routes/openai/completions/validate.rs b/rust/src/server/src/routes/openai/completions/validate.rs index f19defe5e4..cbb040b90d 100644 --- a/rust/src/server/src/routes/openai/completions/validate.rs +++ b/rust/src/server/src/routes/openai/completions/validate.rs @@ -26,8 +26,11 @@ pub(super) fn validate_request_compat( bail_invalid_request!(param = "n", "Only n=1 is supported."); } - if request.max_tokens == Some(0) { - bail_invalid_request!(param = "max_tokens", "max_tokens must be greater than 0."); + if request.max_tokens == Some(0) && !request.echo { + bail_invalid_request!( + param = "max_tokens", + "max_tokens=0 is only supported when echo=true." + ); } if request.echo && matches!(request.prompt, Prompt::TokenIds(_)) { @@ -166,4 +169,30 @@ mod tests { validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok() ); } + + #[test] + fn validate_request_compat_accepts_prompt_only_echo() { + let request = CompletionRequest { + stream: false, + echo: true, + max_tokens: Some(0), + ..base_request() + }; + assert!( + validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok() + ); + } + + #[test] + fn validate_request_compat_rejects_prompt_only_without_echo() { + let request = CompletionRequest { + stream: false, + echo: false, + max_tokens: Some(0), + ..base_request() + }; + assert!( + validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err() + ); + } }