[Rust Frontend] Support prompt-only completions (#44938)

Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
Reid
2026-06-17 06:38:06 +00:00
committed by GitHub
parent e9993a52aa
commit 56e4345226
3 changed files with 550 additions and 27 deletions
+452 -19
View File
@@ -2,6 +2,7 @@ mod convert;
mod types;
mod validate;
use std::collections::HashMap;
use std::convert::Infallible;
use std::result::Result;
use std::sync::Arc;
@@ -16,12 +17,15 @@ use futures::{Stream, StreamExt as _, pin_mut};
use thiserror_ext::AsReport as _;
use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _;
use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _};
use vllm_text::{
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
TextOutputStreamExt as _,
};
use self::convert::{ResponseOptions, prepare_completion_request};
use super::utils::logprobs::{
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
text_len,
decoded_prompt_logprobs_to_openai, text_len,
};
use super::utils::types::Usage;
use crate::config::ApiServerOptions;
@@ -119,6 +123,7 @@ async fn collect_completion(
include_usage: _,
// Ignored: non-streaming responses are collected before usage is attached.
include_continuous_usage: _,
prompt_only,
echo,
requested_logprobs,
include_prompt_logprobs,
@@ -136,17 +141,17 @@ async fn collect_completion(
.map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON"));
let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default();
let prompt_logprobs = if include_prompt_logprobs {
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
server_error!(
"completion response requested prompt_logprobs but generation returned none"
)
let logprobs = if requested_logprobs.is_some() && prompt_only {
let prompt = echo.as_deref().ok_or_else(|| {
server_error!("prompt-only completion response missing echoed prompt")
})?;
Some(prompt_logprobs)
} else {
None
};
let logprobs = if requested_logprobs.is_some() {
Some(prompt_only_logprobs_to_openai(
collected.prompt_logprobs.as_ref(),
prompt,
collected.prompt_token_ids.as_ref(),
return_tokens_as_token_ids,
)?)
} else if requested_logprobs.is_some() {
Some(collected_logprobs_to_openai(
&collected,
echo.is_some(),
@@ -156,10 +161,18 @@ async fn collect_completion(
} else {
None
};
let prompt_logprobs =
prompt_logprobs.map(|lp| decoded_prompt_logprobs_to_maps(lp, return_tokens_as_token_ids));
let prompt_logprobs = if include_prompt_logprobs {
Some(prompt_logprobs_to_maps(
collected.prompt_logprobs.as_ref(),
collected.prompt_token_ids.as_ref(),
return_tokens_as_token_ids,
)?)
} else {
None
};
let text = match &echo {
None => collected.text,
Some(prompt) if prompt_only => prompt.clone(),
Some(prompt) => format!("{prompt}{}", collected.text),
};
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
@@ -211,6 +224,7 @@ async fn completion_chunk_stream(
ResponseOptions {
include_usage,
include_continuous_usage,
prompt_only,
echo,
requested_logprobs,
// Ignored: streaming prompt logprobs are rejected for Python parity.
@@ -239,14 +253,30 @@ async fn completion_chunk_stream(
while let Some(next) = stream.next().await {
match next {
Ok(DecodedTextEvent::Start {
prompt_token_ids, ..
prompt_token_ids,
prompt_logprobs,
}) => {
debug!("completion stream started");
continuous_usage.set_prompt_tokens(prompt_token_ids.len());
if let Some(prompt) = echo.as_ref() {
visible_text_len = text_len(prompt);
let mut chunk =
delta_chunk(&request_id, &response_model, created, prompt.clone(), None);
let logprobs = if prompt_only && requested_logprobs.is_some() {
Some(prompt_only_logprobs_to_openai(
prompt_logprobs.as_ref(),
prompt,
prompt_token_ids.as_ref(),
return_tokens_as_token_ids,
)?)
} else {
None
};
let mut chunk = delta_chunk(
&request_id,
&response_model,
created,
prompt.clone(),
logprobs,
);
if return_token_ids && first_chunk {
if let Some(choice) = chunk.choices.first_mut() {
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
@@ -271,6 +301,48 @@ async fn completion_chunk_stream(
logprobs,
finished,
}) => {
// Prompt-only streaming already emitted the echoed prompt in the Start chunk.
// The one generated token is only used to drive the engine to a finished event,
// so hide its delta and forward only the terminal finish/usage metadata.
if prompt_only {
if let Some(finished) = finished {
if enable_log_requests {
info!(
stream = true,
model = %response_model,
prompt_tokens = finished.usage.prompt_token_count,
output_tokens = finished.usage.output_token_count,
finish_reason = finished.finish_reason.as_str(),
"completion finished"
);
}
continuous_usage.set_final_counts(
finished.usage.prompt_token_count,
finished.usage.output_token_count,
);
let final_chunk = final_chunk(
&request_id,
&response_model,
created,
finished.finish_reason,
)?;
yield_chunk!(final_chunk);
if include_usage {
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
&request_id,
&response_model,
created,
Usage::from_token_usage(
finished.usage,
enable_prompt_tokens_details,
),
)))
.await;
}
}
continue;
}
let delta_text_len = text_len(&delta);
let logprobs = if requested_logprobs.is_some() {
let decoded_logprobs = logprobs.as_ref().ok_or_else(|| {
@@ -386,6 +458,57 @@ fn completion_finish_reason_to_openai(
}
}
fn prompt_only_logprobs_to_openai(
prompt_logprobs: Option<&DecodedPromptLogprobs>,
prompt: &str,
prompt_token_ids: &[u32],
return_tokens_as_token_ids: bool,
) -> Result<LogProbs, ApiError> {
if let Some(prompt_logprobs) = prompt_logprobs {
return decoded_prompt_logprobs_to_openai(prompt_logprobs, 0, return_tokens_as_token_ids);
}
if let [token_id] = prompt_token_ids {
let token = if return_tokens_as_token_ids {
format!("token_id:{token_id}")
} else {
prompt.to_string()
};
return Ok(LogProbs {
tokens: vec![token],
token_logprobs: vec![None],
top_logprobs: vec![None],
text_offset: vec![0],
});
}
Err(server_error!(
"prompt-only completion requested logprobs but generation returned none"
))
}
fn prompt_logprobs_to_maps(
prompt_logprobs: Option<&DecodedPromptLogprobs>,
prompt_token_ids: &[u32],
return_tokens_as_token_ids: bool,
) -> Result<Vec<Option<HashMap<String, f32>>>, ApiError> {
if let Some(prompt_logprobs) = prompt_logprobs {
return Ok(decoded_prompt_logprobs_to_maps(
prompt_logprobs,
return_tokens_as_token_ids,
));
}
if let [_token_id] = prompt_token_ids {
return Ok(vec![None]);
}
Err(server_error!(
"completion response requested prompt_logprobs but generation returned none"
))
}
fn usage_chunk(
request_id: &str,
response_model: &str,
@@ -449,8 +572,8 @@ mod tests {
use futures::{StreamExt as _, stream};
use itertools::Itertools as _;
use vllm_text::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
FinishReason, Finished,
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
DecodedTokenLogprob, FinishReason, Finished,
};
use super::{
@@ -613,4 +736,314 @@ mod tests {
CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"),
}
}
#[tokio::test]
async fn collect_completion_hides_internal_prompt_only_token() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1, 2].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::Length,
kv_transfer_params: None,
}),
}),
]);
let response = super::collect_completion(
stream,
"cmpl-1".to_string(),
"model".to_string(),
1,
ApiServerOptions::default(),
ResponseOptions {
prompt_only: true,
echo: Some("hello".to_string()),
return_token_ids: true,
..Default::default()
},
)
.await
.expect("collect completion");
assert_eq!(response.choices[0].text, "hello");
assert_eq!(response.choices[0].token_ids.as_deref(), Some(&[3][..]));
assert_eq!(
response.choices[0].prompt_token_ids.as_deref(),
Some(&[1, 2][..])
);
let usage = response.usage.expect("usage");
assert_eq!(usage.prompt_tokens, 2);
assert_eq!(usage.completion_tokens, Some(1));
assert_eq!(usage.total_tokens, 3);
}
#[tokio::test]
async fn collect_completion_maps_prompt_logprobs_for_single_token_prompt() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![9707].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::Length,
kv_transfer_params: None,
}),
}),
]);
let response = super::collect_completion(
stream,
"cmpl-1".to_string(),
"model".to_string(),
1,
ApiServerOptions::default(),
ResponseOptions {
prompt_only: true,
echo: Some("Hello".to_string()),
requested_logprobs: Some(1),
include_prompt_logprobs: true,
..Default::default()
},
)
.await
.expect("collect completion");
let choice = &response.choices[0];
assert_eq!(choice.text, "Hello");
assert_eq!(choice.prompt_logprobs, Some(vec![None]));
let logprobs = choice.logprobs.as_ref().expect("logprobs");
assert_eq!(logprobs.tokens, vec!["Hello".to_string()]);
assert_eq!(logprobs.token_logprobs, vec![None]);
assert_eq!(logprobs.top_logprobs, vec![None]);
assert_eq!(logprobs.text_offset, vec![0]);
let usage = response.usage.expect("usage");
assert_eq!(usage.prompt_tokens, 1);
assert_eq!(usage.completion_tokens, Some(1));
assert_eq!(usage.total_tokens, 2);
}
#[tokio::test]
async fn completion_chunk_stream_hides_internal_prompt_only_token() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1, 2].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::Length,
kv_transfer_params: None,
}),
}),
]);
let chunks = completion_chunk_stream(
stream,
"cmpl-1".to_string(),
"model".to_string(),
1,
ApiServerOptions::default(),
ResponseOptions {
include_usage: true,
prompt_only: true,
echo: Some("hello".to_string()),
return_token_ids: true,
..Default::default()
},
)
.collect::<Vec<_>>()
.await;
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
assert_eq!(chunks.len(), 3);
match &chunks[0] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "hello");
assert_eq!(
chunk.choices[0].prompt_token_ids.as_deref(),
Some(&[1, 2][..])
);
}
CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
}
match &chunks[1] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
}
CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
}
match &chunks[2] {
CompletionSseChunk::Usage(chunk) => {
let usage = chunk.usage.as_ref().expect("usage");
assert_eq!(usage.prompt_tokens, 2);
assert_eq!(usage.completion_tokens, Some(1));
assert_eq!(usage.total_tokens, 3);
}
CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"),
}
}
#[tokio::test]
async fn completion_chunk_stream_maps_prompt_logprobs_for_single_token_prompt() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![9707].into(),
prompt_logprobs: None,
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::Length,
kv_transfer_params: None,
}),
}),
]);
let chunks = completion_chunk_stream(
stream,
"cmpl-1".to_string(),
"model".to_string(),
1,
ApiServerOptions::default(),
ResponseOptions {
prompt_only: true,
echo: Some("Hello".to_string()),
requested_logprobs: Some(1),
..Default::default()
},
)
.collect::<Vec<_>>()
.await;
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
assert_eq!(chunks.len(), 2);
match &chunks[0] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "Hello");
let logprobs = chunk.choices[0].logprobs.as_ref().expect("logprobs");
assert_eq!(logprobs.tokens, vec!["Hello".to_string()]);
assert_eq!(logprobs.token_logprobs, vec![None]);
assert_eq!(logprobs.top_logprobs, vec![None]);
assert_eq!(logprobs.text_offset, vec![0]);
}
CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
}
match &chunks[1] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
}
CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
}
}
#[tokio::test]
async fn completion_chunk_stream_maps_prompt_only_logprobs() {
let stream = stream::iter(vec![
Ok(DecodedTextEvent::Start {
prompt_token_ids: vec![1, 2].into(),
prompt_logprobs: Some(DecodedPromptLogprobs {
first_token_id: 1,
first_token: "he".to_string(),
scored_positions: vec![DecodedPositionLogprobs {
entries: vec![DecodedTokenLogprob {
token_id: 2,
token: "llo".to_string(),
logprob: -0.2,
rank: 1,
}],
}],
}),
}),
Ok(DecodedTextEvent::TextDelta {
delta: " leaked".to_string(),
token_ids: vec![3],
logprobs: None,
finished: Some(Finished {
usage: vllm_llm::TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::Length,
kv_transfer_params: None,
}),
}),
]);
let chunks = completion_chunk_stream(
stream,
"cmpl-1".to_string(),
"model".to_string(),
1,
ApiServerOptions::default(),
ResponseOptions {
prompt_only: true,
echo: Some("hello".to_string()),
requested_logprobs: Some(1),
..Default::default()
},
)
.collect::<Vec<_>>()
.await;
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
assert_eq!(chunks.len(), 2);
match &chunks[0] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "hello");
let logprobs = chunk.choices[0].logprobs.as_ref().expect("logprobs");
assert_eq!(logprobs.tokens, vec!["he".to_string(), "llo".to_string()]);
assert_eq!(logprobs.token_logprobs, vec![None, Some(-0.2)]);
assert_eq!(logprobs.text_offset, vec![0, 2]);
}
CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
}
match &chunks[1] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
}
CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
}
}
}
@@ -27,6 +27,8 @@ pub(super) struct ResponseOptions {
pub include_usage: bool,
/// Whether every streamed chunk should carry cumulative usage.
pub include_continuous_usage: bool,
/// Whether the caller requested prompt-only echo via `max_tokens=0`.
pub prompt_only: bool,
/// Original text prompt that should be echoed back northbound when
/// `echo=true`.
pub echo: Option<String>,
@@ -68,11 +70,13 @@ pub(super) fn prepare_completion_request(
})?),
None => None,
};
let prompt_logprobs = request.prompt_logprobs.or(if request.echo && !request.stream {
logprobs
} else {
None
});
let prompt_only = request.echo && request.max_tokens == Some(0);
let prompt_logprobs =
request.prompt_logprobs.or(if request.echo && (!request.stream || prompt_only) {
logprobs
} else {
None
});
let include_usage = (request.stream_options.as_ref())
.and_then(|options| options.include_usage)
.unwrap_or(false);
@@ -83,6 +87,11 @@ pub(super) fn prepare_completion_request(
.and_then(|options| options.continuous_usage_stats)
.unwrap_or(false);
let include_prompt_logprobs = prompt_logprobs.is_some();
let max_tokens = if prompt_only {
Some(1)
} else {
request.max_tokens
};
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
let structured_outputs =
@@ -97,7 +106,7 @@ pub(super) fn prepare_completion_request(
top_p: request.top_p,
top_k: request.top_k,
seed: request.seed,
max_tokens: request.max_tokens,
max_tokens,
min_tokens: request.min_tokens,
logprobs,
prompt_logprobs,
@@ -138,6 +147,7 @@ pub(super) fn prepare_completion_request(
options: ResponseOptions {
include_usage,
include_continuous_usage,
prompt_only,
echo,
requested_logprobs: request.logprobs,
include_prompt_logprobs,
@@ -325,6 +335,57 @@ mod tests {
assert_eq!(prepared.options.echo, Some("hello".to_string()));
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
assert!(!prepared.options.prompt_only);
}
#[test]
fn prepare_completion_request_lowers_prompt_only_echo_as_one_internal_token() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"stream": false,
"echo": true,
"max_tokens": 0
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
assert!(prepared.options.prompt_only);
assert_eq!(prepared.options.echo, Some("hello".to_string()));
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(1));
}
#[test]
fn prepare_completion_request_enables_prompt_logprobs_for_stream_prompt_only_echo() {
let request: CompletionRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"prompt": "hello",
"echo": true,
"stream": true,
"max_tokens": 0,
"logprobs": 3
}))
.expect("parse request");
let prepared = prepare_completion_request(
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
)
.expect("prepare");
assert!(prepared.options.prompt_only);
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(3));
assert_eq!(
prepared.text_request.sampling_params.prompt_logprobs,
Some(3)
);
}
#[test]
@@ -26,8 +26,11 @@ pub(super) fn validate_request_compat(
bail_invalid_request!(param = "n", "Only n=1 is supported.");
}
if request.max_tokens == Some(0) {
bail_invalid_request!(param = "max_tokens", "max_tokens must be greater than 0.");
if request.max_tokens == Some(0) && !request.echo {
bail_invalid_request!(
param = "max_tokens",
"max_tokens=0 is only supported when echo=true."
);
}
if request.echo && matches!(request.prompt, Prompt::TokenIds(_)) {
@@ -166,4 +169,30 @@ mod tests {
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
);
}
#[test]
fn validate_request_compat_accepts_prompt_only_echo() {
let request = CompletionRequest {
stream: false,
echo: true,
max_tokens: Some(0),
..base_request()
};
assert!(
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
);
}
#[test]
fn validate_request_compat_rejects_prompt_only_without_echo() {
let request = CompletionRequest {
stream: false,
echo: false,
max_tokens: Some(0),
..base_request()
};
assert!(
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
);
}
}