[Rust Frontend] Support prompt-only completions (#44938)
Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
@@ -2,6 +2,7 @@ mod convert;
|
||||
mod types;
|
||||
mod validate;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::convert::Infallible;
|
||||
use std::result::Result;
|
||||
use std::sync::Arc;
|
||||
@@ -16,12 +17,15 @@ use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::{debug, error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _};
|
||||
use vllm_text::{
|
||||
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
|
||||
TextOutputStreamExt as _,
|
||||
};
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_completion_request};
|
||||
use super::utils::logprobs::{
|
||||
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
|
||||
text_len,
|
||||
decoded_prompt_logprobs_to_openai, text_len,
|
||||
};
|
||||
use super::utils::types::Usage;
|
||||
use crate::config::ApiServerOptions;
|
||||
@@ -119,6 +123,7 @@ async fn collect_completion(
|
||||
include_usage: _,
|
||||
// Ignored: non-streaming responses are collected before usage is attached.
|
||||
include_continuous_usage: _,
|
||||
prompt_only,
|
||||
echo,
|
||||
requested_logprobs,
|
||||
include_prompt_logprobs,
|
||||
@@ -136,17 +141,17 @@ async fn collect_completion(
|
||||
.map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON"));
|
||||
|
||||
let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default();
|
||||
let prompt_logprobs = if include_prompt_logprobs {
|
||||
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
|
||||
server_error!(
|
||||
"completion response requested prompt_logprobs but generation returned none"
|
||||
)
|
||||
let logprobs = if requested_logprobs.is_some() && prompt_only {
|
||||
let prompt = echo.as_deref().ok_or_else(|| {
|
||||
server_error!("prompt-only completion response missing echoed prompt")
|
||||
})?;
|
||||
Some(prompt_logprobs)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let logprobs = if requested_logprobs.is_some() {
|
||||
Some(prompt_only_logprobs_to_openai(
|
||||
collected.prompt_logprobs.as_ref(),
|
||||
prompt,
|
||||
collected.prompt_token_ids.as_ref(),
|
||||
return_tokens_as_token_ids,
|
||||
)?)
|
||||
} else if requested_logprobs.is_some() {
|
||||
Some(collected_logprobs_to_openai(
|
||||
&collected,
|
||||
echo.is_some(),
|
||||
@@ -156,10 +161,18 @@ async fn collect_completion(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let prompt_logprobs =
|
||||
prompt_logprobs.map(|lp| decoded_prompt_logprobs_to_maps(lp, return_tokens_as_token_ids));
|
||||
let prompt_logprobs = if include_prompt_logprobs {
|
||||
Some(prompt_logprobs_to_maps(
|
||||
collected.prompt_logprobs.as_ref(),
|
||||
collected.prompt_token_ids.as_ref(),
|
||||
return_tokens_as_token_ids,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let text = match &echo {
|
||||
None => collected.text,
|
||||
Some(prompt) if prompt_only => prompt.clone(),
|
||||
Some(prompt) => format!("{prompt}{}", collected.text),
|
||||
};
|
||||
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
|
||||
@@ -211,6 +224,7 @@ async fn completion_chunk_stream(
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
prompt_only,
|
||||
echo,
|
||||
requested_logprobs,
|
||||
// Ignored: streaming prompt logprobs are rejected for Python parity.
|
||||
@@ -239,14 +253,30 @@ async fn completion_chunk_stream(
|
||||
while let Some(next) = stream.next().await {
|
||||
match next {
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids, ..
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
}) => {
|
||||
debug!("completion stream started");
|
||||
continuous_usage.set_prompt_tokens(prompt_token_ids.len());
|
||||
if let Some(prompt) = echo.as_ref() {
|
||||
visible_text_len = text_len(prompt);
|
||||
let mut chunk =
|
||||
delta_chunk(&request_id, &response_model, created, prompt.clone(), None);
|
||||
let logprobs = if prompt_only && requested_logprobs.is_some() {
|
||||
Some(prompt_only_logprobs_to_openai(
|
||||
prompt_logprobs.as_ref(),
|
||||
prompt,
|
||||
prompt_token_ids.as_ref(),
|
||||
return_tokens_as_token_ids,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut chunk = delta_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
prompt.clone(),
|
||||
logprobs,
|
||||
);
|
||||
if return_token_ids && first_chunk {
|
||||
if let Some(choice) = chunk.choices.first_mut() {
|
||||
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
||||
@@ -271,6 +301,48 @@ async fn completion_chunk_stream(
|
||||
logprobs,
|
||||
finished,
|
||||
}) => {
|
||||
// Prompt-only streaming already emitted the echoed prompt in the Start chunk.
|
||||
// The one generated token is only used to drive the engine to a finished event,
|
||||
// so hide its delta and forward only the terminal finish/usage metadata.
|
||||
if prompt_only {
|
||||
if let Some(finished) = finished {
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
stream = true,
|
||||
model = %response_model,
|
||||
prompt_tokens = finished.usage.prompt_token_count,
|
||||
output_tokens = finished.usage.output_token_count,
|
||||
finish_reason = finished.finish_reason.as_str(),
|
||||
"completion finished"
|
||||
);
|
||||
}
|
||||
continuous_usage.set_final_counts(
|
||||
finished.usage.prompt_token_count,
|
||||
finished.usage.output_token_count,
|
||||
);
|
||||
let final_chunk = final_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
finished.finish_reason,
|
||||
)?;
|
||||
yield_chunk!(final_chunk);
|
||||
|
||||
if include_usage {
|
||||
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
Usage::from_token_usage(
|
||||
finished.usage,
|
||||
enable_prompt_tokens_details,
|
||||
),
|
||||
)))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let delta_text_len = text_len(&delta);
|
||||
let logprobs = if requested_logprobs.is_some() {
|
||||
let decoded_logprobs = logprobs.as_ref().ok_or_else(|| {
|
||||
@@ -386,6 +458,57 @@ fn completion_finish_reason_to_openai(
|
||||
}
|
||||
}
|
||||
|
||||
fn prompt_only_logprobs_to_openai(
|
||||
prompt_logprobs: Option<&DecodedPromptLogprobs>,
|
||||
prompt: &str,
|
||||
prompt_token_ids: &[u32],
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<LogProbs, ApiError> {
|
||||
if let Some(prompt_logprobs) = prompt_logprobs {
|
||||
return decoded_prompt_logprobs_to_openai(prompt_logprobs, 0, return_tokens_as_token_ids);
|
||||
}
|
||||
|
||||
if let [token_id] = prompt_token_ids {
|
||||
let token = if return_tokens_as_token_ids {
|
||||
format!("token_id:{token_id}")
|
||||
} else {
|
||||
prompt.to_string()
|
||||
};
|
||||
|
||||
return Ok(LogProbs {
|
||||
tokens: vec![token],
|
||||
token_logprobs: vec![None],
|
||||
top_logprobs: vec![None],
|
||||
text_offset: vec![0],
|
||||
});
|
||||
}
|
||||
|
||||
Err(server_error!(
|
||||
"prompt-only completion requested logprobs but generation returned none"
|
||||
))
|
||||
}
|
||||
|
||||
fn prompt_logprobs_to_maps(
|
||||
prompt_logprobs: Option<&DecodedPromptLogprobs>,
|
||||
prompt_token_ids: &[u32],
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<Vec<Option<HashMap<String, f32>>>, ApiError> {
|
||||
if let Some(prompt_logprobs) = prompt_logprobs {
|
||||
return Ok(decoded_prompt_logprobs_to_maps(
|
||||
prompt_logprobs,
|
||||
return_tokens_as_token_ids,
|
||||
));
|
||||
}
|
||||
|
||||
if let [_token_id] = prompt_token_ids {
|
||||
return Ok(vec![None]);
|
||||
}
|
||||
|
||||
Err(server_error!(
|
||||
"completion response requested prompt_logprobs but generation returned none"
|
||||
))
|
||||
}
|
||||
|
||||
fn usage_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
@@ -449,8 +572,8 @@ mod tests {
|
||||
use futures::{StreamExt as _, stream};
|
||||
use itertools::Itertools as _;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
|
||||
FinishReason, Finished,
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
|
||||
DecodedTokenLogprob, FinishReason, Finished,
|
||||
};
|
||||
|
||||
use super::{
|
||||
@@ -613,4 +736,314 @@ mod tests {
|
||||
CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_completion_hides_internal_prompt_only_token() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![1, 2].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let response = super::collect_completion(
|
||||
stream,
|
||||
"cmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
prompt_only: true,
|
||||
echo: Some("hello".to_string()),
|
||||
return_token_ids: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("collect completion");
|
||||
|
||||
assert_eq!(response.choices[0].text, "hello");
|
||||
assert_eq!(response.choices[0].token_ids.as_deref(), Some(&[3][..]));
|
||||
assert_eq!(
|
||||
response.choices[0].prompt_token_ids.as_deref(),
|
||||
Some(&[1, 2][..])
|
||||
);
|
||||
let usage = response.usage.expect("usage");
|
||||
assert_eq!(usage.prompt_tokens, 2);
|
||||
assert_eq!(usage.completion_tokens, Some(1));
|
||||
assert_eq!(usage.total_tokens, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_completion_maps_prompt_logprobs_for_single_token_prompt() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![9707].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let response = super::collect_completion(
|
||||
stream,
|
||||
"cmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
prompt_only: true,
|
||||
echo: Some("Hello".to_string()),
|
||||
requested_logprobs: Some(1),
|
||||
include_prompt_logprobs: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("collect completion");
|
||||
|
||||
let choice = &response.choices[0];
|
||||
assert_eq!(choice.text, "Hello");
|
||||
assert_eq!(choice.prompt_logprobs, Some(vec![None]));
|
||||
let logprobs = choice.logprobs.as_ref().expect("logprobs");
|
||||
assert_eq!(logprobs.tokens, vec!["Hello".to_string()]);
|
||||
assert_eq!(logprobs.token_logprobs, vec![None]);
|
||||
assert_eq!(logprobs.top_logprobs, vec![None]);
|
||||
assert_eq!(logprobs.text_offset, vec![0]);
|
||||
let usage = response.usage.expect("usage");
|
||||
assert_eq!(usage.prompt_tokens, 1);
|
||||
assert_eq!(usage.completion_tokens, Some(1));
|
||||
assert_eq!(usage.total_tokens, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_chunk_stream_hides_internal_prompt_only_token() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![1, 2].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
stream,
|
||||
"cmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
prompt_only: true,
|
||||
echo: Some("hello".to_string()),
|
||||
return_token_ids: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
|
||||
assert_eq!(chunks.len(), 3);
|
||||
|
||||
match &chunks[0] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "hello");
|
||||
assert_eq!(
|
||||
chunk.choices[0].prompt_token_ids.as_deref(),
|
||||
Some(&[1, 2][..])
|
||||
);
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
|
||||
}
|
||||
match &chunks[1] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
|
||||
}
|
||||
match &chunks[2] {
|
||||
CompletionSseChunk::Usage(chunk) => {
|
||||
let usage = chunk.usage.as_ref().expect("usage");
|
||||
assert_eq!(usage.prompt_tokens, 2);
|
||||
assert_eq!(usage.completion_tokens, Some(1));
|
||||
assert_eq!(usage.total_tokens, 3);
|
||||
}
|
||||
CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_chunk_stream_maps_prompt_logprobs_for_single_token_prompt() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![9707].into(),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
stream,
|
||||
"cmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
prompt_only: true,
|
||||
echo: Some("Hello".to_string()),
|
||||
requested_logprobs: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
|
||||
assert_eq!(chunks.len(), 2);
|
||||
|
||||
match &chunks[0] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "Hello");
|
||||
let logprobs = chunk.choices[0].logprobs.as_ref().expect("logprobs");
|
||||
assert_eq!(logprobs.tokens, vec!["Hello".to_string()]);
|
||||
assert_eq!(logprobs.token_logprobs, vec![None]);
|
||||
assert_eq!(logprobs.top_logprobs, vec![None]);
|
||||
assert_eq!(logprobs.text_offset, vec![0]);
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
|
||||
}
|
||||
match &chunks[1] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_chunk_stream_maps_prompt_only_logprobs() {
|
||||
let stream = stream::iter(vec![
|
||||
Ok(DecodedTextEvent::Start {
|
||||
prompt_token_ids: vec![1, 2].into(),
|
||||
prompt_logprobs: Some(DecodedPromptLogprobs {
|
||||
first_token_id: 1,
|
||||
first_token: "he".to_string(),
|
||||
scored_positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![DecodedTokenLogprob {
|
||||
token_id: 2,
|
||||
token: "llo".to_string(),
|
||||
logprob: -0.2,
|
||||
rank: 1,
|
||||
}],
|
||||
}],
|
||||
}),
|
||||
}),
|
||||
Ok(DecodedTextEvent::TextDelta {
|
||||
delta: " leaked".to_string(),
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finished: Some(Finished {
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::Length,
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks = completion_chunk_stream(
|
||||
stream,
|
||||
"cmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
prompt_only: true,
|
||||
echo: Some("hello".to_string()),
|
||||
requested_logprobs: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
|
||||
assert_eq!(chunks.len(), 2);
|
||||
|
||||
match &chunks[0] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "hello");
|
||||
let logprobs = chunk.choices[0].logprobs.as_ref().expect("logprobs");
|
||||
assert_eq!(logprobs.tokens, vec!["he".to_string(), "llo".to_string()]);
|
||||
assert_eq!(logprobs.token_logprobs, vec![None, Some(-0.2)]);
|
||||
assert_eq!(logprobs.text_offset, vec![0, 2]);
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
|
||||
}
|
||||
match &chunks[1] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ pub(super) struct ResponseOptions {
|
||||
pub include_usage: bool,
|
||||
/// Whether every streamed chunk should carry cumulative usage.
|
||||
pub include_continuous_usage: bool,
|
||||
/// Whether the caller requested prompt-only echo via `max_tokens=0`.
|
||||
pub prompt_only: bool,
|
||||
/// Original text prompt that should be echoed back northbound when
|
||||
/// `echo=true`.
|
||||
pub echo: Option<String>,
|
||||
@@ -68,11 +70,13 @@ pub(super) fn prepare_completion_request(
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
let prompt_logprobs = request.prompt_logprobs.or(if request.echo && !request.stream {
|
||||
logprobs
|
||||
} else {
|
||||
None
|
||||
});
|
||||
let prompt_only = request.echo && request.max_tokens == Some(0);
|
||||
let prompt_logprobs =
|
||||
request.prompt_logprobs.or(if request.echo && (!request.stream || prompt_only) {
|
||||
logprobs
|
||||
} else {
|
||||
None
|
||||
});
|
||||
let include_usage = (request.stream_options.as_ref())
|
||||
.and_then(|options| options.include_usage)
|
||||
.unwrap_or(false);
|
||||
@@ -83,6 +87,11 @@ pub(super) fn prepare_completion_request(
|
||||
.and_then(|options| options.continuous_usage_stats)
|
||||
.unwrap_or(false);
|
||||
let include_prompt_logprobs = prompt_logprobs.is_some();
|
||||
let max_tokens = if prompt_only {
|
||||
Some(1)
|
||||
} else {
|
||||
request.max_tokens
|
||||
};
|
||||
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
|
||||
|
||||
let structured_outputs =
|
||||
@@ -97,7 +106,7 @@ pub(super) fn prepare_completion_request(
|
||||
top_p: request.top_p,
|
||||
top_k: request.top_k,
|
||||
seed: request.seed,
|
||||
max_tokens: request.max_tokens,
|
||||
max_tokens,
|
||||
min_tokens: request.min_tokens,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
@@ -138,6 +147,7 @@ pub(super) fn prepare_completion_request(
|
||||
options: ResponseOptions {
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
prompt_only,
|
||||
echo,
|
||||
requested_logprobs: request.logprobs,
|
||||
include_prompt_logprobs,
|
||||
@@ -325,6 +335,57 @@ mod tests {
|
||||
|
||||
assert_eq!(prepared.options.echo, Some("hello".to_string()));
|
||||
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
|
||||
assert!(!prepared.options.prompt_only);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_lowers_prompt_only_echo_as_one_internal_token() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"stream": false,
|
||||
"echo": true,
|
||||
"max_tokens": 0
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert!(prepared.options.prompt_only);
|
||||
assert_eq!(prepared.options.echo, Some("hello".to_string()));
|
||||
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_enables_prompt_logprobs_for_stream_prompt_only_echo() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"prompt": "hello",
|
||||
"echo": true,
|
||||
"stream": true,
|
||||
"max_tokens": 0,
|
||||
"logprobs": 3
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert!(prepared.options.prompt_only);
|
||||
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(3));
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.prompt_logprobs,
|
||||
Some(3)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -26,8 +26,11 @@ pub(super) fn validate_request_compat(
|
||||
bail_invalid_request!(param = "n", "Only n=1 is supported.");
|
||||
}
|
||||
|
||||
if request.max_tokens == Some(0) {
|
||||
bail_invalid_request!(param = "max_tokens", "max_tokens must be greater than 0.");
|
||||
if request.max_tokens == Some(0) && !request.echo {
|
||||
bail_invalid_request!(
|
||||
param = "max_tokens",
|
||||
"max_tokens=0 is only supported when echo=true."
|
||||
);
|
||||
}
|
||||
|
||||
if request.echo && matches!(request.prompt, Prompt::TokenIds(_)) {
|
||||
@@ -166,4 +169,30 @@ mod tests {
|
||||
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_prompt_only_echo() {
|
||||
let request = CompletionRequest {
|
||||
stream: false,
|
||||
echo: true,
|
||||
max_tokens: Some(0),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_prompt_only_without_echo() {
|
||||
let request = CompletionRequest {
|
||||
stream: false,
|
||||
echo: false,
|
||||
max_tokens: Some(0),
|
||||
..base_request()
|
||||
};
|
||||
assert!(
|
||||
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user