[Rust Frontend] Support continuous_usage_stats stream option (#43965)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: RickyChen / 陳昭儒 <ricky.chen@infinirc.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
b8142294b7
commit
3b03a2cf47
@@ -37,6 +37,7 @@ use crate::routes::openai::utils::logprobs::{
|
|||||||
use crate::routes::openai::utils::types::{
|
use crate::routes::openai::utils::types::{
|
||||||
ChatLogProbs, FunctionCallDelta, FunctionCallResponse, ToolCall, ToolCallDelta, Usage,
|
ChatLogProbs, FunctionCallDelta, FunctionCallResponse, ToolCall, ToolCallDelta, Usage,
|
||||||
};
|
};
|
||||||
|
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||||
@@ -129,6 +130,8 @@ async fn collect_chat_completion(
|
|||||||
ResponseOptions {
|
ResponseOptions {
|
||||||
// Ignored: non-streaming responses always include usage.
|
// Ignored: non-streaming responses always include usage.
|
||||||
include_usage: _,
|
include_usage: _,
|
||||||
|
// Ignored: non-streaming responses are collected before usage is attached.
|
||||||
|
include_continuous_usage: _,
|
||||||
requested_logprobs,
|
requested_logprobs,
|
||||||
include_prompt_logprobs,
|
include_prompt_logprobs,
|
||||||
include_reasoning,
|
include_reasoning,
|
||||||
@@ -249,6 +252,7 @@ async fn chat_completion_chunk_stream(
|
|||||||
}: ApiServerOptions,
|
}: ApiServerOptions,
|
||||||
ResponseOptions {
|
ResponseOptions {
|
||||||
include_usage,
|
include_usage,
|
||||||
|
include_continuous_usage,
|
||||||
requested_logprobs,
|
requested_logprobs,
|
||||||
// Ignored: chat streaming prompt logprobs are rejected for Python parity.
|
// Ignored: chat streaming prompt logprobs are rejected for Python parity.
|
||||||
include_prompt_logprobs: _,
|
include_prompt_logprobs: _,
|
||||||
@@ -265,33 +269,47 @@ async fn chat_completion_chunk_stream(
|
|||||||
// starts or ends, omit its token metadata as well as its visible delta.
|
// starts or ends, omit its token metadata as well as its visible delta.
|
||||||
let mut inside_hidden_reasoning = false;
|
let mut inside_hidden_reasoning = false;
|
||||||
let mut suppress_current_update_metadata = false;
|
let mut suppress_current_update_metadata = false;
|
||||||
|
let mut continuous_usage = ContinuousUsage::default();
|
||||||
|
|
||||||
|
/// Yield a chunk with optional continuous usage attached.
|
||||||
|
macro_rules! yield_chunk {
|
||||||
|
($chunk:expr) => {{
|
||||||
|
let mut chunk = $chunk;
|
||||||
|
if include_continuous_usage {
|
||||||
|
chunk.usage = Some(continuous_usage.to_usage());
|
||||||
|
}
|
||||||
|
y.yield_ok(chunk).await;
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
|
||||||
// If the client requested logprobs or token_ids, we need to buffer chunks until
|
// If the client requested logprobs or token_ids, we need to buffer chunks until
|
||||||
// we receive the separate `LogprobsDelta` event, so that we can emit one
|
// we receive the separate `LogprobsDelta` event, so that we can emit one
|
||||||
// combined chunk with both the semantic delta and its per-update metadata.
|
// combined chunk with both the semantic delta and its per-update metadata.
|
||||||
let mut pending_chunk =
|
// Continuous usage also buffers so the token count from `LogprobsDelta` can
|
||||||
(requested_logprobs || return_token_ids).then(PendingChatChunk::default);
|
// be attached to the matching semantic chunk.
|
||||||
|
let mut pending_chunk = (requested_logprobs || return_token_ids || include_continuous_usage)
|
||||||
|
.then(PendingChatChunk::default);
|
||||||
|
|
||||||
while let Some(next) = stream.next().await {
|
while let Some(next) = stream.next().await {
|
||||||
match next {
|
match next {
|
||||||
Ok(ChatEvent::Start {
|
Ok(ChatEvent::Start {
|
||||||
prompt_token_ids, ..
|
prompt_token_ids, ..
|
||||||
}) => {
|
}) => {
|
||||||
|
continuous_usage.set_prompt_tokens(prompt_token_ids.len());
|
||||||
let mut chunk = start_chunk(&request_id, &response_model, created);
|
let mut chunk = start_chunk(&request_id, &response_model, created);
|
||||||
if return_token_ids {
|
if return_token_ids {
|
||||||
chunk.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
chunk.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
||||||
}
|
}
|
||||||
y.yield_ok(chunk).await;
|
yield_chunk!(chunk);
|
||||||
// When echo=true, emit the last assistant message content as a delta chunk.
|
// When echo=true, emit the last assistant message content as a delta chunk.
|
||||||
if let Some(echo_text) = &echo {
|
if let Some(echo_text) = &echo {
|
||||||
y.yield_ok(block_delta_chunk(
|
yield_chunk!(block_delta_chunk(
|
||||||
&request_id,
|
&request_id,
|
||||||
&response_model,
|
&response_model,
|
||||||
created,
|
created,
|
||||||
AssistantBlockKind::Text,
|
AssistantBlockKind::Text,
|
||||||
echo_text.clone(),
|
echo_text.clone(),
|
||||||
))
|
));
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(ChatEvent::BlockDelta { kind, delta, .. }) => {
|
Ok(ChatEvent::BlockDelta { kind, delta, .. }) => {
|
||||||
@@ -301,14 +319,13 @@ async fn chat_completion_chunk_stream(
|
|||||||
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
||||||
pending_chunk.push_block_delta(kind, delta);
|
pending_chunk.push_block_delta(kind, delta);
|
||||||
} else {
|
} else {
|
||||||
y.yield_ok(block_delta_chunk(
|
yield_chunk!(block_delta_chunk(
|
||||||
&request_id,
|
&request_id,
|
||||||
&response_model,
|
&response_model,
|
||||||
created,
|
created,
|
||||||
kind,
|
kind,
|
||||||
delta,
|
delta,
|
||||||
))
|
));
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
suppress_current_update_metadata = true;
|
suppress_current_update_metadata = true;
|
||||||
@@ -318,6 +335,8 @@ async fn chat_completion_chunk_stream(
|
|||||||
logprobs,
|
logprobs,
|
||||||
token_ids,
|
token_ids,
|
||||||
}) => {
|
}) => {
|
||||||
|
let delta_token_count = token_ids.len();
|
||||||
|
continuous_usage.add_output_tokens(delta_token_count);
|
||||||
let include_metadata =
|
let include_metadata =
|
||||||
!suppress_current_update_metadata && !inside_hidden_reasoning;
|
!suppress_current_update_metadata && !inside_hidden_reasoning;
|
||||||
suppress_current_update_metadata = false;
|
suppress_current_update_metadata = false;
|
||||||
@@ -339,16 +358,15 @@ async fn chat_completion_chunk_stream(
|
|||||||
if let Some(chunk) =
|
if let Some(chunk) =
|
||||||
pending_chunk.take_chunk(&request_id, &response_model, created)
|
pending_chunk.take_chunk(&request_id, &response_model, created)
|
||||||
{
|
{
|
||||||
y.yield_ok(chunk).await;
|
yield_chunk!(chunk);
|
||||||
}
|
}
|
||||||
} else if let Some(logprobs) = openai_logprobs {
|
} else if let Some(logprobs) = openai_logprobs {
|
||||||
y.yield_ok(logprobs_only_chunk(
|
yield_chunk!(logprobs_only_chunk(
|
||||||
&request_id,
|
&request_id,
|
||||||
&response_model,
|
&response_model,
|
||||||
created,
|
created,
|
||||||
logprobs,
|
logprobs,
|
||||||
))
|
));
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(ChatEvent::BlockStart { kind, .. }) => {
|
Ok(ChatEvent::BlockStart { kind, .. }) => {
|
||||||
@@ -376,15 +394,14 @@ async fn chat_completion_chunk_stream(
|
|||||||
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
||||||
pending_chunk.push_tool_call_start(tool_index, id, name);
|
pending_chunk.push_tool_call_start(tool_index, id, name);
|
||||||
} else {
|
} else {
|
||||||
y.yield_ok(tool_call_start_chunk(
|
yield_chunk!(tool_call_start_chunk(
|
||||||
&request_id,
|
&request_id,
|
||||||
&response_model,
|
&response_model,
|
||||||
created,
|
created,
|
||||||
tool_index,
|
tool_index,
|
||||||
id,
|
id,
|
||||||
name,
|
name,
|
||||||
))
|
));
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(ChatEvent::ToolCallArgumentsDelta { index, delta }) => {
|
Ok(ChatEvent::ToolCallArgumentsDelta { index, delta }) => {
|
||||||
@@ -392,21 +409,20 @@ async fn chat_completion_chunk_stream(
|
|||||||
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
||||||
pending_chunk.push_tool_call_arguments(tool_index, delta);
|
pending_chunk.push_tool_call_arguments(tool_index, delta);
|
||||||
} else {
|
} else {
|
||||||
y.yield_ok(tool_call_arguments_chunk(
|
yield_chunk!(tool_call_arguments_chunk(
|
||||||
&request_id,
|
&request_id,
|
||||||
&response_model,
|
&response_model,
|
||||||
created,
|
created,
|
||||||
tool_index,
|
tool_index,
|
||||||
delta,
|
delta,
|
||||||
))
|
));
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(ChatEvent::ToolCallEnd { .. }) => {
|
Ok(ChatEvent::ToolCallEnd { .. }) => {
|
||||||
debug!("ending current tool call");
|
debug!("ending current tool call");
|
||||||
}
|
}
|
||||||
Ok(ChatEvent::Done {
|
Ok(ChatEvent::Done {
|
||||||
usage,
|
usage: final_usage,
|
||||||
finish_reason,
|
finish_reason,
|
||||||
..
|
..
|
||||||
}) => {
|
}) => {
|
||||||
@@ -414,18 +430,23 @@ async fn chat_completion_chunk_stream(
|
|||||||
info!(
|
info!(
|
||||||
stream = true,
|
stream = true,
|
||||||
model = %response_model,
|
model = %response_model,
|
||||||
prompt_tokens = usage.prompt_token_count,
|
prompt_tokens = final_usage.prompt_token_count,
|
||||||
output_tokens = usage.output_token_count,
|
output_tokens = final_usage.output_token_count,
|
||||||
finish_reason = finish_reason.as_str(),
|
finish_reason = finish_reason.as_str(),
|
||||||
"chat completion finished"
|
"chat completion finished"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
continuous_usage.set_final_counts(
|
||||||
|
final_usage.prompt_token_count,
|
||||||
|
final_usage.output_token_count,
|
||||||
|
);
|
||||||
|
|
||||||
if let Some(pending_chunk) = pending_chunk.as_mut()
|
if let Some(pending_chunk) = pending_chunk.as_mut()
|
||||||
&& let Some(chunk) =
|
&& let Some(chunk) =
|
||||||
pending_chunk.take_chunk(&request_id, &response_model, created)
|
pending_chunk.take_chunk(&request_id, &response_model, created)
|
||||||
{
|
{
|
||||||
y.yield_ok(chunk).await;
|
yield_chunk!(chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
match final_chunk(
|
match final_chunk(
|
||||||
@@ -435,7 +456,7 @@ async fn chat_completion_chunk_stream(
|
|||||||
finish_reason,
|
finish_reason,
|
||||||
saw_tool_calls,
|
saw_tool_calls,
|
||||||
) {
|
) {
|
||||||
Ok(chunk) => y.yield_ok(chunk).await,
|
Ok(chunk) => yield_chunk!(chunk),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
error!(
|
error!(
|
||||||
error = %error.to_error_response().error.message,
|
error = %error.to_error_response().error.message,
|
||||||
@@ -450,7 +471,7 @@ async fn chat_completion_chunk_stream(
|
|||||||
&request_id,
|
&request_id,
|
||||||
&response_model,
|
&response_model,
|
||||||
created,
|
created,
|
||||||
Usage::from_token_usage(usage, enable_prompt_tokens_details),
|
Usage::from_token_usage(final_usage, enable_prompt_tokens_details),
|
||||||
))
|
))
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,6 +33,8 @@ pub(super) struct PreparedRequest {
|
|||||||
pub(super) struct ResponseOptions {
|
pub(super) struct ResponseOptions {
|
||||||
/// Whether the caller asked for the final streamed usage chunk.
|
/// Whether the caller asked for the final streamed usage chunk.
|
||||||
pub include_usage: bool,
|
pub include_usage: bool,
|
||||||
|
/// Whether every streamed chunk should carry cumulative usage.
|
||||||
|
pub include_continuous_usage: bool,
|
||||||
/// Whether the caller requested output logprobs on chat choices.
|
/// Whether the caller requested output logprobs on chat choices.
|
||||||
pub requested_logprobs: bool,
|
pub requested_logprobs: bool,
|
||||||
/// Whether the caller requested top-level prompt logprobs.
|
/// Whether the caller requested top-level prompt logprobs.
|
||||||
@@ -82,6 +84,12 @@ pub(super) fn prepare_chat_request(
|
|||||||
let include_usage = (request.stream_options.as_ref())
|
let include_usage = (request.stream_options.as_ref())
|
||||||
.and_then(|options| options.include_usage)
|
.and_then(|options| options.include_usage)
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
|
let include_continuous_usage = include_usage
|
||||||
|
&& request
|
||||||
|
.stream_options
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|options| options.continuous_usage_stats)
|
||||||
|
.unwrap_or(false);
|
||||||
let requested_logprobs = request.logprobs;
|
let requested_logprobs = request.logprobs;
|
||||||
|
|
||||||
// Auto-enable prompt logprobs for non-streaming echo, matching Python vLLM's
|
// Auto-enable prompt logprobs for non-streaming echo, matching Python vLLM's
|
||||||
@@ -154,6 +162,7 @@ pub(super) fn prepare_chat_request(
|
|||||||
response_model,
|
response_model,
|
||||||
options: ResponseOptions {
|
options: ResponseOptions {
|
||||||
include_usage,
|
include_usage,
|
||||||
|
include_continuous_usage,
|
||||||
requested_logprobs,
|
requested_logprobs,
|
||||||
include_prompt_logprobs,
|
include_prompt_logprobs,
|
||||||
include_reasoning,
|
include_reasoning,
|
||||||
@@ -375,8 +384,8 @@ mod tests {
|
|||||||
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
|
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
|
||||||
};
|
};
|
||||||
use crate::routes::openai::utils::types::{
|
use crate::routes::openai::utils::types::{
|
||||||
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent, Tool,
|
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent,
|
||||||
ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
|
StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
|
||||||
};
|
};
|
||||||
use crate::utils::{ResolvedRequestContext, resolve_request_context};
|
use crate::utils::{ResolvedRequestContext, resolve_request_context};
|
||||||
|
|
||||||
@@ -456,6 +465,46 @@ mod tests {
|
|||||||
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
|
assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prepare_chat_request_maps_stream_usage_and_token_format_options() {
|
||||||
|
let mut request = base_request();
|
||||||
|
request.return_tokens_as_token_ids = Some(true);
|
||||||
|
request.stream_options = Some(StreamOptions {
|
||||||
|
include_usage: Some(true),
|
||||||
|
continuous_usage_stats: Some(true),
|
||||||
|
});
|
||||||
|
|
||||||
|
let prepared = prepare_chat_request(
|
||||||
|
request,
|
||||||
|
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||||
|
ResolvedRequestContext::default(),
|
||||||
|
)
|
||||||
|
.expect("request is valid");
|
||||||
|
|
||||||
|
assert!(prepared.options.include_usage);
|
||||||
|
assert!(prepared.options.include_continuous_usage);
|
||||||
|
assert!(prepared.options.return_tokens_as_token_ids);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prepare_chat_request_gates_continuous_usage_on_include_usage() {
|
||||||
|
let mut request = base_request();
|
||||||
|
request.stream_options = Some(StreamOptions {
|
||||||
|
include_usage: None,
|
||||||
|
continuous_usage_stats: Some(true),
|
||||||
|
});
|
||||||
|
|
||||||
|
let prepared = prepare_chat_request(
|
||||||
|
request,
|
||||||
|
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||||
|
ResolvedRequestContext::default(),
|
||||||
|
)
|
||||||
|
.expect("request is valid");
|
||||||
|
|
||||||
|
assert!(!prepared.options.include_usage);
|
||||||
|
assert!(!prepared.options.include_continuous_usage);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn prepare_chat_request_keeps_optional_sampling_fields_unset() {
|
fn prepare_chat_request_keeps_optional_sampling_fields_unset() {
|
||||||
let prepared = prepare_chat_request(
|
let prepared = prepare_chat_request(
|
||||||
|
|||||||
@@ -137,15 +137,6 @@ pub(super) fn validate_request_compat(
|
|||||||
"repetition_detection is not supported.",
|
"repetition_detection is not supported.",
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
if let Some(options) = &request.stream_options
|
|
||||||
&& options.continuous_usage_stats.is_some()
|
|
||||||
{
|
|
||||||
bail_invalid_request!(
|
|
||||||
param = "stream_options",
|
|
||||||
"continuous_usage_stats is not supported."
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ use crate::routes::openai::completions::types::{
|
|||||||
CompletionStreamChoice, CompletionStreamResponse,
|
CompletionStreamChoice, CompletionStreamResponse,
|
||||||
};
|
};
|
||||||
use crate::routes::openai::utils::types::LogProbs;
|
use crate::routes::openai::utils::types::LogProbs;
|
||||||
|
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
use crate::utils::{resolve_request_context, unix_timestamp};
|
use crate::utils::{resolve_request_context, unix_timestamp};
|
||||||
@@ -127,6 +128,8 @@ async fn collect_completion(
|
|||||||
ResponseOptions {
|
ResponseOptions {
|
||||||
// Ignored: non-streaming responses always include usage.
|
// Ignored: non-streaming responses always include usage.
|
||||||
include_usage: _,
|
include_usage: _,
|
||||||
|
// Ignored: non-streaming responses are collected before usage is attached.
|
||||||
|
include_continuous_usage: _,
|
||||||
echo,
|
echo,
|
||||||
requested_logprobs,
|
requested_logprobs,
|
||||||
include_prompt_logprobs,
|
include_prompt_logprobs,
|
||||||
@@ -218,6 +221,7 @@ async fn completion_chunk_stream(
|
|||||||
}: ApiServerOptions,
|
}: ApiServerOptions,
|
||||||
ResponseOptions {
|
ResponseOptions {
|
||||||
include_usage,
|
include_usage,
|
||||||
|
include_continuous_usage,
|
||||||
echo,
|
echo,
|
||||||
requested_logprobs,
|
requested_logprobs,
|
||||||
// Ignored: streaming prompt logprobs are rejected for Python parity.
|
// Ignored: streaming prompt logprobs are rejected for Python parity.
|
||||||
@@ -230,6 +234,18 @@ async fn completion_chunk_stream(
|
|||||||
pin_mut!(stream);
|
pin_mut!(stream);
|
||||||
let mut visible_text_len = 0_u32;
|
let mut visible_text_len = 0_u32;
|
||||||
let mut first_chunk = true;
|
let mut first_chunk = true;
|
||||||
|
let mut continuous_usage = ContinuousUsage::default();
|
||||||
|
|
||||||
|
/// Yield a chunk with optional continuous usage attached.
|
||||||
|
macro_rules! yield_chunk {
|
||||||
|
($chunk:expr) => {{
|
||||||
|
let mut chunk = $chunk;
|
||||||
|
if include_continuous_usage {
|
||||||
|
chunk.usage = Some(continuous_usage.to_usage());
|
||||||
|
}
|
||||||
|
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
|
||||||
while let Some(next) = stream.next().await {
|
while let Some(next) = stream.next().await {
|
||||||
match next {
|
match next {
|
||||||
@@ -237,6 +253,7 @@ async fn completion_chunk_stream(
|
|||||||
prompt_token_ids, ..
|
prompt_token_ids, ..
|
||||||
}) => {
|
}) => {
|
||||||
debug!("completion stream started");
|
debug!("completion stream started");
|
||||||
|
continuous_usage.set_prompt_tokens(prompt_token_ids.len());
|
||||||
if let Some(prompt) = echo.as_ref() {
|
if let Some(prompt) = echo.as_ref() {
|
||||||
visible_text_len = text_len(prompt);
|
visible_text_len = text_len(prompt);
|
||||||
let mut chunk =
|
let mut chunk =
|
||||||
@@ -247,7 +264,7 @@ async fn completion_chunk_stream(
|
|||||||
}
|
}
|
||||||
first_chunk = false;
|
first_chunk = false;
|
||||||
}
|
}
|
||||||
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
|
yield_chunk!(chunk);
|
||||||
} else if return_token_ids {
|
} else if return_token_ids {
|
||||||
// Emit a chunk with prompt_token_ids in the first streaming response
|
// Emit a chunk with prompt_token_ids in the first streaming response
|
||||||
let mut chunk =
|
let mut chunk =
|
||||||
@@ -256,7 +273,7 @@ async fn completion_chunk_stream(
|
|||||||
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
||||||
}
|
}
|
||||||
first_chunk = false;
|
first_chunk = false;
|
||||||
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
|
yield_chunk!(chunk);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(DecodedTextEvent::TextDelta {
|
Ok(DecodedTextEvent::TextDelta {
|
||||||
@@ -281,10 +298,12 @@ async fn completion_chunk_stream(
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
let mut chunk = delta_chunk(&request_id, &response_model, created, delta, logprobs);
|
let mut chunk = delta_chunk(&request_id, &response_model, created, delta, logprobs);
|
||||||
|
let delta_token_count = token_ids.len();
|
||||||
|
continuous_usage.add_output_tokens(delta_token_count);
|
||||||
if return_token_ids && let Some(choice) = chunk.choices.first_mut() {
|
if return_token_ids && let Some(choice) = chunk.choices.first_mut() {
|
||||||
choice.token_ids = Some(token_ids);
|
choice.token_ids = Some(token_ids);
|
||||||
}
|
}
|
||||||
y.yield_ok(CompletionSseChunk::Chunk(chunk)).await;
|
yield_chunk!(chunk);
|
||||||
visible_text_len = visible_text_len.saturating_add(delta_text_len);
|
visible_text_len = visible_text_len.saturating_add(delta_text_len);
|
||||||
|
|
||||||
if let Some(finished) = finished {
|
if let Some(finished) = finished {
|
||||||
@@ -298,13 +317,17 @@ async fn completion_chunk_stream(
|
|||||||
"completion finished"
|
"completion finished"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
y.yield_ok(CompletionSseChunk::Chunk(final_chunk(
|
continuous_usage.set_final_counts(
|
||||||
|
finished.usage.prompt_token_count,
|
||||||
|
finished.usage.output_token_count,
|
||||||
|
);
|
||||||
|
let final_chunk = final_chunk(
|
||||||
&request_id,
|
&request_id,
|
||||||
&response_model,
|
&response_model,
|
||||||
created,
|
created,
|
||||||
finished.finish_reason,
|
finished.finish_reason,
|
||||||
)?))
|
)?;
|
||||||
.await;
|
yield_chunk!(final_chunk);
|
||||||
|
|
||||||
if include_usage {
|
if include_usage {
|
||||||
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
|
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ pub(super) struct PreparedRequest {
|
|||||||
pub(super) struct ResponseOptions {
|
pub(super) struct ResponseOptions {
|
||||||
/// Whether the caller asked for the final streamed usage chunk.
|
/// Whether the caller asked for the final streamed usage chunk.
|
||||||
pub include_usage: bool,
|
pub include_usage: bool,
|
||||||
|
/// Whether every streamed chunk should carry cumulative usage.
|
||||||
|
pub include_continuous_usage: bool,
|
||||||
/// Original text prompt that should be echoed back northbound when
|
/// Original text prompt that should be echoed back northbound when
|
||||||
/// `echo=true`.
|
/// `echo=true`.
|
||||||
pub echo: Option<String>,
|
pub echo: Option<String>,
|
||||||
@@ -74,6 +76,12 @@ pub(super) fn prepare_completion_request(
|
|||||||
let include_usage = (request.stream_options.as_ref())
|
let include_usage = (request.stream_options.as_ref())
|
||||||
.and_then(|options| options.include_usage)
|
.and_then(|options| options.include_usage)
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
|
let include_continuous_usage = include_usage
|
||||||
|
&& request
|
||||||
|
.stream_options
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|options| options.continuous_usage_stats)
|
||||||
|
.unwrap_or(false);
|
||||||
let include_prompt_logprobs = prompt_logprobs.is_some();
|
let include_prompt_logprobs = prompt_logprobs.is_some();
|
||||||
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
|
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
|
||||||
|
|
||||||
@@ -129,6 +137,7 @@ pub(super) fn prepare_completion_request(
|
|||||||
response_model,
|
response_model,
|
||||||
options: ResponseOptions {
|
options: ResponseOptions {
|
||||||
include_usage,
|
include_usage,
|
||||||
|
include_continuous_usage,
|
||||||
echo,
|
echo,
|
||||||
requested_logprobs: request.logprobs,
|
requested_logprobs: request.logprobs,
|
||||||
include_prompt_logprobs,
|
include_prompt_logprobs,
|
||||||
@@ -247,6 +256,55 @@ mod tests {
|
|||||||
assert!(!prepared.text_request.decode_options.skip_special_tokens);
|
assert!(!prepared.text_request.decode_options.skip_special_tokens);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prepare_completion_request_maps_stream_usage_and_token_format_options() {
|
||||||
|
let request: CompletionRequest = serde_json::from_value(json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"prompt": "hello",
|
||||||
|
"stream": true,
|
||||||
|
"stream_options": {
|
||||||
|
"include_usage": true,
|
||||||
|
"continuous_usage_stats": true
|
||||||
|
},
|
||||||
|
"return_tokens_as_token_ids": true
|
||||||
|
}))
|
||||||
|
.expect("parse request");
|
||||||
|
|
||||||
|
let prepared = prepare_completion_request(
|
||||||
|
request,
|
||||||
|
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||||
|
ResolvedRequestContext::default(),
|
||||||
|
)
|
||||||
|
.expect("prepare");
|
||||||
|
|
||||||
|
assert!(prepared.options.include_usage);
|
||||||
|
assert!(prepared.options.include_continuous_usage);
|
||||||
|
assert!(prepared.options.return_tokens_as_token_ids);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn prepare_completion_request_gates_continuous_usage_on_include_usage() {
|
||||||
|
let request: CompletionRequest = serde_json::from_value(json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"prompt": "hello",
|
||||||
|
"stream": true,
|
||||||
|
"stream_options": {
|
||||||
|
"continuous_usage_stats": true
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.expect("parse request");
|
||||||
|
|
||||||
|
let prepared = prepare_completion_request(
|
||||||
|
request,
|
||||||
|
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||||
|
ResolvedRequestContext::default(),
|
||||||
|
)
|
||||||
|
.expect("prepare");
|
||||||
|
|
||||||
|
assert!(!prepared.options.include_usage);
|
||||||
|
assert!(!prepared.options.include_continuous_usage);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn prepare_completion_request_accepts_text_echo() {
|
fn prepare_completion_request_accepts_text_echo() {
|
||||||
let request: CompletionRequest = serde_json::from_value(json!({
|
let request: CompletionRequest = serde_json::from_value(json!({
|
||||||
|
|||||||
@@ -95,15 +95,6 @@ pub(super) fn validate_request_compat(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(options) = &request.stream_options
|
|
||||||
&& options.continuous_usage_stats.is_some()
|
|
||||||
{
|
|
||||||
bail_invalid_request!(
|
|
||||||
param = "stream_options",
|
|
||||||
"continuous_usage_stats is not supported."
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,4 +2,5 @@ pub mod logprobs;
|
|||||||
pub mod structured_outputs;
|
pub mod structured_outputs;
|
||||||
pub mod token_ids;
|
pub mod token_ids;
|
||||||
pub mod types;
|
pub mod types;
|
||||||
|
pub mod usage;
|
||||||
pub mod validated_json;
|
pub mod validated_json;
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
use super::types::Usage;
|
||||||
|
|
||||||
|
/// Tracks cumulative token counts for OpenAI streaming chunks.
|
||||||
|
///
|
||||||
|
/// This helper is intentionally only a counter. Callers decide whether to
|
||||||
|
/// attach `counts()` to each streamed data chunk, while final usage-only chunks
|
||||||
|
/// should still be built from the authoritative terminal `TokenUsage`.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub(crate) struct ContinuousUsage {
|
||||||
|
prompt_tokens: usize,
|
||||||
|
output_tokens: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ContinuousUsage {
|
||||||
|
/// Record the prompt-token count reported when a stream starts.
|
||||||
|
pub(crate) fn set_prompt_tokens(&mut self, prompt_tokens: usize) {
|
||||||
|
self.prompt_tokens = prompt_tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add newly decoded output tokens to the running completion count.
|
||||||
|
pub(crate) fn add_output_tokens(&mut self, output_tokens: usize) {
|
||||||
|
self.output_tokens = self.output_tokens.saturating_add(output_tokens);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Replace the running counts with the final counts reported by generation.
|
||||||
|
pub(crate) fn set_final_counts(&mut self, prompt_tokens: usize, output_tokens: usize) {
|
||||||
|
self.prompt_tokens = prompt_tokens;
|
||||||
|
self.output_tokens = output_tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build a streaming usage snapshot without prompt cache details.
|
||||||
|
pub(crate) fn to_usage(&self) -> Usage {
|
||||||
|
Usage::from_counts(self.prompt_tokens, self.output_tokens, None)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -151,6 +151,14 @@ fn sse_data_payloads(text: &str) -> Vec<&str> {
|
|||||||
text.lines().filter_map(|line| line.strip_prefix("data: ")).collect()
|
text.lines().filter_map(|line| line.strip_prefix("data: ")).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sse_json_payloads(text: &str) -> Vec<serde_json::Value> {
|
||||||
|
sse_data_payloads(text)
|
||||||
|
.into_iter()
|
||||||
|
.filter(|payload| *payload != "[DONE]")
|
||||||
|
.map(|payload| serde_json::from_str(payload).expect("sse json payload"))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
type TestFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
type TestFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||||
|
|
||||||
fn boxed_test_future<'a>(future: impl Future<Output = ()> + Send + 'a) -> TestFuture<'a> {
|
fn boxed_test_future<'a>(future: impl Future<Output = ()> + Send + 'a) -> TestFuture<'a> {
|
||||||
@@ -2341,6 +2349,60 @@ async fn include_usage_adds_final_usage_chunk_before_done() {
|
|||||||
assert_eq!(usage_chunk["usage"]["total_tokens"], 25);
|
assert_eq!(usage_chunk["usage"]["total_tokens"], 25);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn stream_continuous_usage_stats_adds_usage_to_chat_chunks() {
|
||||||
|
let (app, engine_task) = test_app_with_stream_output_specs(default_stream_output_specs()).await;
|
||||||
|
let response = app
|
||||||
|
.clone()
|
||||||
|
.call(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/v1/chat/completions")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from(
|
||||||
|
json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"stream": true,
|
||||||
|
"stream_options": {
|
||||||
|
"include_usage": true,
|
||||||
|
"continuous_usage_stats": true
|
||||||
|
},
|
||||||
|
"messages": [{"role": "user", "content": "hello"}]
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
))
|
||||||
|
.expect("build request"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("call app");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||||
|
engine_task.await.expect("mock engine task");
|
||||||
|
let text = String::from_utf8(body.to_vec()).expect("utf8 body");
|
||||||
|
let payloads = sse_json_payloads(&text);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
payloads.iter().all(|payload| payload.get("usage").is_some()),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
payloads.iter().any(|payload| {
|
||||||
|
payload["choices"].as_array().is_some_and(|choices| !choices.is_empty())
|
||||||
|
&& payload["usage"]["completion_tokens"] == json!(1)
|
||||||
|
}),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
let usage_chunk = payloads
|
||||||
|
.iter()
|
||||||
|
.find(|payload| payload["choices"] == json!([]))
|
||||||
|
.expect("final usage chunk");
|
||||||
|
assert_eq!(usage_chunk["usage"]["prompt_tokens"], 22);
|
||||||
|
assert_eq!(usage_chunk["usage"]["completion_tokens"], 3);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn stream_without_include_usage_keeps_existing_shape() {
|
async fn stream_without_include_usage_keeps_existing_shape() {
|
||||||
@@ -3434,6 +3496,59 @@ async fn completions_happy_path_returns_sse_stream() {
|
|||||||
assert_eq!(usage_chunk["usage"]["completion_tokens"], 3);
|
assert_eq!(usage_chunk["usage"]["completion_tokens"], 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn completions_stream_continuous_usage_stats_adds_usage_to_chunks() {
|
||||||
|
let (app, engine_task) = test_app_with_stream_output_specs(default_stream_output_specs()).await;
|
||||||
|
let response = app
|
||||||
|
.clone()
|
||||||
|
.call(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/v1/completions")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from(
|
||||||
|
json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"prompt": "hello",
|
||||||
|
"stream": true,
|
||||||
|
"stream_options": {
|
||||||
|
"include_usage": true,
|
||||||
|
"continuous_usage_stats": true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
))
|
||||||
|
.expect("build request"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("call app");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||||
|
engine_task.await.expect("mock engine task");
|
||||||
|
let text = String::from_utf8(body.to_vec()).expect("utf8 body");
|
||||||
|
let payloads = sse_json_payloads(&text);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
payloads.iter().all(|payload| payload.get("usage").is_some()),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
payloads.iter().any(|payload| {
|
||||||
|
payload["choices"].as_array().is_some_and(|choices| !choices.is_empty())
|
||||||
|
&& payload["usage"]["completion_tokens"] == json!(1)
|
||||||
|
}),
|
||||||
|
"{text}"
|
||||||
|
);
|
||||||
|
let usage_chunk = payloads
|
||||||
|
.iter()
|
||||||
|
.find(|payload| payload["choices"] == json!([]))
|
||||||
|
.expect("final usage chunk");
|
||||||
|
assert_eq!(usage_chunk["usage"]["completion_tokens"], 3);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn completions_echo_stream_emits_separate_prompt_chunk() {
|
async fn completions_echo_stream_emits_separate_prompt_chunk() {
|
||||||
|
|||||||
Reference in New Issue
Block a user