diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index e93c049b2d..6274a4e98a 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -37,6 +37,7 @@ use crate::routes::openai::utils::logprobs::{ use crate::routes::openai::utils::types::{ ChatLogProbs, FunctionCallDelta, FunctionCallResponse, ToolCall, ToolCallDelta, Usage, }; +use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; use crate::utils::{resolve_request_context, unix_timestamp}; @@ -129,6 +130,8 @@ async fn collect_chat_completion( ResponseOptions { // Ignored: non-streaming responses always include usage. include_usage: _, + // Ignored: non-streaming responses are collected before usage is attached. + include_continuous_usage: _, requested_logprobs, include_prompt_logprobs, include_reasoning, @@ -249,6 +252,7 @@ async fn chat_completion_chunk_stream( }: ApiServerOptions, ResponseOptions { include_usage, + include_continuous_usage, requested_logprobs, // Ignored: chat streaming prompt logprobs are rejected for Python parity. include_prompt_logprobs: _, @@ -265,33 +269,47 @@ async fn chat_completion_chunk_stream( // starts or ends, omit its token metadata as well as its visible delta. let mut inside_hidden_reasoning = false; let mut suppress_current_update_metadata = false; + let mut continuous_usage = ContinuousUsage::default(); + + /// Yield a chunk with optional continuous usage attached. + macro_rules! yield_chunk { + ($chunk:expr) => {{ + let mut chunk = $chunk; + if include_continuous_usage { + chunk.usage = Some(continuous_usage.to_usage()); + } + y.yield_ok(chunk).await; + }}; + } // If the client requested logprobs or token_ids, we need to buffer chunks until // we receive the separate `LogprobsDelta` event, so that we can emit one // combined chunk with both the semantic delta and its per-update metadata. - let mut pending_chunk = - (requested_logprobs || return_token_ids).then(PendingChatChunk::default); + // Continuous usage also buffers so the token count from `LogprobsDelta` can + // be attached to the matching semantic chunk. + let mut pending_chunk = (requested_logprobs || return_token_ids || include_continuous_usage) + .then(PendingChatChunk::default); while let Some(next) = stream.next().await { match next { Ok(ChatEvent::Start { prompt_token_ids, .. }) => { + continuous_usage.set_prompt_tokens(prompt_token_ids.len()); let mut chunk = start_chunk(&request_id, &response_model, created); if return_token_ids { chunk.prompt_token_ids = Some(prompt_token_ids.to_vec()); } - y.yield_ok(chunk).await; + yield_chunk!(chunk); // When echo=true, emit the last assistant message content as a delta chunk. if let Some(echo_text) = &echo { - y.yield_ok(block_delta_chunk( + yield_chunk!(block_delta_chunk( &request_id, &response_model, created, AssistantBlockKind::Text, echo_text.clone(), - )) - .await; + )); } } Ok(ChatEvent::BlockDelta { kind, delta, .. }) => { @@ -301,14 +319,13 @@ async fn chat_completion_chunk_stream( if let Some(pending_chunk) = pending_chunk.as_mut() { pending_chunk.push_block_delta(kind, delta); } else { - y.yield_ok(block_delta_chunk( + yield_chunk!(block_delta_chunk( &request_id, &response_model, created, kind, delta, - )) - .await; + )); } } else { suppress_current_update_metadata = true; @@ -318,6 +335,8 @@ async fn chat_completion_chunk_stream( logprobs, token_ids, }) => { + let delta_token_count = token_ids.len(); + continuous_usage.add_output_tokens(delta_token_count); let include_metadata = !suppress_current_update_metadata && !inside_hidden_reasoning; suppress_current_update_metadata = false; @@ -339,16 +358,15 @@ async fn chat_completion_chunk_stream( if let Some(chunk) = pending_chunk.take_chunk(&request_id, &response_model, created) { - y.yield_ok(chunk).await; + yield_chunk!(chunk); } } else if let Some(logprobs) = openai_logprobs { - y.yield_ok(logprobs_only_chunk( + yield_chunk!(logprobs_only_chunk( &request_id, &response_model, created, logprobs, - )) - .await; + )); } } Ok(ChatEvent::BlockStart { kind, .. }) => { @@ -376,15 +394,14 @@ async fn chat_completion_chunk_stream( if let Some(pending_chunk) = pending_chunk.as_mut() { pending_chunk.push_tool_call_start(tool_index, id, name); } else { - y.yield_ok(tool_call_start_chunk( + yield_chunk!(tool_call_start_chunk( &request_id, &response_model, created, tool_index, id, name, - )) - .await; + )); } } Ok(ChatEvent::ToolCallArgumentsDelta { index, delta }) => { @@ -392,21 +409,20 @@ async fn chat_completion_chunk_stream( if let Some(pending_chunk) = pending_chunk.as_mut() { pending_chunk.push_tool_call_arguments(tool_index, delta); } else { - y.yield_ok(tool_call_arguments_chunk( + yield_chunk!(tool_call_arguments_chunk( &request_id, &response_model, created, tool_index, delta, - )) - .await; + )); } } Ok(ChatEvent::ToolCallEnd { .. }) => { debug!("ending current tool call"); } Ok(ChatEvent::Done { - usage, + usage: final_usage, finish_reason, .. }) => { @@ -414,18 +430,23 @@ async fn chat_completion_chunk_stream( info!( stream = true, model = %response_model, - prompt_tokens = usage.prompt_token_count, - output_tokens = usage.output_token_count, + prompt_tokens = final_usage.prompt_token_count, + output_tokens = final_usage.output_token_count, finish_reason = finish_reason.as_str(), "chat completion finished" ); } + continuous_usage.set_final_counts( + final_usage.prompt_token_count, + final_usage.output_token_count, + ); + if let Some(pending_chunk) = pending_chunk.as_mut() && let Some(chunk) = pending_chunk.take_chunk(&request_id, &response_model, created) { - y.yield_ok(chunk).await; + yield_chunk!(chunk); } match final_chunk( @@ -435,7 +456,7 @@ async fn chat_completion_chunk_stream( finish_reason, saw_tool_calls, ) { - Ok(chunk) => y.yield_ok(chunk).await, + Ok(chunk) => yield_chunk!(chunk), Err(error) => { error!( error = %error.to_error_response().error.message, @@ -450,7 +471,7 @@ async fn chat_completion_chunk_stream( &request_id, &response_model, created, - Usage::from_token_usage(usage, enable_prompt_tokens_details), + Usage::from_token_usage(final_usage, enable_prompt_tokens_details), )) .await; } diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 2b3e3ddb36..aa430db76c 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -33,6 +33,8 @@ pub(super) struct PreparedRequest { pub(super) struct ResponseOptions { /// Whether the caller asked for the final streamed usage chunk. pub include_usage: bool, + /// Whether every streamed chunk should carry cumulative usage. + pub include_continuous_usage: bool, /// Whether the caller requested output logprobs on chat choices. pub requested_logprobs: bool, /// Whether the caller requested top-level prompt logprobs. @@ -82,6 +84,12 @@ pub(super) fn prepare_chat_request( let include_usage = (request.stream_options.as_ref()) .and_then(|options| options.include_usage) .unwrap_or(false); + let include_continuous_usage = include_usage + && request + .stream_options + .as_ref() + .and_then(|options| options.continuous_usage_stats) + .unwrap_or(false); let requested_logprobs = request.logprobs; // Auto-enable prompt logprobs for non-streaming echo, matching Python vLLM's @@ -154,6 +162,7 @@ pub(super) fn prepare_chat_request( response_model, options: ResponseOptions { include_usage, + include_continuous_usage, requested_logprobs, include_prompt_logprobs, include_reasoning, @@ -375,8 +384,8 @@ mod tests { AssistantRole, ChatCompletionMessage, ChatCompletionRequest, }; use crate::routes::openai::utils::types::{ - ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent, Tool, - ToolCall, ToolChoice, ToolChoiceValue, VideoUrl, + ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent, + StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl, }; use crate::utils::{ResolvedRequestContext, resolve_request_context}; @@ -456,6 +465,46 @@ mod tests { assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto); } + #[test] + fn prepare_chat_request_maps_stream_usage_and_token_format_options() { + let mut request = base_request(); + request.return_tokens_as_token_ids = Some(true); + request.stream_options = Some(StreamOptions { + include_usage: Some(true), + continuous_usage_stats: Some(true), + }); + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert!(prepared.options.include_usage); + assert!(prepared.options.include_continuous_usage); + assert!(prepared.options.return_tokens_as_token_ids); + } + + #[test] + fn prepare_chat_request_gates_continuous_usage_on_include_usage() { + let mut request = base_request(); + request.stream_options = Some(StreamOptions { + include_usage: None, + continuous_usage_stats: Some(true), + }); + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert!(!prepared.options.include_usage); + assert!(!prepared.options.include_continuous_usage); + } + #[test] fn prepare_chat_request_keeps_optional_sampling_fields_unset() { let prepared = prepare_chat_request( diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index fb64428e4b..a623925e64 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -137,15 +137,6 @@ pub(super) fn validate_request_compat( "repetition_detection is not supported.", )?; - if let Some(options) = &request.stream_options - && options.continuous_usage_stats.is_some() - { - bail_invalid_request!( - param = "stream_options", - "continuous_usage_stats is not supported." - ); - } - Ok(()) } diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index b6e4383c7d..9dc2e19154 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -31,6 +31,7 @@ use crate::routes::openai::completions::types::{ CompletionStreamChoice, CompletionStreamResponse, }; use crate::routes::openai::utils::types::LogProbs; +use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; use crate::utils::{resolve_request_context, unix_timestamp}; @@ -127,6 +128,8 @@ async fn collect_completion( ResponseOptions { // Ignored: non-streaming responses always include usage. include_usage: _, + // Ignored: non-streaming responses are collected before usage is attached. + include_continuous_usage: _, echo, requested_logprobs, include_prompt_logprobs, @@ -218,6 +221,7 @@ async fn completion_chunk_stream( }: ApiServerOptions, ResponseOptions { include_usage, + include_continuous_usage, echo, requested_logprobs, // Ignored: streaming prompt logprobs are rejected for Python parity. @@ -230,6 +234,18 @@ async fn completion_chunk_stream( pin_mut!(stream); let mut visible_text_len = 0_u32; let mut first_chunk = true; + let mut continuous_usage = ContinuousUsage::default(); + + /// Yield a chunk with optional continuous usage attached. + macro_rules! yield_chunk { + ($chunk:expr) => {{ + let mut chunk = $chunk; + if include_continuous_usage { + chunk.usage = Some(continuous_usage.to_usage()); + } + y.yield_ok(CompletionSseChunk::Chunk(chunk)).await; + }}; + } while let Some(next) = stream.next().await { match next { @@ -237,6 +253,7 @@ async fn completion_chunk_stream( prompt_token_ids, .. }) => { debug!("completion stream started"); + continuous_usage.set_prompt_tokens(prompt_token_ids.len()); if let Some(prompt) = echo.as_ref() { visible_text_len = text_len(prompt); let mut chunk = @@ -247,7 +264,7 @@ async fn completion_chunk_stream( } first_chunk = false; } - y.yield_ok(CompletionSseChunk::Chunk(chunk)).await; + yield_chunk!(chunk); } else if return_token_ids { // Emit a chunk with prompt_token_ids in the first streaming response let mut chunk = @@ -256,7 +273,7 @@ async fn completion_chunk_stream( choice.prompt_token_ids = Some(prompt_token_ids.to_vec()); } first_chunk = false; - y.yield_ok(CompletionSseChunk::Chunk(chunk)).await; + yield_chunk!(chunk); } } Ok(DecodedTextEvent::TextDelta { @@ -281,10 +298,12 @@ async fn completion_chunk_stream( None }; let mut chunk = delta_chunk(&request_id, &response_model, created, delta, logprobs); + let delta_token_count = token_ids.len(); + continuous_usage.add_output_tokens(delta_token_count); if return_token_ids && let Some(choice) = chunk.choices.first_mut() { choice.token_ids = Some(token_ids); } - y.yield_ok(CompletionSseChunk::Chunk(chunk)).await; + yield_chunk!(chunk); visible_text_len = visible_text_len.saturating_add(delta_text_len); if let Some(finished) = finished { @@ -298,13 +317,17 @@ async fn completion_chunk_stream( "completion finished" ); } - y.yield_ok(CompletionSseChunk::Chunk(final_chunk( + continuous_usage.set_final_counts( + finished.usage.prompt_token_count, + finished.usage.output_token_count, + ); + let final_chunk = final_chunk( &request_id, &response_model, created, finished.finish_reason, - )?)) - .await; + )?; + yield_chunk!(final_chunk); if include_usage { y.yield_ok(CompletionSseChunk::Usage(usage_chunk( diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 1dd73a4f53..2f6c760a99 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -25,6 +25,8 @@ pub(super) struct PreparedRequest { pub(super) struct ResponseOptions { /// Whether the caller asked for the final streamed usage chunk. pub include_usage: bool, + /// Whether every streamed chunk should carry cumulative usage. + pub include_continuous_usage: bool, /// Original text prompt that should be echoed back northbound when /// `echo=true`. pub echo: Option, @@ -74,6 +76,12 @@ pub(super) fn prepare_completion_request( let include_usage = (request.stream_options.as_ref()) .and_then(|options| options.include_usage) .unwrap_or(false); + let include_continuous_usage = include_usage + && request + .stream_options + .as_ref() + .and_then(|options| options.continuous_usage_stats) + .unwrap_or(false); let include_prompt_logprobs = prompt_logprobs.is_some(); let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten(); @@ -129,6 +137,7 @@ pub(super) fn prepare_completion_request( response_model, options: ResponseOptions { include_usage, + include_continuous_usage, echo, requested_logprobs: request.logprobs, include_prompt_logprobs, @@ -247,6 +256,55 @@ mod tests { assert!(!prepared.text_request.decode_options.skip_special_tokens); } + #[test] + fn prepare_completion_request_maps_stream_usage_and_token_format_options() { + let request: CompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "stream": true, + "stream_options": { + "include_usage": true, + "continuous_usage_stats": true + }, + "return_tokens_as_token_ids": true + })) + .expect("parse request"); + + let prepared = prepare_completion_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare"); + + assert!(prepared.options.include_usage); + assert!(prepared.options.include_continuous_usage); + assert!(prepared.options.return_tokens_as_token_ids); + } + + #[test] + fn prepare_completion_request_gates_continuous_usage_on_include_usage() { + let request: CompletionRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "stream": true, + "stream_options": { + "continuous_usage_stats": true + } + })) + .expect("parse request"); + + let prepared = prepare_completion_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("prepare"); + + assert!(!prepared.options.include_usage); + assert!(!prepared.options.include_continuous_usage); + } + #[test] fn prepare_completion_request_accepts_text_echo() { let request: CompletionRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/openai/completions/validate.rs b/rust/src/server/src/routes/openai/completions/validate.rs index 2af8c8add1..2af41877bf 100644 --- a/rust/src/server/src/routes/openai/completions/validate.rs +++ b/rust/src/server/src/routes/openai/completions/validate.rs @@ -95,15 +95,6 @@ pub(super) fn validate_request_compat( ); } - if let Some(options) = &request.stream_options - && options.continuous_usage_stats.is_some() - { - bail_invalid_request!( - param = "stream_options", - "continuous_usage_stats is not supported." - ); - } - Ok(()) } diff --git a/rust/src/server/src/routes/openai/utils/mod.rs b/rust/src/server/src/routes/openai/utils/mod.rs index 039df87f9d..7ec1251ddf 100644 --- a/rust/src/server/src/routes/openai/utils/mod.rs +++ b/rust/src/server/src/routes/openai/utils/mod.rs @@ -2,4 +2,5 @@ pub mod logprobs; pub mod structured_outputs; pub mod token_ids; pub mod types; +pub mod usage; pub mod validated_json; diff --git a/rust/src/server/src/routes/openai/utils/usage.rs b/rust/src/server/src/routes/openai/utils/usage.rs new file mode 100644 index 0000000000..c8c9d1e726 --- /dev/null +++ b/rust/src/server/src/routes/openai/utils/usage.rs @@ -0,0 +1,35 @@ +use super::types::Usage; + +/// Tracks cumulative token counts for OpenAI streaming chunks. +/// +/// This helper is intentionally only a counter. Callers decide whether to +/// attach `counts()` to each streamed data chunk, while final usage-only chunks +/// should still be built from the authoritative terminal `TokenUsage`. +#[derive(Debug, Clone, Default)] +pub(crate) struct ContinuousUsage { + prompt_tokens: usize, + output_tokens: usize, +} + +impl ContinuousUsage { + /// Record the prompt-token count reported when a stream starts. + pub(crate) fn set_prompt_tokens(&mut self, prompt_tokens: usize) { + self.prompt_tokens = prompt_tokens; + } + + /// Add newly decoded output tokens to the running completion count. + pub(crate) fn add_output_tokens(&mut self, output_tokens: usize) { + self.output_tokens = self.output_tokens.saturating_add(output_tokens); + } + + /// Replace the running counts with the final counts reported by generation. + pub(crate) fn set_final_counts(&mut self, prompt_tokens: usize, output_tokens: usize) { + self.prompt_tokens = prompt_tokens; + self.output_tokens = output_tokens; + } + + /// Build a streaming usage snapshot without prompt cache details. + pub(crate) fn to_usage(&self) -> Usage { + Usage::from_counts(self.prompt_tokens, self.output_tokens, None) + } +} diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 68ffe04a3b..c6de403402 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -151,6 +151,14 @@ fn sse_data_payloads(text: &str) -> Vec<&str> { text.lines().filter_map(|line| line.strip_prefix("data: ")).collect() } +fn sse_json_payloads(text: &str) -> Vec { + sse_data_payloads(text) + .into_iter() + .filter(|payload| *payload != "[DONE]") + .map(|payload| serde_json::from_str(payload).expect("sse json payload")) + .collect() +} + type TestFuture<'a> = Pin + Send + 'a>>; fn boxed_test_future<'a>(future: impl Future + Send + 'a) -> TestFuture<'a> { @@ -2341,6 +2349,60 @@ async fn include_usage_adds_final_usage_chunk_before_done() { assert_eq!(usage_chunk["usage"]["total_tokens"], 25); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn stream_continuous_usage_stats_adds_usage_to_chat_chunks() { + let (app, engine_task) = test_app_with_stream_output_specs(default_stream_output_specs()).await; + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "stream": true, + "stream_options": { + "include_usage": true, + "continuous_usage_stats": true + }, + "messages": [{"role": "user", "content": "hello"}] + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + engine_task.await.expect("mock engine task"); + let text = String::from_utf8(body.to_vec()).expect("utf8 body"); + let payloads = sse_json_payloads(&text); + + assert!( + payloads.iter().all(|payload| payload.get("usage").is_some()), + "{text}" + ); + assert!( + payloads.iter().any(|payload| { + payload["choices"].as_array().is_some_and(|choices| !choices.is_empty()) + && payload["usage"]["completion_tokens"] == json!(1) + }), + "{text}" + ); + let usage_chunk = payloads + .iter() + .find(|payload| payload["choices"] == json!([])) + .expect("final usage chunk"); + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 22); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 3); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn stream_without_include_usage_keeps_existing_shape() { @@ -3434,6 +3496,59 @@ async fn completions_happy_path_returns_sse_stream() { assert_eq!(usage_chunk["usage"]["completion_tokens"], 3); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn completions_stream_continuous_usage_stats_adds_usage_to_chunks() { + let (app, engine_task) = test_app_with_stream_output_specs(default_stream_output_specs()).await; + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/v1/completions") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "prompt": "hello", + "stream": true, + "stream_options": { + "include_usage": true, + "continuous_usage_stats": true + } + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + engine_task.await.expect("mock engine task"); + let text = String::from_utf8(body.to_vec()).expect("utf8 body"); + let payloads = sse_json_payloads(&text); + + assert!( + payloads.iter().all(|payload| payload.get("usage").is_some()), + "{text}" + ); + assert!( + payloads.iter().any(|payload| { + payload["choices"].as_array().is_some_and(|choices| !choices.is_empty()) + && payload["usage"]["completion_tokens"] == json!(1) + }), + "{text}" + ); + let usage_chunk = payloads + .iter() + .find(|payload| payload["choices"] == json!([])) + .expect("final usage chunk"); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 3); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn completions_echo_stream_emits_separate_prompt_chunk() {