[Rust Frontend] Extract shared options in route helper params (#44884)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -14,7 +14,8 @@ mod utils;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use axum::{Router, serve::ListenerExt as _};
|
||||
use axum::Router;
|
||||
use axum::serve::ListenerExt as _;
|
||||
pub use config::{Config, CoordinatorMode, HttpListenerMode};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::time::{Instant, sleep_until};
|
||||
|
||||
@@ -22,7 +22,7 @@ use vllm_llm::{
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _,
|
||||
};
|
||||
|
||||
use self::convert::prepare_generate_request;
|
||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||
use self::types::{
|
||||
GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice, GenerateStreamResponse,
|
||||
@@ -54,10 +54,7 @@ pub async fn generate(
|
||||
);
|
||||
|
||||
let log_request = state.enable_log_requests;
|
||||
let include_logprobs = prepared.include_logprobs;
|
||||
let include_prompt_logprobs = prepared.include_prompt_logprobs;
|
||||
let stream = prepared.stream;
|
||||
|
||||
let raw_stream = match state
|
||||
.chat
|
||||
.text()
|
||||
@@ -80,9 +77,7 @@ pub async fn generate(
|
||||
raw_stream,
|
||||
prepared.request_id,
|
||||
log_request,
|
||||
prepared.include_usage,
|
||||
prepared.include_continuous_usage,
|
||||
include_logprobs,
|
||||
prepared.options,
|
||||
);
|
||||
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
@@ -100,21 +95,11 @@ pub async fn generate(
|
||||
}
|
||||
};
|
||||
|
||||
if log_request {
|
||||
info!(
|
||||
parent: &request_span,
|
||||
prompt_tokens = collected.prompt_token_ids.len(),
|
||||
output_tokens = collected.token_ids.len(),
|
||||
finish_reason = collected.finish_reason.as_str(),
|
||||
"generate finished"
|
||||
);
|
||||
}
|
||||
|
||||
let response = match collect_generate(
|
||||
collected,
|
||||
prepared.request_id,
|
||||
include_logprobs,
|
||||
include_prompt_logprobs,
|
||||
log_request,
|
||||
prepared.options,
|
||||
) {
|
||||
Ok(response) => response,
|
||||
Err(error) => return error.into_response(),
|
||||
@@ -128,9 +113,13 @@ async fn generate_chunk_stream(
|
||||
stream: impl Stream<Item = vllm_llm::Result<GenerateOutput>>,
|
||||
request_id: String,
|
||||
log_request: bool,
|
||||
include_usage: bool,
|
||||
include_continuous_usage: bool,
|
||||
include_logprobs: bool,
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
include_logprobs,
|
||||
// Ignored: raw generate streaming has no prompt-logprobs wire shape.
|
||||
include_prompt_logprobs: _,
|
||||
}: ResponseOptions,
|
||||
mut y: TryYielder<GenerateStreamResponse, ApiError>,
|
||||
) -> Result<(), ApiError> {
|
||||
pin_mut!(stream);
|
||||
@@ -222,8 +211,15 @@ async fn generate_chunk_stream(
|
||||
fn collect_generate(
|
||||
collected: CollectedGenerateOutput,
|
||||
request_id: String,
|
||||
include_logprobs: bool,
|
||||
include_prompt_logprobs: bool,
|
||||
log_request: bool,
|
||||
ResponseOptions {
|
||||
// Ignored: non-streaming raw generate responses do not include usage.
|
||||
include_usage: _,
|
||||
// Ignored: continuous usage is a streaming-only option.
|
||||
include_continuous_usage: _,
|
||||
include_logprobs,
|
||||
include_prompt_logprobs,
|
||||
}: ResponseOptions,
|
||||
) -> Result<GenerateResponse, ApiError> {
|
||||
let logprobs = if include_logprobs {
|
||||
let logprobs = collected.logprobs.as_ref().ok_or_else(|| {
|
||||
@@ -246,13 +242,23 @@ fn collect_generate(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let finish_reason = collected.finish_reason.as_str().to_string();
|
||||
|
||||
if log_request {
|
||||
info!(
|
||||
prompt_tokens = collected.prompt_token_ids.len(),
|
||||
output_tokens = collected.token_ids.len(),
|
||||
%finish_reason,
|
||||
"generate finished"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(GenerateResponse {
|
||||
request_id,
|
||||
choices: vec![GenerateResponseChoice {
|
||||
index: 0,
|
||||
logprobs,
|
||||
finish_reason: Some(collected.finish_reason.as_str().to_string()),
|
||||
finish_reason: Some(finish_reason),
|
||||
token_ids: collected.token_ids,
|
||||
}],
|
||||
prompt_logprobs,
|
||||
@@ -408,11 +414,19 @@ mod tests {
|
||||
}),
|
||||
]);
|
||||
|
||||
let chunks: Vec<_> =
|
||||
generate_chunk_stream(stream, "raw-stream".to_string(), false, true, true, false)
|
||||
.try_collect()
|
||||
.await
|
||||
.expect("collect chunks");
|
||||
let chunks: Vec<_> = generate_chunk_stream(
|
||||
stream,
|
||||
"raw-stream".to_string(),
|
||||
false,
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.try_collect()
|
||||
.await
|
||||
.expect("collect chunks");
|
||||
|
||||
assert_eq!(chunks.len(), 2);
|
||||
assert_eq!(
|
||||
|
||||
@@ -8,19 +8,29 @@ use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};
|
||||
|
||||
/// Lowered generate request plus the response request ID.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PreparedRequest {
|
||||
pub(super) struct PreparedRequest {
|
||||
pub request_id: String,
|
||||
pub text_request: TextRequest,
|
||||
pub stream: bool,
|
||||
/// Public response rendering options for route-layer helpers.
|
||||
pub options: ResponseOptions,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq)]
|
||||
pub(super) struct ResponseOptions {
|
||||
/// Whether the caller asked for the final streamed usage chunk.
|
||||
pub include_usage: bool,
|
||||
/// Whether the caller asked for usage on every streamed chunk.
|
||||
pub include_continuous_usage: bool,
|
||||
/// Whether the caller requested output logprobs on generate choices.
|
||||
pub include_logprobs: bool,
|
||||
/// Whether the caller requested top-level prompt logprobs.
|
||||
pub include_prompt_logprobs: bool,
|
||||
}
|
||||
|
||||
/// Validate and lower one raw generate request into the internal
|
||||
/// text-generation format.
|
||||
pub fn prepare_generate_request(
|
||||
pub(super) fn prepare_generate_request(
|
||||
request: GenerateRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
@@ -65,10 +75,12 @@ pub fn prepare_generate_request(
|
||||
request_id: ctx.request_id,
|
||||
text_request,
|
||||
stream,
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
include_logprobs,
|
||||
include_prompt_logprobs,
|
||||
options: ResponseOptions {
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
include_logprobs,
|
||||
include_prompt_logprobs,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -158,7 +170,7 @@ mod tests {
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert!(!prepared.include_usage);
|
||||
assert!(!prepared.include_continuous_usage);
|
||||
assert!(!prepared.options.include_usage);
|
||||
assert!(!prepared.options.include_continuous_usage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
pub mod convert;
|
||||
mod convert;
|
||||
mod types;
|
||||
mod validate;
|
||||
|
||||
@@ -23,8 +23,8 @@ use vllm_chat::{
|
||||
};
|
||||
use vllm_engine_core_client::protocol::StopReason;
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_chat_request};
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::routes::openai::chat_completions::convert::prepare_chat_request;
|
||||
use crate::routes::openai::chat_completions::types::{
|
||||
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
|
||||
ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
|
||||
@@ -83,12 +83,7 @@ pub async fn chat_completions(
|
||||
prepared.response_model,
|
||||
created,
|
||||
log_request,
|
||||
prepared.include_usage,
|
||||
prepared.requested_logprobs,
|
||||
prepared.include_reasoning,
|
||||
prepared.echo,
|
||||
prepared.return_token_ids,
|
||||
prepared.return_tokens_as_token_ids,
|
||||
prepared.options,
|
||||
);
|
||||
let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
@@ -99,12 +94,8 @@ pub async fn chat_completions(
|
||||
prepared.request_id,
|
||||
prepared.response_model,
|
||||
created,
|
||||
prepared.requested_logprobs,
|
||||
prepared.include_prompt_logprobs,
|
||||
prepared.include_reasoning,
|
||||
prepared.echo,
|
||||
prepared.return_token_ids,
|
||||
prepared.return_tokens_as_token_ids,
|
||||
log_request,
|
||||
prepared.options,
|
||||
)
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
@@ -113,18 +104,6 @@ pub async fn chat_completions(
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
|
||||
if log_request {
|
||||
let usage = response.usage.as_ref();
|
||||
info!(
|
||||
parent: &request_span,
|
||||
model = %response.model,
|
||||
prompt_tokens = usage.map_or(0, |u| u.prompt_tokens),
|
||||
output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0),
|
||||
finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"),
|
||||
"chat completion finished"
|
||||
);
|
||||
}
|
||||
|
||||
Json(response).into_response()
|
||||
}
|
||||
}
|
||||
@@ -134,12 +113,17 @@ async fn collect_chat_completion(
|
||||
request_id: String,
|
||||
response_model: String,
|
||||
created: u64,
|
||||
requested_logprobs: bool,
|
||||
include_prompt_logprobs: bool,
|
||||
include_reasoning: bool,
|
||||
echo: Option<String>,
|
||||
return_token_ids: bool,
|
||||
return_tokens_as_token_ids: bool,
|
||||
log_request: bool,
|
||||
ResponseOptions {
|
||||
// Ignored: non-streaming responses always include usage.
|
||||
include_usage: _,
|
||||
requested_logprobs,
|
||||
include_prompt_logprobs,
|
||||
include_reasoning,
|
||||
echo,
|
||||
return_token_ids,
|
||||
return_tokens_as_token_ids,
|
||||
}: ResponseOptions,
|
||||
) -> Result<ChatCompletionResponse, ApiError> {
|
||||
let collected = stream.collect_message().await.map_err(|error| {
|
||||
server_error!(
|
||||
@@ -201,6 +185,16 @@ async fn collect_chat_completion(
|
||||
};
|
||||
let usage = Usage::from_counts(prompt_token_count as u32, output_token_count as u32);
|
||||
|
||||
if log_request {
|
||||
info!(
|
||||
model = %response_model,
|
||||
prompt_tokens = usage.prompt_tokens,
|
||||
output_tokens = usage.completion_tokens.unwrap_or(0),
|
||||
finish_reason = %finish_reason,
|
||||
"chat completion finished"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(ChatCompletionResponse {
|
||||
id: request_id,
|
||||
object: "chat.completion".to_string(),
|
||||
@@ -238,12 +232,16 @@ async fn chat_completion_chunk_stream(
|
||||
response_model: String,
|
||||
created: u64,
|
||||
log_request: bool,
|
||||
include_usage: bool,
|
||||
requested_logprobs: bool,
|
||||
include_reasoning: bool,
|
||||
echo: Option<String>,
|
||||
return_token_ids: bool,
|
||||
return_tokens_as_token_ids: bool,
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
requested_logprobs,
|
||||
// Ignored: chat streaming prompt logprobs are rejected for Python parity.
|
||||
include_prompt_logprobs: _,
|
||||
include_reasoning,
|
||||
echo,
|
||||
return_token_ids,
|
||||
return_tokens_as_token_ids,
|
||||
}: ResponseOptions,
|
||||
mut y: TryYielder<ChatCompletionStreamResponse, ApiError>,
|
||||
) -> Result<(), ApiError> {
|
||||
let mut saw_tool_calls = false;
|
||||
@@ -806,7 +804,7 @@ mod tests {
|
||||
use vllm_engine_core_client::protocol::StopReason;
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
||||
|
||||
use super::{block_delta_chunk, chat_completion_chunk_stream, final_chunk};
|
||||
use super::{ResponseOptions, block_delta_chunk, chat_completion_chunk_stream, final_chunk};
|
||||
|
||||
#[test]
|
||||
fn text_chunk_uses_content_only_delta() {
|
||||
@@ -932,12 +930,11 @@ mod tests {
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
ResponseOptions {
|
||||
requested_logprobs: true,
|
||||
include_reasoning: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
@@ -996,12 +993,11 @@ mod tests {
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
ResponseOptions {
|
||||
requested_logprobs: true,
|
||||
include_reasoning: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
@@ -1049,12 +1045,7 @@ mod tests {
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
ResponseOptions::default(),
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
@@ -1132,12 +1123,11 @@ mod tests {
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
ResponseOptions {
|
||||
requested_logprobs: true,
|
||||
return_token_ids: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
@@ -1263,12 +1253,11 @@ mod tests {
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
false,
|
||||
ResponseOptions {
|
||||
requested_logprobs: true,
|
||||
return_token_ids: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
@@ -1342,12 +1331,10 @@ mod tests {
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
ResponseOptions {
|
||||
include_reasoning: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
|
||||
@@ -18,11 +18,19 @@ use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer
|
||||
/// Lowered chat request plus the public response metadata carried by every SSE
|
||||
/// chunk.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PreparedRequest {
|
||||
pub(super) struct PreparedRequest {
|
||||
/// Stable OpenAI-style request ID, reused as the external chat request ID.
|
||||
pub request_id: String,
|
||||
/// Public model ID echoed back to the client.
|
||||
pub response_model: String,
|
||||
/// Public response rendering options for route-layer helpers.
|
||||
pub options: ResponseOptions,
|
||||
/// Lowered chat request for `vllm-chat`.
|
||||
pub chat_request: ChatRequest,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub(super) struct ResponseOptions {
|
||||
/// Whether the caller asked for the final streamed usage chunk.
|
||||
pub include_usage: bool,
|
||||
/// Whether the caller requested output logprobs on chat choices.
|
||||
@@ -31,8 +39,6 @@ pub struct PreparedRequest {
|
||||
pub include_prompt_logprobs: bool,
|
||||
/// Whether to include reasoning content in OpenAI responses.
|
||||
pub include_reasoning: bool,
|
||||
/// Lowered chat request for `vllm-chat`.
|
||||
pub chat_request: ChatRequest,
|
||||
/// Last assistant-role message content to echo back when `echo=true`.
|
||||
pub echo: Option<String>,
|
||||
/// Whether to include token IDs alongside generated text.
|
||||
@@ -46,7 +52,7 @@ pub struct PreparedRequest {
|
||||
///
|
||||
/// `lora_resolution.model_names` must be non-empty; the first entry is used as
|
||||
/// the base `model` field in responses when no LoRA adapter is selected.
|
||||
pub(crate) fn prepare_chat_request(
|
||||
pub(super) fn prepare_chat_request(
|
||||
request: ChatCompletionRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
@@ -146,14 +152,16 @@ pub(crate) fn prepare_chat_request(
|
||||
Ok(PreparedRequest {
|
||||
request_id,
|
||||
response_model,
|
||||
include_usage,
|
||||
requested_logprobs,
|
||||
include_prompt_logprobs,
|
||||
include_reasoning,
|
||||
options: ResponseOptions {
|
||||
include_usage,
|
||||
requested_logprobs,
|
||||
include_prompt_logprobs,
|
||||
include_reasoning,
|
||||
echo,
|
||||
return_token_ids: request.return_token_ids.unwrap_or(false),
|
||||
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
|
||||
},
|
||||
chat_request,
|
||||
echo,
|
||||
return_token_ids: request.return_token_ids.unwrap_or(false),
|
||||
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
pub(crate) fn normalize_generation_prompt_mode(
|
||||
@@ -497,7 +505,7 @@ mod tests {
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(!prepared.include_reasoning);
|
||||
assert!(!prepared.options.include_reasoning);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -867,8 +875,8 @@ mod tests {
|
||||
)
|
||||
.expect("request is valid");
|
||||
|
||||
assert!(prepared.requested_logprobs);
|
||||
assert!(prepared.include_prompt_logprobs);
|
||||
assert!(prepared.options.requested_logprobs);
|
||||
assert!(prepared.options.include_prompt_logprobs);
|
||||
assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(0));
|
||||
assert_eq!(
|
||||
prepared.chat_request.sampling_params.prompt_logprobs,
|
||||
@@ -894,7 +902,7 @@ mod tests {
|
||||
|
||||
assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(3));
|
||||
assert_eq!(prepared.chat_request.sampling_params.prompt_logprobs, None);
|
||||
assert!(!prepared.include_prompt_logprobs);
|
||||
assert!(!prepared.options.include_prompt_logprobs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -18,13 +18,13 @@ use tracing::{debug, error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _};
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_completion_request};
|
||||
use super::utils::logprobs::{
|
||||
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
|
||||
text_len,
|
||||
};
|
||||
use super::utils::types::Usage;
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::routes::openai::completions::convert::prepare_completion_request;
|
||||
use crate::routes::openai::completions::types::{
|
||||
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
|
||||
CompletionStreamChoice, CompletionStreamResponse,
|
||||
@@ -42,7 +42,6 @@ pub async fn completions(
|
||||
ValidatedJson(body): ValidatedJson<CompletionRequest>,
|
||||
) -> Response {
|
||||
let stream = body.stream;
|
||||
let logprobs = body.logprobs;
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
|
||||
|
||||
@@ -57,9 +56,7 @@ pub async fn completions(
|
||||
);
|
||||
|
||||
let created = unix_timestamp();
|
||||
let include_prompt_logprobs = prepared.text_request.sampling_params.prompt_logprobs.is_some();
|
||||
let log_request = state.enable_log_requests;
|
||||
|
||||
let text_stream = match state
|
||||
.chat
|
||||
.text()
|
||||
@@ -84,11 +81,7 @@ pub async fn completions(
|
||||
prepared.response_model,
|
||||
created,
|
||||
log_request,
|
||||
prepared.include_usage,
|
||||
prepared.echo,
|
||||
logprobs,
|
||||
prepared.return_token_ids,
|
||||
prepared.return_tokens_as_token_ids,
|
||||
prepared.options,
|
||||
);
|
||||
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
@@ -99,11 +92,8 @@ pub async fn completions(
|
||||
prepared.request_id,
|
||||
prepared.response_model,
|
||||
created,
|
||||
prepared.echo,
|
||||
logprobs,
|
||||
include_prompt_logprobs,
|
||||
prepared.return_token_ids,
|
||||
prepared.return_tokens_as_token_ids,
|
||||
log_request,
|
||||
prepared.options,
|
||||
)
|
||||
.instrument(request_span.clone())
|
||||
.await
|
||||
@@ -112,18 +102,6 @@ pub async fn completions(
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
|
||||
if log_request {
|
||||
let usage = response.usage.as_ref();
|
||||
info!(
|
||||
parent: &request_span,
|
||||
model = %response.model,
|
||||
prompt_tokens = usage.map_or(0, |u| u.prompt_tokens),
|
||||
output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0),
|
||||
finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"),
|
||||
"completion finished"
|
||||
);
|
||||
}
|
||||
|
||||
Json(response).into_response()
|
||||
}
|
||||
}
|
||||
@@ -133,11 +111,16 @@ async fn collect_completion(
|
||||
request_id: String,
|
||||
response_model: String,
|
||||
created: u64,
|
||||
echo: Option<String>,
|
||||
requested_logprobs: Option<u32>,
|
||||
include_prompt_logprobs: bool,
|
||||
return_token_ids: bool,
|
||||
return_tokens_as_token_ids: bool,
|
||||
log_request: bool,
|
||||
ResponseOptions {
|
||||
// Ignored: non-streaming responses always include usage.
|
||||
include_usage: _,
|
||||
echo,
|
||||
requested_logprobs,
|
||||
include_prompt_logprobs,
|
||||
return_token_ids,
|
||||
return_tokens_as_token_ids,
|
||||
}: ResponseOptions,
|
||||
) -> Result<CompletionResponse, ApiError> {
|
||||
let collected = stream
|
||||
.collect_output()
|
||||
@@ -175,6 +158,21 @@ async fn collect_completion(
|
||||
None => collected.text,
|
||||
Some(prompt) => format!("{prompt}{}", collected.text),
|
||||
};
|
||||
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
|
||||
let usage = Usage::from_counts(
|
||||
collected.prompt_token_ids.len() as u32,
|
||||
collected.token_ids.len() as u32,
|
||||
);
|
||||
|
||||
if log_request {
|
||||
info!(
|
||||
model = %response_model,
|
||||
prompt_tokens = usage.prompt_tokens,
|
||||
output_tokens = usage.completion_tokens.unwrap_or(0),
|
||||
%finish_reason,
|
||||
"completion finished"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(CompletionResponse {
|
||||
id: request_id,
|
||||
@@ -185,16 +183,13 @@ async fn collect_completion(
|
||||
index: 0,
|
||||
text,
|
||||
logprobs,
|
||||
finish_reason: Some(completion_finish_reason_to_openai(finish_reason)?.into()),
|
||||
finish_reason: Some(finish_reason),
|
||||
stop_reason,
|
||||
prompt_logprobs,
|
||||
token_ids: return_token_ids.then(|| collected.token_ids.clone()),
|
||||
prompt_token_ids: return_token_ids.then(|| collected.prompt_token_ids.to_vec()),
|
||||
}],
|
||||
usage: Some(Usage::from_counts(
|
||||
collected.prompt_token_ids.len() as u32,
|
||||
collected.token_ids.len() as u32,
|
||||
)),
|
||||
usage: Some(usage),
|
||||
system_fingerprint: None,
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
})
|
||||
@@ -208,11 +203,15 @@ async fn completion_chunk_stream(
|
||||
response_model: String,
|
||||
created: u64,
|
||||
log_request: bool,
|
||||
include_usage: bool,
|
||||
echo: Option<String>,
|
||||
requested_logprobs: Option<u32>,
|
||||
return_token_ids: bool,
|
||||
return_tokens_as_token_ids: bool,
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
echo,
|
||||
requested_logprobs,
|
||||
// Ignored: streaming prompt logprobs are rejected for Python parity.
|
||||
include_prompt_logprobs: _,
|
||||
return_token_ids,
|
||||
return_tokens_as_token_ids,
|
||||
}: ResponseOptions,
|
||||
mut y: TryYielder<CompletionSseChunk, ApiError>,
|
||||
) -> Result<(), ApiError> {
|
||||
pin_mut!(stream);
|
||||
@@ -432,7 +431,7 @@ mod tests {
|
||||
FinishReason, Finished,
|
||||
};
|
||||
|
||||
use super::{CompletionSseChunk, completion_chunk_stream, final_chunk};
|
||||
use super::{CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk};
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_stop_finish_reason() {
|
||||
@@ -527,11 +526,10 @@ mod tests {
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
Some(1),
|
||||
false,
|
||||
false,
|
||||
ResponseOptions {
|
||||
requested_logprobs: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
@@ -10,18 +10,28 @@ use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer
|
||||
/// Lowered completion request plus the public response metadata carried by
|
||||
/// every SSE chunk.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct PreparedRequest {
|
||||
pub(super) struct PreparedRequest {
|
||||
/// Stable OpenAI-style request ID, reused as the external text request ID.
|
||||
pub request_id: String,
|
||||
/// Public model ID echoed back to the client.
|
||||
pub response_model: String,
|
||||
/// Whether the caller asked for the final streamed usage chunk.
|
||||
pub include_usage: bool,
|
||||
/// Public response rendering options for route-layer helpers.
|
||||
pub options: ResponseOptions,
|
||||
/// Lowered text request for the shared `vllm-text` facade.
|
||||
pub text_request: TextRequest,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub(super) struct ResponseOptions {
|
||||
/// Whether the caller asked for the final streamed usage chunk.
|
||||
pub include_usage: bool,
|
||||
/// Original text prompt that should be echoed back northbound when
|
||||
/// `echo=true`.
|
||||
pub echo: Option<String>,
|
||||
/// Whether the caller requested output logprobs on completion choices.
|
||||
pub requested_logprobs: Option<u32>,
|
||||
/// Whether the caller requested choice-level prompt logprobs.
|
||||
pub include_prompt_logprobs: bool,
|
||||
/// Whether to include token IDs alongside generated text.
|
||||
pub return_token_ids: bool,
|
||||
/// Whether to format logprob tokens as `token_id:{id}`.
|
||||
@@ -33,7 +43,7 @@ pub struct PreparedRequest {
|
||||
///
|
||||
/// `lora_resolution.model_names` must be non-empty; the first entry is used as
|
||||
/// the base `model` field in responses when no LoRA adapter is selected.
|
||||
pub(crate) fn prepare_completion_request(
|
||||
pub(super) fn prepare_completion_request(
|
||||
request: CompletionRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
@@ -64,6 +74,7 @@ pub(crate) fn prepare_completion_request(
|
||||
let include_usage = (request.stream_options.as_ref())
|
||||
.and_then(|options| options.include_usage)
|
||||
.unwrap_or(false);
|
||||
let include_prompt_logprobs = prompt_logprobs.is_some();
|
||||
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
|
||||
|
||||
let structured_outputs =
|
||||
@@ -116,11 +127,15 @@ pub(crate) fn prepare_completion_request(
|
||||
Ok(PreparedRequest {
|
||||
request_id,
|
||||
response_model,
|
||||
include_usage,
|
||||
options: ResponseOptions {
|
||||
include_usage,
|
||||
echo,
|
||||
requested_logprobs: request.logprobs,
|
||||
include_prompt_logprobs,
|
||||
return_token_ids: request.return_token_ids.unwrap_or(false),
|
||||
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
|
||||
},
|
||||
text_request,
|
||||
echo,
|
||||
return_token_ids: request.return_token_ids.unwrap_or(false),
|
||||
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -206,7 +221,7 @@ mod tests {
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert!(prepared.include_usage);
|
||||
assert!(prepared.options.include_usage);
|
||||
assert_eq!(
|
||||
prepared.text_request.prompt,
|
||||
Prompt::TokenIds(vec![11, 22, 33])
|
||||
@@ -250,7 +265,7 @@ mod tests {
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
assert_eq!(prepared.echo, Some("hello".to_string()));
|
||||
assert_eq!(prepared.options.echo, Some("hello".to_string()));
|
||||
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
|
||||
}
|
||||
|
||||
|
||||
@@ -14,15 +14,14 @@ use std::{fmt, fs};
|
||||
use axum::body::{Body, to_bytes};
|
||||
use axum::http::{Request, StatusCode};
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt as _;
|
||||
use rmpv::Value;
|
||||
use serde_json::json;
|
||||
use serial_test::serial;
|
||||
use tower::{Service as _, ServiceExt as _};
|
||||
use vllm_chat::{
|
||||
ChatBackend, ChatContent, ChatContentPart, ChatEvent, ChatLlm, ChatMessage, ChatRenderer,
|
||||
ChatRequest, ChatRole, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor,
|
||||
DynChatRenderer, NewChatOutputProcessorOptions, SamplingParams,
|
||||
ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest,
|
||||
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
|
||||
NewChatOutputProcessorOptions,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::logprobs::{
|
||||
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
|
||||
@@ -44,8 +43,6 @@ use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::routes::openai::chat_completions::convert::prepare_chat_request;
|
||||
use crate::state::AppState;
|
||||
|
||||
fn request_output(
|
||||
@@ -3478,92 +3475,6 @@ async fn completions_echo_stream_emits_separate_prompt_chunk() {
|
||||
assert_eq!(usage_chunk["usage"]["completion_tokens"], 3);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn chat_harness_streams_text_events() {
|
||||
let (chat, engine_task) = test_chat_with_engine_handle().await;
|
||||
let mut stream = chat
|
||||
.chat(ChatRequest {
|
||||
messages: vec![ChatMessage::text(ChatRole::User, "hello")],
|
||||
sampling_params: SamplingParams {
|
||||
max_tokens: Some(8),
|
||||
..Default::default()
|
||||
},
|
||||
request_id: "chat-harness".to_string(),
|
||||
..ChatRequest::for_test()
|
||||
})
|
||||
.await
|
||||
.expect("submit chat request");
|
||||
|
||||
let mut saw_text = false;
|
||||
let mut saw_done = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
match event.expect("chat event") {
|
||||
ChatEvent::BlockDelta { .. } => saw_text = true,
|
||||
ChatEvent::Done { .. } => {
|
||||
saw_done = true;
|
||||
break;
|
||||
}
|
||||
ChatEvent::Start { .. }
|
||||
| ChatEvent::LogprobsDelta { .. }
|
||||
| ChatEvent::BlockStart { .. }
|
||||
| ChatEvent::BlockEnd { .. }
|
||||
| ChatEvent::ToolCallStart { .. }
|
||||
| ChatEvent::ToolCallArgumentsDelta { .. }
|
||||
| ChatEvent::ToolCallEnd { .. } => {}
|
||||
}
|
||||
}
|
||||
engine_task.await.expect("mock engine task");
|
||||
|
||||
assert!(saw_text);
|
||||
assert!(saw_done);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn prepared_openai_request_streams_text_events() {
|
||||
let (chat, engine_task) = test_chat_with_engine_handle().await;
|
||||
let prepared = prepare_chat_request(
|
||||
serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"stream": true,
|
||||
"messages": [{"role": "user", "content": "hello"}]
|
||||
}))
|
||||
.expect("decode request"),
|
||||
&LoraModelResolution {
|
||||
model_names: vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
lora_request: None,
|
||||
},
|
||||
crate::utils::ResolvedRequestContext::default(),
|
||||
)
|
||||
.expect("prepare request");
|
||||
|
||||
let mut stream = chat.chat(prepared.chat_request).await.expect("submit chat request");
|
||||
|
||||
let mut saw_text = false;
|
||||
let mut saw_done = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
match event.expect("chat event") {
|
||||
ChatEvent::BlockDelta { .. } => saw_text = true,
|
||||
ChatEvent::Done { .. } => {
|
||||
saw_done = true;
|
||||
break;
|
||||
}
|
||||
ChatEvent::Start { .. }
|
||||
| ChatEvent::LogprobsDelta { .. }
|
||||
| ChatEvent::BlockStart { .. }
|
||||
| ChatEvent::BlockEnd { .. }
|
||||
| ChatEvent::ToolCallStart { .. }
|
||||
| ChatEvent::ToolCallArgumentsDelta { .. }
|
||||
| ChatEvent::ToolCallEnd { .. } => {}
|
||||
}
|
||||
}
|
||||
engine_task.await.expect("mock engine task");
|
||||
|
||||
assert!(saw_text);
|
||||
assert!(saw_done);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn reasoning_blocks_are_mapped_to_reasoning_sse_chunks() {
|
||||
|
||||
@@ -10,7 +10,6 @@ use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
|
||||
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
|
||||
|
||||
use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
|
||||
|
||||
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
Reference in New Issue
Block a user