[Rust Frontend] Populate cached_token_count in responses (#44887)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -14,8 +14,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{
|
||||
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
|
||||
RendererSelection, serve,
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode,
|
||||
ParserSelection, RendererSelection, serve,
|
||||
};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
@@ -68,8 +68,7 @@ async fn main() -> Result<()> {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
enable_log_requests: false,
|
||||
enable_request_id_headers: false,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
api_keys: Vec::new(),
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
|
||||
@@ -34,6 +34,17 @@ pub enum CoordinatorMode {
|
||||
External { address: String },
|
||||
}
|
||||
|
||||
/// HTTP/API-server behavior switches that affect route-layer responses.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Default)]
|
||||
pub struct ApiServerOptions {
|
||||
/// Log a summary line for each completed request.
|
||||
pub enable_log_requests: bool,
|
||||
/// When `true`, include prompt token cache details in response usage.
|
||||
pub enable_prompt_tokens_details: bool,
|
||||
/// When `true`, set `X-Request-Id` on every HTTP response.
|
||||
pub enable_request_id_headers: bool,
|
||||
}
|
||||
|
||||
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
|
||||
#[derive(Educe, Clone, PartialEq, Eq, Serialize)]
|
||||
#[educe(Debug)]
|
||||
@@ -66,10 +77,8 @@ pub struct Config {
|
||||
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
/// How to serialize `message.content` for chat-template rendering.
|
||||
pub chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
/// Log a summary line for each completed request.
|
||||
pub enable_log_requests: bool,
|
||||
/// When `true`, set `X-Request-Id` on every HTTP response.
|
||||
pub enable_request_id_headers: bool,
|
||||
/// HTTP/API-server behavior switches.
|
||||
pub api_server_options: ApiServerOptions,
|
||||
/// API keys accepted as bearer tokens for guarded routes.
|
||||
#[serde(skip_serializing)]
|
||||
#[educe(Debug(method(fmt_redacted_api_keys)))]
|
||||
|
||||
@@ -350,7 +350,7 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
|
||||
};
|
||||
|
||||
pb::FinishInfo {
|
||||
num_output_tokens: finished.output_token_count as u32,
|
||||
num_output_tokens: finished.usage.output_token_count as u32,
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),
|
||||
@@ -590,8 +590,11 @@ mod tests {
|
||||
|
||||
fn finished(reason: FinishReason) -> Finished {
|
||||
Finished {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 0,
|
||||
output_token_count: 0,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: reason,
|
||||
kv_transfer_params: None,
|
||||
}
|
||||
|
||||
@@ -71,8 +71,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
|
||||
);
|
||||
|
||||
let finish_info = vllm_text::Finished {
|
||||
prompt_token_count: collected.prompt_token_ids.len(),
|
||||
output_token_count: collected.token_ids.len(),
|
||||
usage: collected.usage,
|
||||
finish_reason: collected.finish_reason,
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
};
|
||||
|
||||
@@ -16,7 +16,7 @@ use std::sync::{Arc, OnceLock};
|
||||
use anyhow::{Context as _, Result};
|
||||
use axum::Router;
|
||||
use axum::serve::ListenerExt as _;
|
||||
pub use config::{Config, CoordinatorMode, HttpListenerMode};
|
||||
pub use config::{ApiServerOptions, Config, CoordinatorMode, HttpListenerMode};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::time::{Instant, sleep_until};
|
||||
use tokio_stream::wrappers::TcpListenerStream;
|
||||
@@ -91,8 +91,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
|
||||
Ok(Arc::new(
|
||||
AppState::new(served_model_names, chat)
|
||||
.with_log_requests(config.enable_log_requests)
|
||||
.with_request_id_headers(config.enable_request_id_headers)
|
||||
.with_api_server_options(config.api_server_options)
|
||||
.with_server_info(ServerInfoSnapshot::from_config(config))
|
||||
.with_api_keys(config.api_keys.clone()),
|
||||
))
|
||||
|
||||
@@ -100,7 +100,7 @@ fn build_router_with_options(
|
||||
.route("/server_info", get(server_info::server_info))
|
||||
}
|
||||
|
||||
let enable_request_id_headers = state.enable_request_id_headers;
|
||||
let enable_request_id_headers = state.api_server_options.enable_request_id_headers;
|
||||
let enable_api_key_auth = state.has_api_keys();
|
||||
let mut router = router
|
||||
.with_state(state.clone())
|
||||
|
||||
@@ -19,7 +19,7 @@ use tracing::{error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
|
||||
use vllm_llm::{
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _,
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
|
||||
};
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||
@@ -27,6 +27,7 @@ use self::types::{
|
||||
GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
|
||||
GenerateResponseStreamChoice, GenerateStreamResponse,
|
||||
};
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::routes::openai::utils::logprobs::clamp_logprob;
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage};
|
||||
@@ -53,7 +54,7 @@ pub async fn generate(
|
||||
engine_request_id = tracing::field::Empty,
|
||||
);
|
||||
|
||||
let log_request = state.enable_log_requests;
|
||||
let api_server_options = state.api_server_options;
|
||||
let stream = prepared.stream;
|
||||
let raw_stream = match state
|
||||
.chat
|
||||
@@ -76,7 +77,7 @@ pub async fn generate(
|
||||
let chunk_stream = generate_chunk_stream(
|
||||
raw_stream,
|
||||
prepared.request_id,
|
||||
log_request,
|
||||
api_server_options,
|
||||
prepared.options,
|
||||
);
|
||||
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
|
||||
@@ -98,7 +99,7 @@ pub async fn generate(
|
||||
let response = match collect_generate(
|
||||
collected,
|
||||
prepared.request_id,
|
||||
log_request,
|
||||
api_server_options,
|
||||
prepared.options,
|
||||
) {
|
||||
Ok(response) => response,
|
||||
@@ -112,7 +113,11 @@ pub async fn generate(
|
||||
async fn generate_chunk_stream(
|
||||
stream: impl Stream<Item = vllm_llm::Result<GenerateOutput>>,
|
||||
request_id: String,
|
||||
log_request: bool,
|
||||
ApiServerOptions {
|
||||
enable_log_requests,
|
||||
enable_prompt_tokens_details,
|
||||
..
|
||||
}: ApiServerOptions,
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
@@ -123,20 +128,21 @@ async fn generate_chunk_stream(
|
||||
mut y: TryYielder<GenerateStreamResponse, ApiError>,
|
||||
) -> Result<(), ApiError> {
|
||||
pin_mut!(stream);
|
||||
let mut prompt_tokens: Option<u32> = None;
|
||||
let mut output_tokens = 0_u32;
|
||||
let mut prompt_tokens = None;
|
||||
let mut usage = TokenUsage::default();
|
||||
|
||||
while let Some(next) = stream.next().await {
|
||||
match next {
|
||||
Ok(output) => {
|
||||
if prompt_tokens.is_none() {
|
||||
prompt_tokens =
|
||||
output.prompt_info.as_ref().map(|info| info.prompt_token_ids.len() as u32);
|
||||
output.prompt_info.as_ref().map(|info| info.prompt_token_ids.len());
|
||||
}
|
||||
let usage_prompt_tokens = prompt_tokens.unwrap_or_default();
|
||||
usage.prompt_token_count = prompt_tokens.unwrap_or_default();
|
||||
usage.cached_token_count = usage.cached_token_count.max(output.cached_token_count);
|
||||
|
||||
let token_ids = output.token_ids;
|
||||
output_tokens = output_tokens.saturating_add(token_ids.len() as u32);
|
||||
usage.output_token_count = usage.output_token_count.saturating_add(token_ids.len());
|
||||
let finish_reason = output.finish_reason;
|
||||
|
||||
if matches!(finish_reason.as_ref(), Some(FinishReason::Error)) {
|
||||
@@ -144,12 +150,12 @@ async fn generate_chunk_stream(
|
||||
}
|
||||
|
||||
if let Some(finish_reason) = finish_reason.as_ref()
|
||||
&& log_request
|
||||
&& enable_log_requests
|
||||
{
|
||||
info!(
|
||||
stream = true,
|
||||
prompt_tokens = usage_prompt_tokens,
|
||||
output_tokens,
|
||||
prompt_tokens = usage.prompt_token_count,
|
||||
output_tokens = usage.output_token_count,
|
||||
finish_reason = finish_reason.as_str(),
|
||||
"generate finished"
|
||||
);
|
||||
@@ -179,7 +185,7 @@ async fn generate_chunk_stream(
|
||||
token_ids,
|
||||
}],
|
||||
usage: include_continuous_usage
|
||||
.then(|| Usage::from_counts(usage_prompt_tokens, output_tokens)),
|
||||
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -197,10 +203,7 @@ async fn generate_chunk_stream(
|
||||
y.yield_ok(GenerateStreamResponse {
|
||||
request_id,
|
||||
choices: Vec::new(),
|
||||
usage: Some(Usage::from_counts(
|
||||
prompt_tokens.unwrap_or_default(),
|
||||
output_tokens,
|
||||
)),
|
||||
usage: Some(Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -211,7 +214,10 @@ async fn generate_chunk_stream(
|
||||
fn collect_generate(
|
||||
collected: CollectedGenerateOutput,
|
||||
request_id: String,
|
||||
log_request: bool,
|
||||
ApiServerOptions {
|
||||
enable_log_requests,
|
||||
..
|
||||
}: ApiServerOptions,
|
||||
ResponseOptions {
|
||||
// Ignored: non-streaming raw generate responses do not include usage.
|
||||
include_usage: _,
|
||||
@@ -244,7 +250,7 @@ fn collect_generate(
|
||||
};
|
||||
let finish_reason = collected.finish_reason.as_str().to_string();
|
||||
|
||||
if log_request {
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
prompt_tokens = collected.prompt_token_ids.len(),
|
||||
output_tokens = collected.token_ids.len(),
|
||||
@@ -399,6 +405,7 @@ mod tests {
|
||||
token_ids: Vec::new(),
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
cached_token_count: 0,
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
Ok(GenerateOutput {
|
||||
@@ -410,6 +417,7 @@ mod tests {
|
||||
token_ids: vec![33],
|
||||
logprobs: None,
|
||||
finish_reason: Some(FinishReason::stop_eos()),
|
||||
cached_token_count: 2,
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
]);
|
||||
@@ -417,7 +425,10 @@ mod tests {
|
||||
let chunks: Vec<_> = generate_chunk_stream(
|
||||
stream,
|
||||
"raw-stream".to_string(),
|
||||
false,
|
||||
ApiServerOptions {
|
||||
enable_prompt_tokens_details: true,
|
||||
..Default::default()
|
||||
},
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
@@ -433,9 +444,29 @@ mod tests {
|
||||
chunks[0].usage.as_ref().expect("chunk usage").prompt_tokens,
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
chunks[0]
|
||||
.usage
|
||||
.as_ref()
|
||||
.expect("chunk usage")
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.map(|details| details.cached_tokens),
|
||||
Some(2)
|
||||
);
|
||||
assert_eq!(
|
||||
chunks[1].usage.as_ref().expect("final usage").prompt_tokens,
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
chunks[1]
|
||||
.usage
|
||||
.as_ref()
|
||||
.expect("final usage")
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.map(|details| details.cached_tokens),
|
||||
Some(2)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ use vllm_chat::{
|
||||
use vllm_engine_core_client::protocol::StopReason;
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_chat_request};
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::routes::openai::chat_completions::types::{
|
||||
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
|
||||
@@ -62,7 +63,7 @@ pub async fn chat_completions(
|
||||
);
|
||||
|
||||
let created = unix_timestamp();
|
||||
let log_request = state.enable_log_requests;
|
||||
let api_server_options = state.api_server_options;
|
||||
|
||||
let chat_stream =
|
||||
match state.chat.chat(prepared.chat_request).instrument(request_span.clone()).await {
|
||||
@@ -82,7 +83,7 @@ pub async fn chat_completions(
|
||||
prepared.request_id,
|
||||
prepared.response_model,
|
||||
created,
|
||||
log_request,
|
||||
api_server_options,
|
||||
prepared.options,
|
||||
);
|
||||
let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
@@ -94,7 +95,7 @@ pub async fn chat_completions(
|
||||
prepared.request_id,
|
||||
prepared.response_model,
|
||||
created,
|
||||
log_request,
|
||||
api_server_options,
|
||||
prepared.options,
|
||||
)
|
||||
.instrument(request_span.clone())
|
||||
@@ -113,7 +114,11 @@ async fn collect_chat_completion(
|
||||
request_id: String,
|
||||
response_model: String,
|
||||
created: u64,
|
||||
log_request: bool,
|
||||
ApiServerOptions {
|
||||
enable_log_requests,
|
||||
enable_prompt_tokens_details,
|
||||
..
|
||||
}: ApiServerOptions,
|
||||
ResponseOptions {
|
||||
// Ignored: non-streaming responses always include usage.
|
||||
include_usage: _,
|
||||
@@ -133,12 +138,11 @@ async fn collect_chat_completion(
|
||||
})?;
|
||||
let CollectedAssistantMessage {
|
||||
message,
|
||||
prompt_token_count,
|
||||
prompt_token_ids,
|
||||
prompt_logprobs,
|
||||
logprobs,
|
||||
token_ids,
|
||||
output_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
kv_transfer_params,
|
||||
} = collected;
|
||||
@@ -183,9 +187,9 @@ async fn collect_chat_completion(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let usage = Usage::from_counts(prompt_token_count as u32, output_token_count as u32);
|
||||
let usage = Usage::from_token_usage(usage, enable_prompt_tokens_details);
|
||||
|
||||
if log_request {
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
model = %response_model,
|
||||
prompt_tokens = usage.prompt_tokens,
|
||||
@@ -231,7 +235,11 @@ async fn chat_completion_chunk_stream(
|
||||
request_id: String,
|
||||
response_model: String,
|
||||
created: u64,
|
||||
log_request: bool,
|
||||
ApiServerOptions {
|
||||
enable_log_requests,
|
||||
enable_prompt_tokens_details,
|
||||
..
|
||||
}: ApiServerOptions,
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
requested_logprobs,
|
||||
@@ -391,17 +399,16 @@ async fn chat_completion_chunk_stream(
|
||||
debug!("ending current tool call");
|
||||
}
|
||||
Ok(ChatEvent::Done {
|
||||
prompt_token_count,
|
||||
usage,
|
||||
finish_reason,
|
||||
output_token_count,
|
||||
..
|
||||
}) => {
|
||||
if log_request {
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
stream = true,
|
||||
model = %response_model,
|
||||
prompt_tokens = prompt_token_count,
|
||||
output_tokens = output_token_count,
|
||||
prompt_tokens = usage.prompt_token_count,
|
||||
output_tokens = usage.output_token_count,
|
||||
finish_reason = finish_reason.as_str(),
|
||||
"chat completion finished"
|
||||
);
|
||||
@@ -436,7 +443,7 @@ async fn chat_completion_chunk_stream(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
Usage::from_counts(prompt_token_count as u32, output_token_count as u32),
|
||||
Usage::from_token_usage(usage, enable_prompt_tokens_details),
|
||||
))
|
||||
.await;
|
||||
}
|
||||
@@ -804,7 +811,10 @@ mod tests {
|
||||
use vllm_engine_core_client::protocol::StopReason;
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
||||
|
||||
use super::{ResponseOptions, block_delta_chunk, chat_completion_chunk_stream, final_chunk};
|
||||
use super::{
|
||||
ApiServerOptions, ResponseOptions, block_delta_chunk, chat_completion_chunk_stream,
|
||||
final_chunk,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn text_chunk_uses_content_only_delta() {
|
||||
@@ -917,8 +927,11 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 1,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -929,8 +942,12 @@ mod tests {
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
ApiServerOptions {
|
||||
enable_prompt_tokens_details: true,
|
||||
..Default::default()
|
||||
},
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
requested_logprobs: true,
|
||||
include_reasoning: true,
|
||||
..Default::default()
|
||||
@@ -942,11 +959,21 @@ mod tests {
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("stream chunks");
|
||||
|
||||
assert_eq!(chunks.len(), 3);
|
||||
assert_eq!(chunks.len(), 4);
|
||||
assert_eq!(chunks[1].choices[0].delta.content.as_deref(), Some("hi"));
|
||||
let logprobs = chunks[1].choices[0].logprobs.as_ref().expect("logprobs");
|
||||
let content = logprobs.content.as_ref().expect("logprobs content");
|
||||
assert_eq!(content[0].token, "hi");
|
||||
assert_eq!(
|
||||
chunks[3]
|
||||
.usage
|
||||
.as_ref()
|
||||
.expect("usage")
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.map(|details| details.cached_tokens),
|
||||
Some(1)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -980,8 +1007,11 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -992,7 +1022,7 @@ mod tests {
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
requested_logprobs: true,
|
||||
include_reasoning: true,
|
||||
@@ -1032,8 +1062,11 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -1044,7 +1077,7 @@ mod tests {
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions::default(),
|
||||
)
|
||||
.collect::<Vec<_>>()
|
||||
@@ -1110,8 +1143,11 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -1122,7 +1158,7 @@ mod tests {
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
requested_logprobs: true,
|
||||
return_token_ids: true,
|
||||
@@ -1240,8 +1276,11 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 4,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 4,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -1252,7 +1291,7 @@ mod tests {
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
requested_logprobs: true,
|
||||
return_token_ids: true,
|
||||
@@ -1318,8 +1357,11 @@ mod tests {
|
||||
}),
|
||||
Ok(ChatEvent::Done {
|
||||
message: Default::default(),
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 1,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -1330,7 +1372,7 @@ mod tests {
|
||||
"chatcmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_reasoning: true,
|
||||
..Default::default()
|
||||
|
||||
@@ -24,6 +24,7 @@ use super::utils::logprobs::{
|
||||
text_len,
|
||||
};
|
||||
use super::utils::types::Usage;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error};
|
||||
use crate::routes::openai::completions::types::{
|
||||
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
|
||||
@@ -56,7 +57,7 @@ pub async fn completions(
|
||||
);
|
||||
|
||||
let created = unix_timestamp();
|
||||
let log_request = state.enable_log_requests;
|
||||
let api_server_options = state.api_server_options;
|
||||
let text_stream = match state
|
||||
.chat
|
||||
.text()
|
||||
@@ -80,7 +81,7 @@ pub async fn completions(
|
||||
prepared.request_id,
|
||||
prepared.response_model,
|
||||
created,
|
||||
log_request,
|
||||
api_server_options,
|
||||
prepared.options,
|
||||
);
|
||||
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
@@ -92,7 +93,7 @@ pub async fn completions(
|
||||
prepared.request_id,
|
||||
prepared.response_model,
|
||||
created,
|
||||
log_request,
|
||||
api_server_options,
|
||||
prepared.options,
|
||||
)
|
||||
.instrument(request_span.clone())
|
||||
@@ -111,7 +112,11 @@ async fn collect_completion(
|
||||
request_id: String,
|
||||
response_model: String,
|
||||
created: u64,
|
||||
log_request: bool,
|
||||
ApiServerOptions {
|
||||
enable_log_requests,
|
||||
enable_prompt_tokens_details,
|
||||
..
|
||||
}: ApiServerOptions,
|
||||
ResponseOptions {
|
||||
// Ignored: non-streaming responses always include usage.
|
||||
include_usage: _,
|
||||
@@ -159,12 +164,9 @@ async fn collect_completion(
|
||||
Some(prompt) => format!("{prompt}{}", collected.text),
|
||||
};
|
||||
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
|
||||
let usage = Usage::from_counts(
|
||||
collected.prompt_token_ids.len() as u32,
|
||||
collected.token_ids.len() as u32,
|
||||
);
|
||||
let usage = Usage::from_token_usage(collected.usage, enable_prompt_tokens_details);
|
||||
|
||||
if log_request {
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
model = %response_model,
|
||||
prompt_tokens = usage.prompt_tokens,
|
||||
@@ -202,7 +204,11 @@ async fn completion_chunk_stream(
|
||||
request_id: String,
|
||||
response_model: String,
|
||||
created: u64,
|
||||
log_request: bool,
|
||||
ApiServerOptions {
|
||||
enable_log_requests,
|
||||
enable_prompt_tokens_details,
|
||||
..
|
||||
}: ApiServerOptions,
|
||||
ResponseOptions {
|
||||
include_usage,
|
||||
echo,
|
||||
@@ -275,12 +281,12 @@ async fn completion_chunk_stream(
|
||||
visible_text_len = visible_text_len.saturating_add(delta_text_len);
|
||||
|
||||
if let Some(finished) = finished {
|
||||
if log_request {
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
stream = true,
|
||||
model = %response_model,
|
||||
prompt_tokens = finished.prompt_token_count,
|
||||
output_tokens = finished.output_token_count,
|
||||
prompt_tokens = finished.usage.prompt_token_count,
|
||||
output_tokens = finished.usage.output_token_count,
|
||||
finish_reason = finished.finish_reason.as_str(),
|
||||
"completion finished"
|
||||
);
|
||||
@@ -298,10 +304,7 @@ async fn completion_chunk_stream(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
Usage::from_counts(
|
||||
finished.prompt_token_count as u32,
|
||||
finished.output_token_count as u32,
|
||||
),
|
||||
Usage::from_token_usage(finished.usage, enable_prompt_tokens_details),
|
||||
)))
|
||||
.await;
|
||||
}
|
||||
@@ -431,7 +434,9 @@ mod tests {
|
||||
FinishReason, Finished,
|
||||
};
|
||||
|
||||
use super::{CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk};
|
||||
use super::{
|
||||
ApiServerOptions, CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_stop_finish_reason() {
|
||||
@@ -512,8 +517,11 @@ mod tests {
|
||||
}],
|
||||
}),
|
||||
finished: Some(Finished {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 3,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
@@ -525,8 +533,12 @@ mod tests {
|
||||
"cmpl-1".to_string(),
|
||||
"model".to_string(),
|
||||
1,
|
||||
false,
|
||||
ApiServerOptions {
|
||||
enable_prompt_tokens_details: true,
|
||||
..Default::default()
|
||||
},
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
requested_logprobs: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -565,5 +577,21 @@ mod tests {
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
|
||||
}
|
||||
|
||||
match &chunks[3] {
|
||||
CompletionSseChunk::Usage(chunk) => {
|
||||
assert_eq!(
|
||||
chunk
|
||||
.usage
|
||||
.as_ref()
|
||||
.expect("usage")
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.map(|details| details.cached_tokens),
|
||||
Some(3)
|
||||
);
|
||||
}
|
||||
CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::slice;
|
||||
use llm_multimodal::ImageDetail;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use vllm_llm::TokenUsage;
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
@@ -313,29 +314,82 @@ pub enum MessageContent {
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
pub completion_tokens: Option<u32>,
|
||||
pub prompt_tokens: usize,
|
||||
pub total_tokens: usize,
|
||||
pub completion_tokens: Option<usize>,
|
||||
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
|
||||
}
|
||||
|
||||
impl Usage {
|
||||
/// Create a Usage from prompt and completion token counts.
|
||||
pub fn from_counts(prompt_tokens: u32, completion_tokens: u32) -> Self {
|
||||
/// Create a Usage with prompt-token cache details.
|
||||
pub fn from_counts(
|
||||
prompt_tokens: usize,
|
||||
completion_tokens: usize,
|
||||
cached_tokens: Option<usize>,
|
||||
) -> Self {
|
||||
Self {
|
||||
prompt_tokens,
|
||||
total_tokens: prompt_tokens + completion_tokens,
|
||||
completion_tokens: Some(completion_tokens),
|
||||
prompt_tokens_details: None,
|
||||
prompt_tokens_details: cached_tokens
|
||||
.filter(|&c| c > 0)
|
||||
.map(|c| PromptTokenUsageInfo { cached_tokens: c }),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_token_usage(usage: TokenUsage, enable_prompt_tokens_details: bool) -> Self {
|
||||
Self::from_counts(
|
||||
usage.prompt_token_count,
|
||||
usage.output_token_count,
|
||||
enable_prompt_tokens_details.then_some(usage.cached_token_count),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `PromptTokenUsageInfo` class.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct PromptTokenUsageInfo {
|
||||
pub cached_tokens: Option<u32>,
|
||||
pub cached_tokens: usize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod usage_tests {
|
||||
use vllm_llm::TokenUsage;
|
||||
|
||||
use super::Usage;
|
||||
|
||||
#[test]
|
||||
fn token_usage_hides_prompt_token_details_by_default() {
|
||||
let usage = Usage::from_token_usage(
|
||||
TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 3,
|
||||
},
|
||||
false,
|
||||
);
|
||||
|
||||
assert_eq!(usage.prompt_tokens, 5);
|
||||
assert_eq!(usage.completion_tokens, Some(2));
|
||||
assert!(usage.prompt_tokens_details.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_usage_includes_prompt_token_details_when_enabled() {
|
||||
let usage = Usage::from_token_usage(
|
||||
TokenUsage {
|
||||
prompt_token_count: 5,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 3,
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
usage.prompt_tokens_details.as_ref().map(|details| details.cached_tokens),
|
||||
Some(3)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// OpenAI completions-style logprobs.
|
||||
|
||||
@@ -43,6 +43,7 @@ use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::state::AppState;
|
||||
|
||||
fn request_output(
|
||||
@@ -787,8 +788,12 @@ async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) {
|
||||
)
|
||||
.await;
|
||||
let app = build_router(Arc::new(
|
||||
AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat)
|
||||
.with_request_id_headers(true),
|
||||
AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat).with_api_server_options(
|
||||
ApiServerOptions {
|
||||
enable_request_id_headers: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
));
|
||||
(app, engine_task)
|
||||
}
|
||||
|
||||
@@ -134,11 +134,12 @@ impl Normalizable for DetokenizeRequest {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::routes::openai::utils::types::{ChatMessage, MessageContent};
|
||||
use serde_json::json;
|
||||
use vllm_chat::ChatTool;
|
||||
|
||||
use super::*;
|
||||
use crate::routes::openai::utils::types::{ChatMessage, MessageContent};
|
||||
|
||||
#[test]
|
||||
fn tokenize_request_converts_openai_tools() {
|
||||
// The untagged `TokenizeRequest` must resolve a messages+tools body to
|
||||
|
||||
@@ -9,6 +9,7 @@ use vllm_chat::ChatLlm;
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
|
||||
use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
|
||||
|
||||
@@ -27,10 +28,8 @@ pub struct AppState {
|
||||
served_model_names: Vec<String>,
|
||||
/// Shared chat facade used by all requests.
|
||||
pub chat: ChatLlm,
|
||||
/// Whether to log a summary line for each completed request.
|
||||
pub enable_log_requests: bool,
|
||||
/// Whether to set X-Request-Id on every HTTP response.
|
||||
pub enable_request_id_headers: bool,
|
||||
/// HTTP/API-server behavior switches.
|
||||
pub api_server_options: ApiServerOptions,
|
||||
/// Runtime server information returned by `/server_info`, when available.
|
||||
server_info: Option<ServerInfoSnapshot>,
|
||||
/// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes.
|
||||
@@ -58,8 +57,7 @@ impl AppState {
|
||||
Self {
|
||||
served_model_names,
|
||||
chat,
|
||||
enable_log_requests: false,
|
||||
enable_request_id_headers: false,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
server_info: None,
|
||||
api_key_hashes: Vec::new(),
|
||||
server_load: AtomicU64::new(0),
|
||||
@@ -67,15 +65,9 @@ impl AppState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable per-request completion logging.
|
||||
pub fn with_log_requests(mut self, enabled: bool) -> Self {
|
||||
self.enable_log_requests = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable X-Request-Id response headers.
|
||||
pub fn with_request_id_headers(mut self, enabled: bool) -> Self {
|
||||
self.enable_request_id_headers = enabled;
|
||||
/// Set HTTP/API-server behavior switches.
|
||||
pub fn with_api_server_options(mut self, options: ApiServerOptions) -> Self {
|
||||
self.api_server_options = options;
|
||||
self
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user