[Rust Frontend] Populate cached_token_count in responses (#44887)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-10 20:50:05 -07:00
committed by GitHub
parent 85a0ffae42
commit 5d5591d99b
34 changed files with 556 additions and 262 deletions
@@ -14,8 +14,8 @@ use tokio_util::sync::CancellationToken;
use tracing_subscriber::EnvFilter;
use vllm_engine_core_client::TransportMode;
use vllm_server::{
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
RendererSelection, serve,
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode,
ParserSelection, RendererSelection, serve,
};
#[derive(Debug, Parser)]
@@ -68,8 +68,7 @@ async fn main() -> Result<()> {
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
enable_log_requests: false,
enable_request_id_headers: false,
api_server_options: ApiServerOptions::default(),
api_keys: Vec::new(),
disable_log_stats: false,
grpc_port: None,
+13 -4
View File
@@ -34,6 +34,17 @@ pub enum CoordinatorMode {
External { address: String },
}
/// HTTP/API-server behavior switches that affect route-layer responses.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Default)]
pub struct ApiServerOptions {
/// Log a summary line for each completed request.
pub enable_log_requests: bool,
/// When `true`, include prompt token cache details in response usage.
pub enable_prompt_tokens_details: bool,
/// When `true`, set `X-Request-Id` on every HTTP response.
pub enable_request_id_headers: bool,
}
/// Normalized runtime configuration for the minimal OpenAI-compatible server.
#[derive(Educe, Clone, PartialEq, Eq, Serialize)]
#[educe(Debug)]
@@ -66,10 +77,8 @@ pub struct Config {
pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
/// How to serialize `message.content` for chat-template rendering.
pub chat_template_content_format: ChatTemplateContentFormatOption,
/// Log a summary line for each completed request.
pub enable_log_requests: bool,
/// When `true`, set `X-Request-Id` on every HTTP response.
pub enable_request_id_headers: bool,
/// HTTP/API-server behavior switches.
pub api_server_options: ApiServerOptions,
/// API keys accepted as bearer tokens for guarded routes.
#[serde(skip_serializing)]
#[educe(Debug(method(fmt_redacted_api_keys)))]
+6 -3
View File
@@ -350,7 +350,7 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
};
pb::FinishInfo {
num_output_tokens: finished.output_token_count as u32,
num_output_tokens: finished.usage.output_token_count as u32,
finish_reason,
stop_reason,
kv_transfer_params: finished.kv_transfer_params.as_ref().and_then(json_to_proto_struct),
@@ -590,8 +590,11 @@ mod tests {
fn finished(reason: FinishReason) -> Finished {
Finished {
prompt_token_count: 0,
output_token_count: 0,
usage: vllm_llm::TokenUsage {
prompt_token_count: 0,
output_token_count: 0,
cached_token_count: 0,
},
finish_reason: reason,
kv_transfer_params: None,
}
+1 -2
View File
@@ -71,8 +71,7 @@ impl pb::generate_server::Generate for GenerateServiceImpl {
);
let finish_info = vllm_text::Finished {
prompt_token_count: collected.prompt_token_ids.len(),
output_token_count: collected.token_ids.len(),
usage: collected.usage,
finish_reason: collected.finish_reason,
kv_transfer_params: collected.kv_transfer_params,
};
+2 -3
View File
@@ -16,7 +16,7 @@ use std::sync::{Arc, OnceLock};
use anyhow::{Context as _, Result};
use axum::Router;
use axum::serve::ListenerExt as _;
pub use config::{Config, CoordinatorMode, HttpListenerMode};
pub use config::{ApiServerOptions, Config, CoordinatorMode, HttpListenerMode};
use tokio::net::TcpListener;
use tokio::time::{Instant, sleep_until};
use tokio_stream::wrappers::TcpListenerStream;
@@ -91,8 +91,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
Ok(Arc::new(
AppState::new(served_model_names, chat)
.with_log_requests(config.enable_log_requests)
.with_request_id_headers(config.enable_request_id_headers)
.with_api_server_options(config.api_server_options)
.with_server_info(ServerInfoSnapshot::from_config(config))
.with_api_keys(config.api_keys.clone()),
))
+1 -1
View File
@@ -100,7 +100,7 @@ fn build_router_with_options(
.route("/server_info", get(server_info::server_info))
}
let enable_request_id_headers = state.enable_request_id_headers;
let enable_request_id_headers = state.api_server_options.enable_request_id_headers;
let enable_api_key_auth = state.has_api_keys();
let mut router = router
.with_state(state.clone())
@@ -19,7 +19,7 @@ use tracing::{error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
use vllm_llm::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _,
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
};
use self::convert::{ResponseOptions, prepare_generate_request};
@@ -27,6 +27,7 @@ use self::types::{
GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
GenerateResponseStreamChoice, GenerateStreamResponse,
};
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error};
use crate::routes::openai::utils::logprobs::clamp_logprob;
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage};
@@ -53,7 +54,7 @@ pub async fn generate(
engine_request_id = tracing::field::Empty,
);
let log_request = state.enable_log_requests;
let api_server_options = state.api_server_options;
let stream = prepared.stream;
let raw_stream = match state
.chat
@@ -76,7 +77,7 @@ pub async fn generate(
let chunk_stream = generate_chunk_stream(
raw_stream,
prepared.request_id,
log_request,
api_server_options,
prepared.options,
);
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
@@ -98,7 +99,7 @@ pub async fn generate(
let response = match collect_generate(
collected,
prepared.request_id,
log_request,
api_server_options,
prepared.options,
) {
Ok(response) => response,
@@ -112,7 +113,11 @@ pub async fn generate(
async fn generate_chunk_stream(
stream: impl Stream<Item = vllm_llm::Result<GenerateOutput>>,
request_id: String,
log_request: bool,
ApiServerOptions {
enable_log_requests,
enable_prompt_tokens_details,
..
}: ApiServerOptions,
ResponseOptions {
include_usage,
include_continuous_usage,
@@ -123,20 +128,21 @@ async fn generate_chunk_stream(
mut y: TryYielder<GenerateStreamResponse, ApiError>,
) -> Result<(), ApiError> {
pin_mut!(stream);
let mut prompt_tokens: Option<u32> = None;
let mut output_tokens = 0_u32;
let mut prompt_tokens = None;
let mut usage = TokenUsage::default();
while let Some(next) = stream.next().await {
match next {
Ok(output) => {
if prompt_tokens.is_none() {
prompt_tokens =
output.prompt_info.as_ref().map(|info| info.prompt_token_ids.len() as u32);
output.prompt_info.as_ref().map(|info| info.prompt_token_ids.len());
}
let usage_prompt_tokens = prompt_tokens.unwrap_or_default();
usage.prompt_token_count = prompt_tokens.unwrap_or_default();
usage.cached_token_count = usage.cached_token_count.max(output.cached_token_count);
let token_ids = output.token_ids;
output_tokens = output_tokens.saturating_add(token_ids.len() as u32);
usage.output_token_count = usage.output_token_count.saturating_add(token_ids.len());
let finish_reason = output.finish_reason;
if matches!(finish_reason.as_ref(), Some(FinishReason::Error)) {
@@ -144,12 +150,12 @@ async fn generate_chunk_stream(
}
if let Some(finish_reason) = finish_reason.as_ref()
&& log_request
&& enable_log_requests
{
info!(
stream = true,
prompt_tokens = usage_prompt_tokens,
output_tokens,
prompt_tokens = usage.prompt_token_count,
output_tokens = usage.output_token_count,
finish_reason = finish_reason.as_str(),
"generate finished"
);
@@ -179,7 +185,7 @@ async fn generate_chunk_stream(
token_ids,
}],
usage: include_continuous_usage
.then(|| Usage::from_counts(usage_prompt_tokens, output_tokens)),
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
})
.await;
}
@@ -197,10 +203,7 @@ async fn generate_chunk_stream(
y.yield_ok(GenerateStreamResponse {
request_id,
choices: Vec::new(),
usage: Some(Usage::from_counts(
prompt_tokens.unwrap_or_default(),
output_tokens,
)),
usage: Some(Usage::from_token_usage(usage, enable_prompt_tokens_details)),
})
.await;
}
@@ -211,7 +214,10 @@ async fn generate_chunk_stream(
fn collect_generate(
collected: CollectedGenerateOutput,
request_id: String,
log_request: bool,
ApiServerOptions {
enable_log_requests,
..
}: ApiServerOptions,
ResponseOptions {
// Ignored: non-streaming raw generate responses do not include usage.
include_usage: _,
@@ -244,7 +250,7 @@ fn collect_generate(
};
let finish_reason = collected.finish_reason.as_str().to_string();
if log_request {
if enable_log_requests {
info!(
prompt_tokens = collected.prompt_token_ids.len(),
output_tokens = collected.token_ids.len(),
@@ -399,6 +405,7 @@ mod tests {
token_ids: Vec::new(),
logprobs: None,
finish_reason: None,
cached_token_count: 0,
kv_transfer_params: None,
}),
Ok(GenerateOutput {
@@ -410,6 +417,7 @@ mod tests {
token_ids: vec![33],
logprobs: None,
finish_reason: Some(FinishReason::stop_eos()),
cached_token_count: 2,
kv_transfer_params: None,
}),
]);
@@ -417,7 +425,10 @@ mod tests {
let chunks: Vec<_> = generate_chunk_stream(
stream,
"raw-stream".to_string(),
false,
ApiServerOptions {
enable_prompt_tokens_details: true,
..Default::default()
},
ResponseOptions {
include_usage: true,
include_continuous_usage: true,
@@ -433,9 +444,29 @@ mod tests {
chunks[0].usage.as_ref().expect("chunk usage").prompt_tokens,
2
);
assert_eq!(
chunks[0]
.usage
.as_ref()
.expect("chunk usage")
.prompt_tokens_details
.as_ref()
.map(|details| details.cached_tokens),
Some(2)
);
assert_eq!(
chunks[1].usage.as_ref().expect("final usage").prompt_tokens,
2
);
assert_eq!(
chunks[1]
.usage
.as_ref()
.expect("final usage")
.prompt_tokens_details
.as_ref()
.map(|details| details.cached_tokens),
Some(2)
);
}
}
@@ -24,6 +24,7 @@ use vllm_chat::{
use vllm_engine_core_client::protocol::StopReason;
use self::convert::{ResponseOptions, prepare_chat_request};
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error};
use crate::routes::openai::chat_completions::types::{
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
@@ -62,7 +63,7 @@ pub async fn chat_completions(
);
let created = unix_timestamp();
let log_request = state.enable_log_requests;
let api_server_options = state.api_server_options;
let chat_stream =
match state.chat.chat(prepared.chat_request).instrument(request_span.clone()).await {
@@ -82,7 +83,7 @@ pub async fn chat_completions(
prepared.request_id,
prepared.response_model,
created,
log_request,
api_server_options,
prepared.options,
);
let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span);
@@ -94,7 +95,7 @@ pub async fn chat_completions(
prepared.request_id,
prepared.response_model,
created,
log_request,
api_server_options,
prepared.options,
)
.instrument(request_span.clone())
@@ -113,7 +114,11 @@ async fn collect_chat_completion(
request_id: String,
response_model: String,
created: u64,
log_request: bool,
ApiServerOptions {
enable_log_requests,
enable_prompt_tokens_details,
..
}: ApiServerOptions,
ResponseOptions {
// Ignored: non-streaming responses always include usage.
include_usage: _,
@@ -133,12 +138,11 @@ async fn collect_chat_completion(
})?;
let CollectedAssistantMessage {
message,
prompt_token_count,
prompt_token_ids,
prompt_logprobs,
logprobs,
token_ids,
output_token_count,
usage,
finish_reason,
kv_transfer_params,
} = collected;
@@ -183,9 +187,9 @@ async fn collect_chat_completion(
} else {
None
};
let usage = Usage::from_counts(prompt_token_count as u32, output_token_count as u32);
let usage = Usage::from_token_usage(usage, enable_prompt_tokens_details);
if log_request {
if enable_log_requests {
info!(
model = %response_model,
prompt_tokens = usage.prompt_tokens,
@@ -231,7 +235,11 @@ async fn chat_completion_chunk_stream(
request_id: String,
response_model: String,
created: u64,
log_request: bool,
ApiServerOptions {
enable_log_requests,
enable_prompt_tokens_details,
..
}: ApiServerOptions,
ResponseOptions {
include_usage,
requested_logprobs,
@@ -391,17 +399,16 @@ async fn chat_completion_chunk_stream(
debug!("ending current tool call");
}
Ok(ChatEvent::Done {
prompt_token_count,
usage,
finish_reason,
output_token_count,
..
}) => {
if log_request {
if enable_log_requests {
info!(
stream = true,
model = %response_model,
prompt_tokens = prompt_token_count,
output_tokens = output_token_count,
prompt_tokens = usage.prompt_token_count,
output_tokens = usage.output_token_count,
finish_reason = finish_reason.as_str(),
"chat completion finished"
);
@@ -436,7 +443,7 @@ async fn chat_completion_chunk_stream(
&request_id,
&response_model,
created,
Usage::from_counts(prompt_token_count as u32, output_token_count as u32),
Usage::from_token_usage(usage, enable_prompt_tokens_details),
))
.await;
}
@@ -804,7 +811,10 @@ mod tests {
use vllm_engine_core_client::protocol::StopReason;
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
use super::{ResponseOptions, block_delta_chunk, chat_completion_chunk_stream, final_chunk};
use super::{
ApiServerOptions, ResponseOptions, block_delta_chunk, chat_completion_chunk_stream,
final_chunk,
};
#[test]
fn text_chunk_uses_content_only_delta() {
@@ -917,8 +927,11 @@ mod tests {
}),
Ok(ChatEvent::Done {
message: Default::default(),
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 1,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -929,8 +942,12 @@ mod tests {
"chatcmpl-1".to_string(),
"model".to_string(),
1,
false,
ApiServerOptions {
enable_prompt_tokens_details: true,
..Default::default()
},
ResponseOptions {
include_usage: true,
requested_logprobs: true,
include_reasoning: true,
..Default::default()
@@ -942,11 +959,21 @@ mod tests {
.collect::<Result<Vec<_>, _>>()
.expect("stream chunks");
assert_eq!(chunks.len(), 3);
assert_eq!(chunks.len(), 4);
assert_eq!(chunks[1].choices[0].delta.content.as_deref(), Some("hi"));
let logprobs = chunks[1].choices[0].logprobs.as_ref().expect("logprobs");
let content = logprobs.content.as_ref().expect("logprobs content");
assert_eq!(content[0].token, "hi");
assert_eq!(
chunks[3]
.usage
.as_ref()
.expect("usage")
.prompt_tokens_details
.as_ref()
.map(|details| details.cached_tokens),
Some(1)
);
}
#[tokio::test]
@@ -980,8 +1007,11 @@ mod tests {
}),
Ok(ChatEvent::Done {
message: Default::default(),
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -992,7 +1022,7 @@ mod tests {
"chatcmpl-1".to_string(),
"model".to_string(),
1,
false,
ApiServerOptions::default(),
ResponseOptions {
requested_logprobs: true,
include_reasoning: true,
@@ -1032,8 +1062,11 @@ mod tests {
}),
Ok(ChatEvent::Done {
message: Default::default(),
prompt_token_count: 1,
output_token_count: 2,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 2,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -1044,7 +1077,7 @@ mod tests {
"chatcmpl-1".to_string(),
"model".to_string(),
1,
false,
ApiServerOptions::default(),
ResponseOptions::default(),
)
.collect::<Vec<_>>()
@@ -1110,8 +1143,11 @@ mod tests {
}),
Ok(ChatEvent::Done {
message: Default::default(),
prompt_token_count: 1,
output_token_count: 2,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 2,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -1122,7 +1158,7 @@ mod tests {
"chatcmpl-1".to_string(),
"model".to_string(),
1,
false,
ApiServerOptions::default(),
ResponseOptions {
requested_logprobs: true,
return_token_ids: true,
@@ -1240,8 +1276,11 @@ mod tests {
}),
Ok(ChatEvent::Done {
message: Default::default(),
prompt_token_count: 1,
output_token_count: 4,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 4,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -1252,7 +1291,7 @@ mod tests {
"chatcmpl-1".to_string(),
"model".to_string(),
1,
false,
ApiServerOptions::default(),
ResponseOptions {
requested_logprobs: true,
return_token_ids: true,
@@ -1318,8 +1357,11 @@ mod tests {
}),
Ok(ChatEvent::Done {
message: Default::default(),
prompt_token_count: 1,
output_token_count: 1,
usage: vllm_llm::TokenUsage {
prompt_token_count: 1,
output_token_count: 1,
cached_token_count: 0,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -1330,7 +1372,7 @@ mod tests {
"chatcmpl-1".to_string(),
"model".to_string(),
1,
false,
ApiServerOptions::default(),
ResponseOptions {
include_reasoning: true,
..Default::default()
@@ -24,6 +24,7 @@ use super::utils::logprobs::{
text_len,
};
use super::utils::types::Usage;
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error};
use crate::routes::openai::completions::types::{
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
@@ -56,7 +57,7 @@ pub async fn completions(
);
let created = unix_timestamp();
let log_request = state.enable_log_requests;
let api_server_options = state.api_server_options;
let text_stream = match state
.chat
.text()
@@ -80,7 +81,7 @@ pub async fn completions(
prepared.request_id,
prepared.response_model,
created,
log_request,
api_server_options,
prepared.options,
);
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
@@ -92,7 +93,7 @@ pub async fn completions(
prepared.request_id,
prepared.response_model,
created,
log_request,
api_server_options,
prepared.options,
)
.instrument(request_span.clone())
@@ -111,7 +112,11 @@ async fn collect_completion(
request_id: String,
response_model: String,
created: u64,
log_request: bool,
ApiServerOptions {
enable_log_requests,
enable_prompt_tokens_details,
..
}: ApiServerOptions,
ResponseOptions {
// Ignored: non-streaming responses always include usage.
include_usage: _,
@@ -159,12 +164,9 @@ async fn collect_completion(
Some(prompt) => format!("{prompt}{}", collected.text),
};
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
let usage = Usage::from_counts(
collected.prompt_token_ids.len() as u32,
collected.token_ids.len() as u32,
);
let usage = Usage::from_token_usage(collected.usage, enable_prompt_tokens_details);
if log_request {
if enable_log_requests {
info!(
model = %response_model,
prompt_tokens = usage.prompt_tokens,
@@ -202,7 +204,11 @@ async fn completion_chunk_stream(
request_id: String,
response_model: String,
created: u64,
log_request: bool,
ApiServerOptions {
enable_log_requests,
enable_prompt_tokens_details,
..
}: ApiServerOptions,
ResponseOptions {
include_usage,
echo,
@@ -275,12 +281,12 @@ async fn completion_chunk_stream(
visible_text_len = visible_text_len.saturating_add(delta_text_len);
if let Some(finished) = finished {
if log_request {
if enable_log_requests {
info!(
stream = true,
model = %response_model,
prompt_tokens = finished.prompt_token_count,
output_tokens = finished.output_token_count,
prompt_tokens = finished.usage.prompt_token_count,
output_tokens = finished.usage.output_token_count,
finish_reason = finished.finish_reason.as_str(),
"completion finished"
);
@@ -298,10 +304,7 @@ async fn completion_chunk_stream(
&request_id,
&response_model,
created,
Usage::from_counts(
finished.prompt_token_count as u32,
finished.output_token_count as u32,
),
Usage::from_token_usage(finished.usage, enable_prompt_tokens_details),
)))
.await;
}
@@ -431,7 +434,9 @@ mod tests {
FinishReason, Finished,
};
use super::{CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk};
use super::{
ApiServerOptions, CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk,
};
#[test]
fn final_chunk_maps_stop_finish_reason() {
@@ -512,8 +517,11 @@ mod tests {
}],
}),
finished: Some(Finished {
prompt_token_count: 5,
output_token_count: 2,
usage: vllm_llm::TokenUsage {
prompt_token_count: 5,
output_token_count: 2,
cached_token_count: 3,
},
finish_reason: FinishReason::stop_eos(),
kv_transfer_params: None,
}),
@@ -525,8 +533,12 @@ mod tests {
"cmpl-1".to_string(),
"model".to_string(),
1,
false,
ApiServerOptions {
enable_prompt_tokens_details: true,
..Default::default()
},
ResponseOptions {
include_usage: true,
requested_logprobs: Some(1),
..Default::default()
},
@@ -565,5 +577,21 @@ mod tests {
}
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
}
match &chunks[3] {
CompletionSseChunk::Usage(chunk) => {
assert_eq!(
chunk
.usage
.as_ref()
.expect("usage")
.prompt_tokens_details
.as_ref()
.map(|details| details.cached_tokens),
Some(3)
);
}
CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"),
}
}
}
@@ -4,6 +4,7 @@ use std::slice;
use llm_multimodal::ImageDetail;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use vllm_llm::TokenUsage;
// ============================================================================
// Constants
@@ -313,29 +314,82 @@ pub enum MessageContent {
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub struct Usage {
pub prompt_tokens: u32,
pub total_tokens: u32,
pub completion_tokens: Option<u32>,
pub prompt_tokens: usize,
pub total_tokens: usize,
pub completion_tokens: Option<usize>,
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
}
impl Usage {
/// Create a Usage from prompt and completion token counts.
pub fn from_counts(prompt_tokens: u32, completion_tokens: u32) -> Self {
/// Create a Usage with prompt-token cache details.
pub fn from_counts(
prompt_tokens: usize,
completion_tokens: usize,
cached_tokens: Option<usize>,
) -> Self {
Self {
prompt_tokens,
total_tokens: prompt_tokens + completion_tokens,
completion_tokens: Some(completion_tokens),
prompt_tokens_details: None,
prompt_tokens_details: cached_tokens
.filter(|&c| c > 0)
.map(|c| PromptTokenUsageInfo { cached_tokens: c }),
}
}
pub fn from_token_usage(usage: TokenUsage, enable_prompt_tokens_details: bool) -> Self {
Self::from_counts(
usage.prompt_token_count,
usage.output_token_count,
enable_prompt_tokens_details.then_some(usage.cached_token_count),
)
}
}
/// Mirrors the Python vLLM `PromptTokenUsageInfo` class.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub struct PromptTokenUsageInfo {
pub cached_tokens: Option<u32>,
pub cached_tokens: usize,
}
#[cfg(test)]
mod usage_tests {
use vllm_llm::TokenUsage;
use super::Usage;
#[test]
fn token_usage_hides_prompt_token_details_by_default() {
let usage = Usage::from_token_usage(
TokenUsage {
prompt_token_count: 5,
output_token_count: 2,
cached_token_count: 3,
},
false,
);
assert_eq!(usage.prompt_tokens, 5);
assert_eq!(usage.completion_tokens, Some(2));
assert!(usage.prompt_tokens_details.is_none());
}
#[test]
fn token_usage_includes_prompt_token_details_when_enabled() {
let usage = Usage::from_token_usage(
TokenUsage {
prompt_token_count: 5,
output_token_count: 2,
cached_token_count: 3,
},
true,
);
assert_eq!(
usage.prompt_tokens_details.as_ref().map(|details| details.cached_tokens),
Some(3)
);
}
}
/// OpenAI completions-style logprobs.
+7 -2
View File
@@ -43,6 +43,7 @@ use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
use crate::config::ApiServerOptions;
use crate::state::AppState;
fn request_output(
@@ -787,8 +788,12 @@ async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) {
)
.await;
let app = build_router(Arc::new(
AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat)
.with_request_id_headers(true),
AppState::new(vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], chat).with_api_server_options(
ApiServerOptions {
enable_request_id_headers: true,
..Default::default()
},
),
));
(app, engine_task)
}
+3 -2
View File
@@ -134,11 +134,12 @@ impl Normalizable for DetokenizeRequest {}
#[cfg(test)]
mod tests {
use super::*;
use crate::routes::openai::utils::types::{ChatMessage, MessageContent};
use serde_json::json;
use vllm_chat::ChatTool;
use super::*;
use crate::routes::openai::utils::types::{ChatMessage, MessageContent};
#[test]
fn tokenize_request_converts_openai_tools() {
// The untagged `TokenizeRequest` must resolve a messages+tools body to
+7 -15
View File
@@ -9,6 +9,7 @@ use vllm_chat::ChatLlm;
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use crate::config::ApiServerOptions;
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
@@ -27,10 +28,8 @@ pub struct AppState {
served_model_names: Vec<String>,
/// Shared chat facade used by all requests.
pub chat: ChatLlm,
/// Whether to log a summary line for each completed request.
pub enable_log_requests: bool,
/// Whether to set X-Request-Id on every HTTP response.
pub enable_request_id_headers: bool,
/// HTTP/API-server behavior switches.
pub api_server_options: ApiServerOptions,
/// Runtime server information returned by `/server_info`, when available.
server_info: Option<ServerInfoSnapshot>,
/// SHA-256 hashes of API keys accepted as bearer tokens for guarded routes.
@@ -58,8 +57,7 @@ impl AppState {
Self {
served_model_names,
chat,
enable_log_requests: false,
enable_request_id_headers: false,
api_server_options: ApiServerOptions::default(),
server_info: None,
api_key_hashes: Vec::new(),
server_load: AtomicU64::new(0),
@@ -67,15 +65,9 @@ impl AppState {
}
}
/// Enable per-request completion logging.
pub fn with_log_requests(mut self, enabled: bool) -> Self {
self.enable_log_requests = enabled;
self
}
/// Enable X-Request-Id response headers.
pub fn with_request_id_headers(mut self, enabled: bool) -> Self {
self.enable_request_id_headers = enabled;
/// Set HTTP/API-server behavior switches.
pub fn with_api_server_options(mut self, options: ApiServerOptions) -> Self {
self.api_server_options = options;
self
}