[Rust Frontend] Populate cached_token_count in responses (#44887)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-10 20:50:05 -07:00
committed by GitHub
parent 85a0ffae42
commit 5d5591d99b
34 changed files with 556 additions and 262 deletions
+24 -6
View File
@@ -23,8 +23,8 @@ use vllm_engine_core_client::TransportMode;
use vllm_managed_engine::ManagedEngineConfig;
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
use vllm_server::{
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
RendererSelection,
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode,
ParserSelection, RendererSelection,
};
use crate::cli::unsupported::UnsupportedArgs;
@@ -171,6 +171,16 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub enable_log_requests: bool,
/// Include prompt_tokens_details in usage when cached prompt tokens are
/// present.
#[arg(
long,
default_missing_value = "true",
num_args = 0..=1
)]
#[serde(default)]
pub enable_prompt_tokens_details: bool,
/// If specified, API server will add X-Request-Id header to responses.
#[arg(
long,
@@ -248,6 +258,7 @@ impl SharedRuntimeArgs {
) -> Config {
let ready_timeout = self.ready_timeout();
let shutdown_timeout = self.shutdown_timeout();
let api_server_options = self.api_server_options();
Config {
transport_mode: TransportMode::Bootstrapped {
@@ -270,8 +281,7 @@ impl SharedRuntimeArgs {
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
chat_template_content_format: self.chat_template_content_format,
enable_log_requests: self.enable_log_requests,
enable_request_id_headers: self.enable_request_id_headers,
api_server_options,
api_keys: self.api_key,
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
@@ -292,6 +302,7 @@ impl SharedRuntimeArgs {
) -> Config {
let ready_timeout = self.ready_timeout();
let shutdown_timeout = self.shutdown_timeout();
let api_server_options = self.api_server_options();
Config {
transport_mode: TransportMode::HandshakeOwner {
@@ -313,14 +324,21 @@ impl SharedRuntimeArgs {
chat_template: self.chat_template,
default_chat_template_kwargs: self.default_chat_template_kwargs,
chat_template_content_format: self.chat_template_content_format,
enable_log_requests: self.enable_log_requests,
enable_request_id_headers: self.enable_request_id_headers,
api_server_options,
api_keys: self.api_key,
disable_log_stats: self.disable_log_stats,
grpc_port: self.grpc_port,
shutdown_timeout,
}
}
fn api_server_options(&self) -> ApiServerOptions {
ApiServerOptions {
enable_log_requests: self.enable_log_requests,
enable_prompt_tokens_details: self.enable_prompt_tokens_details,
enable_request_id_headers: self.enable_request_id_headers,
}
}
}
fn default_engine_ready_timeout_secs() -> u64 {
+40 -10
View File
@@ -44,6 +44,7 @@ fn serve_args_forward_python_flags_with_separator() {
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
disable_log_stats: false,
served_model_name: [],
@@ -143,7 +144,24 @@ fn serve_passes_enable_request_id_headers_into_config() {
panic!("expected serve args");
};
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
assert!(config.enable_request_id_headers);
assert!(config.api_server_options.enable_request_id_headers);
}
#[test]
fn serve_passes_enable_prompt_tokens_details_into_config() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--enable-prompt-tokens-details",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
assert!(config.api_server_options.enable_prompt_tokens_details);
}
#[test]
@@ -166,7 +184,7 @@ fn frontend_args_json_passes_enable_request_id_headers_into_config() {
panic!("expected frontend args");
};
let config = args.into_config();
assert!(config.enable_request_id_headers);
assert!(config.api_server_options.enable_request_id_headers);
}
#[test]
@@ -342,6 +360,7 @@ fn frontend_args_accept_json() {
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
disable_log_stats: false,
served_model_name: [],
@@ -456,7 +475,7 @@ fn frontend_args_json_ignores_unknown_fields() {
}
#[test]
fn frontend_args_json_accepts_noop_fields() {
fn frontend_args_json_sets_prompt_tokens_details_flag() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
@@ -467,7 +486,7 @@ fn frontend_args_json_accepts_noop_fields() {
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","api_server_count":2}"#,
r#"{"model_tag":"Qwen/Qwen3-0.6B","api_server_count":2,"enable_prompt_tokens_details":true}"#,
])
.unwrap();
@@ -475,6 +494,7 @@ fn frontend_args_json_accepts_noop_fields() {
panic!("expected frontend args");
};
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
assert!(args.runtime.enable_prompt_tokens_details);
}
#[test]
@@ -744,6 +764,7 @@ fn serve_args_accept_handshake_aliases() {
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
disable_log_stats: false,
served_model_name: [],
@@ -862,8 +883,11 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
enable_request_id_headers: false,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
},
api_keys: [],
disable_log_stats: false,
grpc_port: None,
@@ -927,8 +951,11 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
enable_request_id_headers: false,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
},
api_keys: [],
disable_log_stats: false,
grpc_port: None,
@@ -1007,8 +1034,11 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
chat_template: None,
default_chat_template_kwargs: None,
chat_template_content_format: Auto,
enable_log_requests: false,
enable_request_id_headers: false,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
},
api_keys: [],
disable_log_stats: false,
grpc_port: None,
-9
View File
@@ -444,15 +444,6 @@ pub struct ServerUnsupportedArgs {
#[arg(long)]
pub max_log_len: Option<Unsupported>,
/// If set to True, enable prompt_tokens_details in usage.
#[arg(
long,
visible_alias = "no-enable-prompt-tokens-details",
default_missing_value = "true",
num_args = 0..=1
)]
pub enable_prompt_tokens_details: Option<Unsupported>,
/// If set to True, enable tracking server_load_metrics in the app state.
#[arg(
long,