[Rust Frontend] Populate cached_token_count in responses (#44887)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
+24
-6
@@ -23,8 +23,8 @@ use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
use vllm_server::{
|
||||
ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode, ParserSelection,
|
||||
RendererSelection,
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, HttpListenerMode,
|
||||
ParserSelection, RendererSelection,
|
||||
};
|
||||
|
||||
use crate::cli::unsupported::UnsupportedArgs;
|
||||
@@ -171,6 +171,16 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub enable_log_requests: bool,
|
||||
|
||||
/// Include prompt_tokens_details in usage when cached prompt tokens are
|
||||
/// present.
|
||||
#[arg(
|
||||
long,
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
#[serde(default)]
|
||||
pub enable_prompt_tokens_details: bool,
|
||||
|
||||
/// If specified, API server will add X-Request-Id header to responses.
|
||||
#[arg(
|
||||
long,
|
||||
@@ -248,6 +258,7 @@ impl SharedRuntimeArgs {
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
let api_server_options = self.api_server_options();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::Bootstrapped {
|
||||
@@ -270,8 +281,7 @@ impl SharedRuntimeArgs {
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
enable_log_requests: self.enable_log_requests,
|
||||
enable_request_id_headers: self.enable_request_id_headers,
|
||||
api_server_options,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
@@ -292,6 +302,7 @@ impl SharedRuntimeArgs {
|
||||
) -> Config {
|
||||
let ready_timeout = self.ready_timeout();
|
||||
let shutdown_timeout = self.shutdown_timeout();
|
||||
let api_server_options = self.api_server_options();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
@@ -313,14 +324,21 @@ impl SharedRuntimeArgs {
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
enable_log_requests: self.enable_log_requests,
|
||||
enable_request_id_headers: self.enable_request_id_headers,
|
||||
api_server_options,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
fn api_server_options(&self) -> ApiServerOptions {
|
||||
ApiServerOptions {
|
||||
enable_log_requests: self.enable_log_requests,
|
||||
enable_prompt_tokens_details: self.enable_prompt_tokens_details,
|
||||
enable_request_id_headers: self.enable_request_id_headers,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_engine_ready_timeout_secs() -> u64 {
|
||||
|
||||
@@ -44,6 +44,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
@@ -143,7 +144,24 @@ fn serve_passes_enable_request_id_headers_into_config() {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
assert!(config.enable_request_id_headers);
|
||||
assert!(config.api_server_options.enable_request_id_headers);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_passes_enable_prompt_tokens_details_into_config() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--enable-prompt-tokens-details",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
assert!(config.api_server_options.enable_prompt_tokens_details);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -166,7 +184,7 @@ fn frontend_args_json_passes_enable_request_id_headers_into_config() {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
let config = args.into_config();
|
||||
assert!(config.enable_request_id_headers);
|
||||
assert!(config.api_server_options.enable_request_id_headers);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -342,6 +360,7 @@ fn frontend_args_accept_json() {
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
@@ -456,7 +475,7 @@ fn frontend_args_json_ignores_unknown_fields() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_accepts_noop_fields() {
|
||||
fn frontend_args_json_sets_prompt_tokens_details_flag() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
@@ -467,7 +486,7 @@ fn frontend_args_json_accepts_noop_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","api_server_count":2}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","api_server_count":2,"enable_prompt_tokens_details":true}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
@@ -475,6 +494,7 @@ fn frontend_args_json_accepts_noop_fields() {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
assert!(args.runtime.enable_prompt_tokens_details);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -744,6 +764,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
@@ -862,8 +883,11 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_request_id_headers: false,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
},
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
@@ -927,8 +951,11 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_request_id_headers: false,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
},
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
@@ -1007,8 +1034,11 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
enable_log_requests: false,
|
||||
enable_request_id_headers: false,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
},
|
||||
api_keys: [],
|
||||
disable_log_stats: false,
|
||||
grpc_port: None,
|
||||
|
||||
@@ -444,15 +444,6 @@ pub struct ServerUnsupportedArgs {
|
||||
#[arg(long)]
|
||||
pub max_log_len: Option<Unsupported>,
|
||||
|
||||
/// If set to True, enable prompt_tokens_details in usage.
|
||||
#[arg(
|
||||
long,
|
||||
visible_alias = "no-enable-prompt-tokens-details",
|
||||
default_missing_value = "true",
|
||||
num_args = 0..=1
|
||||
)]
|
||||
pub enable_prompt_tokens_details: Option<Unsupported>,
|
||||
|
||||
/// If set to True, enable tracking server_load_metrics in the app state.
|
||||
#[arg(
|
||||
long,
|
||||
|
||||
Reference in New Issue
Block a user