feat(rust): expose profiler control routes in Rust frontend (#46306)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Pranav Thakur
2026-07-02 08:46:07 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent b0b8a286dd
commit 08a8a4af3f
11 changed files with 381 additions and 2 deletions
+47 -1
View File
@@ -13,8 +13,8 @@ use std::time::Duration;
use clap::{Args, Parser, Subcommand};
use educe::Educe;
use serde::Deserialize;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
use thiserror_ext::AsReport as _;
@@ -290,6 +290,15 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub ssl_ciphers: Option<String>,
/// Profiler configuration forwarded by the Python supervisor.
///
/// When set with a non-null `profiler` type, the Rust frontend registers
/// the `/start_profile` and `/stop_profile` routes and forwards calls to
/// the engine via the `"profile"` utility RPC.
#[arg(long, value_parser = parse_json::<ProfilerConfig>, value_name = "JSON")]
#[serde(default)]
pub profiler_config: Option<ProfilerConfig>,
/// Unsupported Python vLLM frontend arguments recognized but not yet
/// implemented in Rust.
#[educe(Debug(ignore))]
@@ -317,6 +326,20 @@ impl SharedRuntimeArgs {
.map_or(DEFAULT_KEEP_ALIVE_TIMEOUT, Duration::from_secs)
}
/// Return the configured profiler mode, when profiling is enabled.
pub fn profiler(&self) -> Option<String> {
self.profiler_config.as_ref().and_then(|c| c.profiler.clone())
}
/// Return the profiler config JSON for managed Python engine forwarding.
pub fn profiler_config_json(&self) -> Option<String> {
self.profiler_config
.as_ref()
.map(serde_json::to_string)
.transpose()
.expect("profiler config serialization should not fail")
}
/// Apply fallback logic for API key configuration from env variables.
fn apply_env_api_key_fallback(&mut self) {
if self.api_key.is_empty()
@@ -345,6 +368,7 @@ impl SharedRuntimeArgs {
let api_server_options = self.api_server_options();
let cors = self.cors_config();
let tls = self.tls_config();
let profiler = self.profiler();
Config {
transport_mode: TransportMode::Bootstrapped {
@@ -377,6 +401,7 @@ impl SharedRuntimeArgs {
grpc_port: self.grpc_port,
shutdown_timeout,
keep_alive_timeout,
profiler,
}
}
@@ -397,6 +422,7 @@ impl SharedRuntimeArgs {
let api_server_options = self.api_server_options();
let cors = self.cors_config();
let tls = self.tls_config();
let profiler = self.profiler();
Config {
transport_mode: TransportMode::HandshakeOwner {
@@ -427,6 +453,7 @@ impl SharedRuntimeArgs {
grpc_port: self.grpc_port,
shutdown_timeout,
keep_alive_timeout,
profiler,
}
}
@@ -477,6 +504,23 @@ fn default_py_bootstrap_parser_selection() -> ParserSelection {
ParserSelection::None
}
/// Minimal profiler configuration parsed from `--profiler-config`.
///
/// Only the `profiler` field is inspected by the Rust frontend to decide
/// whether to register the `/start_profile` and `/stop_profile` routes.
/// All other fields are accepted but ignored — they are consumed by the
/// Python engine layer.
#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
pub struct ProfilerConfig {
/// Profiler backend type (e.g. `"torch"`, `"cuda"`). When `null` or
/// absent, profiling is disabled.
#[serde(default)]
pub profiler: Option<String>,
/// Additional Python profiler config fields consumed by the engine layer.
#[serde(flatten)]
pub extra: serde_json::Map<String, Value>,
}
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
}
@@ -603,11 +647,13 @@ impl ServeArgs {
pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
let reasoning_parser =
effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model);
let profiler_config = self.runtime.profiler_config_json();
self.managed_engine.clone().into_config(
self.runtime.model.clone(),
self.runtime.max_model_len,
self.runtime.max_logprobs,
profiler_config,
reasoning_parser.as_deref(),
self.runtime.language_model_only,
self.runtime.disable_log_stats,
+112
View File
@@ -71,6 +71,7 @@ fn serve_args_forward_python_flags_with_separator() {
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
profiler_config: None,
},
managed_engine: ManagedEngineArgs {
python: "../vllm/.venv/bin/python",
@@ -180,6 +181,40 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() {
.assert_debug_eq(&config.python_args);
}
#[test]
fn serve_args_forward_profiler_config_to_managed_engine() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--profiler-config",
r#"{"profiler":"torch","torch_profiler_dir":"/tmp/profile"}"#,
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
assert_eq!(args.runtime.profiler().as_deref(), Some("torch"));
let config = args.to_managed_engine_config(5555);
let profiler_flag_index = config
.python_args
.iter()
.position(|arg| arg == "--profiler-config")
.expect("profiler config flag");
let profiler_config: serde_json::Value =
serde_json::from_str(&config.python_args[profiler_flag_index + 1])
.expect("profiler config json");
assert_eq!(
profiler_config,
serde_json::json!({
"profiler": "torch",
"torch_profiler_dir": "/tmp/profile",
})
);
}
#[test]
fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
let cli = Cli::try_parse_from([
@@ -732,6 +767,7 @@ fn frontend_args_accept_json() {
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
profiler_config: None,
},
},
),
@@ -1253,6 +1289,7 @@ fn serve_args_accept_handshake_aliases() {
ssl_ca_certs: None,
ssl_cert_reqs: 0,
ssl_ciphers: None,
profiler_config: None,
},
managed_engine: ManagedEngineArgs {
python: "python3",
@@ -1392,6 +1429,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
grpc_port: None,
shutdown_timeout: 0ns,
keep_alive_timeout: 5s,
profiler: None,
}
"#]]
.assert_debug_eq(&Config {
@@ -1475,6 +1513,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
grpc_port: None,
shutdown_timeout: 0ns,
keep_alive_timeout: 5s,
profiler: None,
}
"#]]
.assert_debug_eq(&config);
@@ -1576,6 +1615,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
grpc_port: None,
shutdown_timeout: 0ns,
keep_alive_timeout: 5s,
profiler: None,
}
"#]]
.assert_debug_eq(&config);
@@ -1604,3 +1644,75 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
}
);
}
#[test]
fn frontend_args_json_enables_profiling_when_profiler_config_set() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","profiler_config":{"profiler":"torch","torch_profiler_dir":"/tmp/profile"}}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
assert_eq!(args.runtime.profiler().as_deref(), Some("torch"));
let config = args.into_config();
assert_eq!(config.profiler.as_deref(), Some("torch"));
}
#[test]
fn frontend_args_json_disables_profiling_when_profiler_config_absent() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
assert_eq!(args.runtime.profiler(), None);
let config = args.into_config();
assert_eq!(config.profiler, None);
}
#[test]
fn frontend_args_json_disables_profiling_when_profiler_type_is_null() {
let cli = Cli::try_parse_from([
"vllm-rs",
"frontend",
"--listen-fd",
"3",
"--input-address",
"ipc:///tmp/input.sock",
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","profiler_config":{"profiler":null}}"#,
])
.unwrap();
let Command::Frontend(args) = cli.command else {
panic!("expected frontend args");
};
assert_eq!(args.runtime.profiler(), None);
let config = args.into_config();
assert_eq!(config.profiler, None);
}