feat(rust): expose profiler control routes in Rust frontend (#46306)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
b0b8a286dd
commit
08a8a4af3f
+47
-1
@@ -13,8 +13,8 @@ use std::time::Duration;
|
||||
|
||||
use clap::{Args, Parser, Subcommand};
|
||||
use educe::Educe;
|
||||
use serde::Deserialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
||||
use thiserror_ext::AsReport as _;
|
||||
@@ -290,6 +290,15 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub ssl_ciphers: Option<String>,
|
||||
|
||||
/// Profiler configuration forwarded by the Python supervisor.
|
||||
///
|
||||
/// When set with a non-null `profiler` type, the Rust frontend registers
|
||||
/// the `/start_profile` and `/stop_profile` routes and forwards calls to
|
||||
/// the engine via the `"profile"` utility RPC.
|
||||
#[arg(long, value_parser = parse_json::<ProfilerConfig>, value_name = "JSON")]
|
||||
#[serde(default)]
|
||||
pub profiler_config: Option<ProfilerConfig>,
|
||||
|
||||
/// Unsupported Python vLLM frontend arguments recognized but not yet
|
||||
/// implemented in Rust.
|
||||
#[educe(Debug(ignore))]
|
||||
@@ -317,6 +326,20 @@ impl SharedRuntimeArgs {
|
||||
.map_or(DEFAULT_KEEP_ALIVE_TIMEOUT, Duration::from_secs)
|
||||
}
|
||||
|
||||
/// Return the configured profiler mode, when profiling is enabled.
|
||||
pub fn profiler(&self) -> Option<String> {
|
||||
self.profiler_config.as_ref().and_then(|c| c.profiler.clone())
|
||||
}
|
||||
|
||||
/// Return the profiler config JSON for managed Python engine forwarding.
|
||||
pub fn profiler_config_json(&self) -> Option<String> {
|
||||
self.profiler_config
|
||||
.as_ref()
|
||||
.map(serde_json::to_string)
|
||||
.transpose()
|
||||
.expect("profiler config serialization should not fail")
|
||||
}
|
||||
|
||||
/// Apply fallback logic for API key configuration from env variables.
|
||||
fn apply_env_api_key_fallback(&mut self) {
|
||||
if self.api_key.is_empty()
|
||||
@@ -345,6 +368,7 @@ impl SharedRuntimeArgs {
|
||||
let api_server_options = self.api_server_options();
|
||||
let cors = self.cors_config();
|
||||
let tls = self.tls_config();
|
||||
let profiler = self.profiler();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::Bootstrapped {
|
||||
@@ -377,6 +401,7 @@ impl SharedRuntimeArgs {
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
keep_alive_timeout,
|
||||
profiler,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -397,6 +422,7 @@ impl SharedRuntimeArgs {
|
||||
let api_server_options = self.api_server_options();
|
||||
let cors = self.cors_config();
|
||||
let tls = self.tls_config();
|
||||
let profiler = self.profiler();
|
||||
|
||||
Config {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
@@ -427,6 +453,7 @@ impl SharedRuntimeArgs {
|
||||
grpc_port: self.grpc_port,
|
||||
shutdown_timeout,
|
||||
keep_alive_timeout,
|
||||
profiler,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -477,6 +504,23 @@ fn default_py_bootstrap_parser_selection() -> ParserSelection {
|
||||
ParserSelection::None
|
||||
}
|
||||
|
||||
/// Minimal profiler configuration parsed from `--profiler-config`.
|
||||
///
|
||||
/// Only the `profiler` field is inspected by the Rust frontend to decide
|
||||
/// whether to register the `/start_profile` and `/stop_profile` routes.
|
||||
/// All other fields are accepted but ignored — they are consumed by the
|
||||
/// Python engine layer.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
|
||||
pub struct ProfilerConfig {
|
||||
/// Profiler backend type (e.g. `"torch"`, `"cuda"`). When `null` or
|
||||
/// absent, profiling is disabled.
|
||||
#[serde(default)]
|
||||
pub profiler: Option<String>,
|
||||
/// Additional Python profiler config fields consumed by the engine layer.
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, Value>,
|
||||
}
|
||||
|
||||
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
|
||||
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
|
||||
}
|
||||
@@ -603,11 +647,13 @@ impl ServeArgs {
|
||||
pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
|
||||
let reasoning_parser =
|
||||
effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model);
|
||||
let profiler_config = self.runtime.profiler_config_json();
|
||||
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
self.runtime.max_model_len,
|
||||
self.runtime.max_logprobs,
|
||||
profiler_config,
|
||||
reasoning_parser.as_deref(),
|
||||
self.runtime.language_model_only,
|
||||
self.runtime.disable_log_stats,
|
||||
|
||||
@@ -71,6 +71,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
ssl_ca_certs: None,
|
||||
ssl_cert_reqs: 0,
|
||||
ssl_ciphers: None,
|
||||
profiler_config: None,
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "../vllm/.venv/bin/python",
|
||||
@@ -180,6 +181,40 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() {
|
||||
.assert_debug_eq(&config.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_profiler_config_to_managed_engine() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--profiler-config",
|
||||
r#"{"profiler":"torch","torch_profiler_dir":"/tmp/profile"}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.runtime.profiler().as_deref(), Some("torch"));
|
||||
|
||||
let config = args.to_managed_engine_config(5555);
|
||||
let profiler_flag_index = config
|
||||
.python_args
|
||||
.iter()
|
||||
.position(|arg| arg == "--profiler-config")
|
||||
.expect("profiler config flag");
|
||||
let profiler_config: serde_json::Value =
|
||||
serde_json::from_str(&config.python_args[profiler_flag_index + 1])
|
||||
.expect("profiler config json");
|
||||
assert_eq!(
|
||||
profiler_config,
|
||||
serde_json::json!({
|
||||
"profiler": "torch",
|
||||
"torch_profiler_dir": "/tmp/profile",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
|
||||
let cli = Cli::try_parse_from([
|
||||
@@ -732,6 +767,7 @@ fn frontend_args_accept_json() {
|
||||
ssl_ca_certs: None,
|
||||
ssl_cert_reqs: 0,
|
||||
ssl_ciphers: None,
|
||||
profiler_config: None,
|
||||
},
|
||||
},
|
||||
),
|
||||
@@ -1253,6 +1289,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
ssl_ca_certs: None,
|
||||
ssl_cert_reqs: 0,
|
||||
ssl_ciphers: None,
|
||||
profiler_config: None,
|
||||
},
|
||||
managed_engine: ManagedEngineArgs {
|
||||
python: "python3",
|
||||
@@ -1392,6 +1429,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
keep_alive_timeout: 5s,
|
||||
profiler: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&Config {
|
||||
@@ -1475,6 +1513,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
keep_alive_timeout: 5s,
|
||||
profiler: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&config);
|
||||
@@ -1576,6 +1615,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0ns,
|
||||
keep_alive_timeout: 5s,
|
||||
profiler: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&config);
|
||||
@@ -1604,3 +1644,75 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_enables_profiling_when_profiler_config_set() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","profiler_config":{"profiler":"torch","torch_profiler_dir":"/tmp/profile"}}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.profiler().as_deref(), Some("torch"));
|
||||
let config = args.into_config();
|
||||
assert_eq!(config.profiler.as_deref(), Some("torch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_disables_profiling_when_profiler_config_absent() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.profiler(), None);
|
||||
let config = args.into_config();
|
||||
assert_eq!(config.profiler, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_disables_profiling_when_profiler_type_is_null() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","profiler_config":{"profiler":null}}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.profiler(), None);
|
||||
let config = args.into_config();
|
||||
assert_eq!(config.profiler, None);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user