[Rust Frontend] Correct --reasoning-parser semantics (#46359)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -29,6 +29,7 @@ tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-chat.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-managed-engine.workspace = true
|
||||
vllm-server.workspace = true
|
||||
|
||||
+27
-3
@@ -19,6 +19,7 @@ use serde_json::Value;
|
||||
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_chat::ReasoningParserFactory;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
@@ -91,7 +92,12 @@ pub enum Command {
|
||||
#[serde(transparent)]
|
||||
pub struct JsonStringList(pub Vec<String>);
|
||||
|
||||
/// Runtime arguments shared by the external-engine and managed-engine paths.
|
||||
/// Runtime arguments shared by both paths of the Rust frontend:
|
||||
///
|
||||
/// - External-engine mode: Python-supervised bootstrap, `vllm serve` -> `vllm-rs frontend`.
|
||||
/// Arguments are deserialized from a single JSON object and defaults follow `serde` attrs.
|
||||
/// - Managed-engine mode: Rust-managed Python engine, `vllm-rs serve`.
|
||||
/// Arguments are parsed from CLI flags and defaults follow `clap` attrs.
|
||||
#[serde_as]
|
||||
#[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
|
||||
#[educe(Debug)]
|
||||
@@ -114,12 +120,12 @@ pub struct SharedRuntimeArgs {
|
||||
/// Select the tool call parser depending on the model that you're using.
|
||||
/// Use `auto` to infer from the model or `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
#[serde(default = "default_py_bootstrap_parser_selection")]
|
||||
pub tool_call_parser: ParserSelection,
|
||||
/// Select the reasoning parser depending on the model that you're using.
|
||||
/// Use `auto` to infer from the model or `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
#[serde(default = "default_py_bootstrap_parser_selection")]
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Select the chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
@@ -402,6 +408,10 @@ fn default_cors_wildcard() -> JsonStringList {
|
||||
JsonStringList(vec!["*".to_string()])
|
||||
}
|
||||
|
||||
fn default_py_bootstrap_parser_selection() -> ParserSelection {
|
||||
ParserSelection::None
|
||||
}
|
||||
|
||||
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
|
||||
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
|
||||
}
|
||||
@@ -526,10 +536,14 @@ impl ServeArgs {
|
||||
/// Build the managed Python-engine spawn configuration with the given
|
||||
/// handshake port.
|
||||
pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
|
||||
let reasoning_parser =
|
||||
effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model);
|
||||
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
self.runtime.max_model_len,
|
||||
self.runtime.max_logprobs,
|
||||
reasoning_parser.as_deref(),
|
||||
self.runtime.language_model_only,
|
||||
self.runtime.disable_log_stats,
|
||||
self.runtime.shutdown_timeout,
|
||||
@@ -538,6 +552,16 @@ impl ServeArgs {
|
||||
}
|
||||
}
|
||||
|
||||
fn effective_engine_reasoning_parser(selection: &ParserSelection, model: &str) -> Option<String> {
|
||||
match selection {
|
||||
ParserSelection::Auto => ReasoningParserFactory::global()
|
||||
.resolve_name_for_model(model)
|
||||
.map(str::to_string),
|
||||
ParserSelection::None => None,
|
||||
ParserSelection::Explicit(name) => Some(name.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Allocate fresh IPC endpoints for one managed frontend instance.
|
||||
fn frontend_ipc_addresses() -> (String, String) {
|
||||
let preferred_base_path = std::env::var_os("VLLM_RPC_BASE_PATH")
|
||||
|
||||
+191
-39
@@ -100,10 +100,13 @@ fn serve_args_auto_forward_python_flags_without_separator() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--quantization", "awq"]
|
||||
);
|
||||
expect![[r#"
|
||||
[
|
||||
"--quantization",
|
||||
"awq",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -114,7 +117,12 @@ fn serve_args_auto_forward_enable_lora_to_python() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]);
|
||||
expect![[r#"
|
||||
[
|
||||
"--enable-lora",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -134,7 +142,15 @@ fn serve_args_forward_shutdown_timeout_to_managed_engine() {
|
||||
assert_eq!(args.runtime.shutdown_timeout, 60);
|
||||
|
||||
let config = args.to_managed_engine_config(5555);
|
||||
assert_eq!(config.python_args, vec!["--shutdown-timeout", "60"]);
|
||||
expect![[r#"
|
||||
[
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--shutdown-timeout",
|
||||
"60",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&config.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -148,7 +164,14 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() {
|
||||
assert!(args.runtime.disable_log_stats);
|
||||
|
||||
let config = args.to_managed_engine_config(5555);
|
||||
assert_eq!(config.python_args, vec!["--disable-log-stats"]);
|
||||
expect![[r#"
|
||||
[
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--disable-log-stats",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&config.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -171,7 +194,106 @@ fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
|
||||
assert_eq!(frontend_config.max_logprobs, Some(-1));
|
||||
|
||||
let engine_config = args.to_managed_engine_config(5555);
|
||||
assert_eq!(engine_config.python_args, vec!["--max-logprobs", "-1"]);
|
||||
expect![[r#"
|
||||
[
|
||||
"--max-logprobs",
|
||||
"-1",
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&engine_config.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_resolve_auto_reasoning_parser_for_managed_engine() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
|
||||
|
||||
let config = args.to_managed_engine_config(5555);
|
||||
expect![[r#"
|
||||
[
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&config.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_explicit_reasoning_parser_to_managed_engine() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Unknown/Model",
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
|
||||
let config = args.to_managed_engine_config(5555);
|
||||
expect![[r#"
|
||||
[
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&config.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_do_not_forward_disabled_reasoning_parser_to_managed_engine() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--reasoning-parser",
|
||||
"none",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
|
||||
let config = args.to_managed_engine_config(5555);
|
||||
assert!(config.python_args.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_reasoning_parser_even_with_passthrough_reasoning_parser() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--",
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
|
||||
let config = args.to_managed_engine_config(5555);
|
||||
expect![[r#"
|
||||
[
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&config.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -181,10 +303,13 @@ fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--tensor-parallel-size", "2"]
|
||||
);
|
||||
expect![[r#"
|
||||
[
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -429,8 +554,8 @@ fn frontend_args_accept_json() {
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_model_len: None,
|
||||
@@ -490,8 +615,8 @@ fn frontend_args_json_applies_defaults() {
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
|
||||
assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
|
||||
assert_eq!(args.runtime.tool_call_parser, ParserSelection::None);
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
||||
assert_eq!(args.runtime.max_model_len, None);
|
||||
assert_eq!(args.runtime.max_logprobs, None);
|
||||
@@ -784,10 +909,15 @@ fn serve_args_keep_python_passthrough_flags_after_separator() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--tensor-parallel-size", "2", "--dtype", "float16"]
|
||||
);
|
||||
expect![[r#"
|
||||
[
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--dtype",
|
||||
"float16",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -809,10 +939,15 @@ fn serve_args_keep_python_multi_char_alias_after_separator() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["-tp", "2", "--dtype", "float16"]
|
||||
);
|
||||
expect![[r#"
|
||||
[
|
||||
"-tp",
|
||||
"2",
|
||||
"--dtype",
|
||||
"float16",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -830,10 +965,13 @@ fn serve_args_keep_frontend_arg_after_separator() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--uds", "/tmp/vllm.sock"]
|
||||
);
|
||||
expect![[r#"
|
||||
[
|
||||
"--uds",
|
||||
"/tmp/vllm.sock",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -853,10 +991,15 @@ fn serve_args_keep_python_multi_char_engine_aliases_after_separator() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["-dpr", "1", "-dpl", "2"]
|
||||
);
|
||||
expect![[r#"
|
||||
[
|
||||
"-dpr",
|
||||
"1",
|
||||
"-dpl",
|
||||
"2",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -866,7 +1009,13 @@ fn serve_args_auto_forward_unknown_flags_without_separator() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.managed_engine.python_args, vec!["--foo", "bar"]);
|
||||
expect![[r#"
|
||||
[
|
||||
"--foo",
|
||||
"bar",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -883,10 +1032,13 @@ fn serve_args_auto_forward_negative_value_without_separator() {
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.managed_engine.python_args,
|
||||
vec!["--num-gpu-blocks-override", "-1"]
|
||||
);
|
||||
expect![[r#"
|
||||
[
|
||||
"--num-gpu-blocks-override",
|
||||
"-1",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&args.managed_engine.python_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1237,8 +1389,8 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
listener_mode: InheritedFd {
|
||||
fd: 3,
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
|
||||
Reference in New Issue
Block a user