From 25bc3be49cc7ce8e7b185ca7da11b2fc9778832c Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 23 Jun 2026 15:38:39 +0800 Subject: [PATCH] [Rust Frontend] Correct `--reasoning-parser` semantics (#46359) Signed-off-by: Bugen Zhao --- .buildkite/test_areas/rust_frontend.yaml | 2 +- rust/Cargo.lock | 1 + rust/src/cmd/Cargo.toml | 1 + rust/src/cmd/src/cli.rs | 30 ++- rust/src/cmd/src/cli/tests.rs | 230 +++++++++++++++++++---- rust/src/managed-engine/src/cli.rs | 10 + rust/src/text/src/lower.rs | 1 + 7 files changed, 232 insertions(+), 43 deletions(-) diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index f9abac2004..5ea0f7ef77 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -26,7 +26,7 @@ steps: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)" - - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py + - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex" # - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid" # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 60aa6c1241..e580589b1e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5750,6 +5750,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", + "vllm-chat", "vllm-engine-core-client", "vllm-managed-engine", "vllm-server", diff --git a/rust/src/cmd/Cargo.toml b/rust/src/cmd/Cargo.toml index b0caa65b4e..030d4c6d11 100644 --- a/rust/src/cmd/Cargo.toml +++ b/rust/src/cmd/Cargo.toml @@ -29,6 +29,7 @@ tokio-util.workspace = true tracing.workspace = true tracing-subscriber.workspace = true uuid.workspace = true +vllm-chat.workspace = true vllm-engine-core-client.workspace = true vllm-managed-engine.workspace = true vllm-server.workspace = true diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 003d96fa92..a3fa9b0550 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -19,6 +19,7 @@ use serde_json::Value; use serde_with::{DefaultOnNull, OneOrMany, serde_as}; use thiserror_ext::AsReport as _; use uuid::Uuid; +use vllm_chat::ReasoningParserFactory; use vllm_engine_core_client::TransportMode; use vllm_managed_engine::ManagedEngineConfig; use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args}; @@ -91,7 +92,12 @@ pub enum Command { #[serde(transparent)] pub struct JsonStringList(pub Vec); -/// Runtime arguments shared by the external-engine and managed-engine paths. +/// Runtime arguments shared by both paths of the Rust frontend: +/// +/// - External-engine mode: Python-supervised bootstrap, `vllm serve` -> `vllm-rs frontend`. +/// Arguments are deserialized from a single JSON object and defaults follow `serde` attrs. +/// - Managed-engine mode: Rust-managed Python engine, `vllm-rs serve`. +/// Arguments are parsed from CLI flags and defaults follow `clap` attrs. #[serde_as] #[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)] #[educe(Debug)] @@ -114,12 +120,12 @@ pub struct SharedRuntimeArgs { /// Select the tool call parser depending on the model that you're using. /// Use `auto` to infer from the model or `none` to disable parsing. #[arg(long, default_value_t)] - #[serde(default)] + #[serde(default = "default_py_bootstrap_parser_selection")] pub tool_call_parser: ParserSelection, /// Select the reasoning parser depending on the model that you're using. /// Use `auto` to infer from the model or `none` to disable parsing. #[arg(long, default_value_t)] - #[serde(default)] + #[serde(default = "default_py_bootstrap_parser_selection")] pub reasoning_parser: ParserSelection, /// Select the chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] @@ -402,6 +408,10 @@ fn default_cors_wildcard() -> JsonStringList { JsonStringList(vec!["*".to_string()]) } +fn default_py_bootstrap_parser_selection() -> ParserSelection { + ParserSelection::None +} + fn parse_json(value: &str) -> Result { serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report())) } @@ -526,10 +536,14 @@ impl ServeArgs { /// Build the managed Python-engine spawn configuration with the given /// handshake port. pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig { + let reasoning_parser = + effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model); + self.managed_engine.clone().into_config( self.runtime.model.clone(), self.runtime.max_model_len, self.runtime.max_logprobs, + reasoning_parser.as_deref(), self.runtime.language_model_only, self.runtime.disable_log_stats, self.runtime.shutdown_timeout, @@ -538,6 +552,16 @@ impl ServeArgs { } } +fn effective_engine_reasoning_parser(selection: &ParserSelection, model: &str) -> Option { + match selection { + ParserSelection::Auto => ReasoningParserFactory::global() + .resolve_name_for_model(model) + .map(str::to_string), + ParserSelection::None => None, + ParserSelection::Explicit(name) => Some(name.clone()), + } +} + /// Allocate fresh IPC endpoints for one managed frontend instance. fn frontend_ipc_addresses() -> (String, String) { let preferred_base_path = std::env::var_os("VLLM_RPC_BASE_PATH") diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index c57c23e017..345cc9f60d 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -100,10 +100,13 @@ fn serve_args_auto_forward_python_flags_without_separator() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!( - args.managed_engine.python_args, - vec!["--quantization", "awq"] - ); + expect![[r#" + [ + "--quantization", + "awq", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -114,7 +117,12 @@ fn serve_args_auto_forward_enable_lora_to_python() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]); + expect![[r#" + [ + "--enable-lora", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -134,7 +142,15 @@ fn serve_args_forward_shutdown_timeout_to_managed_engine() { assert_eq!(args.runtime.shutdown_timeout, 60); let config = args.to_managed_engine_config(5555); - assert_eq!(config.python_args, vec!["--shutdown-timeout", "60"]); + expect![[r#" + [ + "--reasoning-parser", + "qwen3", + "--shutdown-timeout", + "60", + ] + "#]] + .assert_debug_eq(&config.python_args); } #[test] @@ -148,7 +164,14 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() { assert!(args.runtime.disable_log_stats); let config = args.to_managed_engine_config(5555); - assert_eq!(config.python_args, vec!["--disable-log-stats"]); + expect![[r#" + [ + "--reasoning-parser", + "qwen3", + "--disable-log-stats", + ] + "#]] + .assert_debug_eq(&config.python_args); } #[test] @@ -171,7 +194,106 @@ fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() { assert_eq!(frontend_config.max_logprobs, Some(-1)); let engine_config = args.to_managed_engine_config(5555); - assert_eq!(engine_config.python_args, vec!["--max-logprobs", "-1"]); + expect![[r#" + [ + "--max-logprobs", + "-1", + "--reasoning-parser", + "qwen3", + ] + "#]] + .assert_debug_eq(&engine_config.python_args); +} + +#[test] +fn serve_args_resolve_auto_reasoning_parser_for_managed_engine() { + let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto); + + let config = args.to_managed_engine_config(5555); + expect![[r#" + [ + "--reasoning-parser", + "qwen3", + ] + "#]] + .assert_debug_eq(&config.python_args); +} + +#[test] +fn serve_args_forward_explicit_reasoning_parser_to_managed_engine() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Unknown/Model", + "--reasoning-parser", + "deepseek_r1", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + + let config = args.to_managed_engine_config(5555); + expect![[r#" + [ + "--reasoning-parser", + "deepseek_r1", + ] + "#]] + .assert_debug_eq(&config.python_args); +} + +#[test] +fn serve_args_do_not_forward_disabled_reasoning_parser_to_managed_engine() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--reasoning-parser", + "none", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + + let config = args.to_managed_engine_config(5555); + assert!(config.python_args.is_empty()); +} + +#[test] +fn serve_args_forward_reasoning_parser_even_with_passthrough_reasoning_parser() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--", + "--reasoning-parser", + "deepseek_r1", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + + let config = args.to_managed_engine_config(5555); + expect![[r#" + [ + "--reasoning-parser", + "deepseek_r1", + "--reasoning-parser", + "qwen3", + ] + "#]] + .assert_debug_eq(&config.python_args); } #[test] @@ -181,10 +303,13 @@ fn serve_args_auto_forward_python_multi_char_alias_without_separator() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!( - args.managed_engine.python_args, - vec!["--tensor-parallel-size", "2"] - ); + expect![[r#" + [ + "--tensor-parallel-size", + "2", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -429,8 +554,8 @@ fn frontend_args_accept_json() { runtime: SharedRuntimeArgs { model: "Qwen/Qwen3-0.6B", engine_ready_timeout_secs: 600, - tool_call_parser: Auto, - reasoning_parser: Auto, + tool_call_parser: None, + reasoning_parser: None, renderer: Auto, language_model_only: false, max_model_len: None, @@ -490,8 +615,8 @@ fn frontend_args_json_applies_defaults() { }; assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B"); assert_eq!(args.runtime.engine_ready_timeout_secs, 600); - assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto); - assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto); + assert_eq!(args.runtime.tool_call_parser, ParserSelection::None); + assert_eq!(args.runtime.reasoning_parser, ParserSelection::None); assert_eq!(args.runtime.renderer, RendererSelection::Auto); assert_eq!(args.runtime.max_model_len, None); assert_eq!(args.runtime.max_logprobs, None); @@ -784,10 +909,15 @@ fn serve_args_keep_python_passthrough_flags_after_separator() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!( - args.managed_engine.python_args, - vec!["--tensor-parallel-size", "2", "--dtype", "float16"] - ); + expect![[r#" + [ + "--tensor-parallel-size", + "2", + "--dtype", + "float16", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -809,10 +939,15 @@ fn serve_args_keep_python_multi_char_alias_after_separator() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!( - args.managed_engine.python_args, - vec!["-tp", "2", "--dtype", "float16"] - ); + expect![[r#" + [ + "-tp", + "2", + "--dtype", + "float16", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -830,10 +965,13 @@ fn serve_args_keep_frontend_arg_after_separator() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!( - args.managed_engine.python_args, - vec!["--uds", "/tmp/vllm.sock"] - ); + expect![[r#" + [ + "--uds", + "/tmp/vllm.sock", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -853,10 +991,15 @@ fn serve_args_keep_python_multi_char_engine_aliases_after_separator() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!( - args.managed_engine.python_args, - vec!["-dpr", "1", "-dpl", "2"] - ); + expect![[r#" + [ + "-dpr", + "1", + "-dpl", + "2", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -866,7 +1009,13 @@ fn serve_args_auto_forward_unknown_flags_without_separator() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!(args.managed_engine.python_args, vec!["--foo", "bar"]); + expect![[r#" + [ + "--foo", + "bar", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -883,10 +1032,13 @@ fn serve_args_auto_forward_negative_value_without_separator() { let Command::Serve(args) = cli.command else { panic!("expected serve args"); }; - assert_eq!( - args.managed_engine.python_args, - vec!["--num-gpu-blocks-override", "-1"] - ); + expect![[r#" + [ + "--num-gpu-blocks-override", + "-1", + ] + "#]] + .assert_debug_eq(&args.managed_engine.python_args); } #[test] @@ -1237,8 +1389,8 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present listener_mode: InheritedFd { fd: 3, }, - tool_call_parser: Auto, - reasoning_parser: Auto, + tool_call_parser: None, + reasoning_parser: None, renderer: Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/managed-engine/src/cli.rs b/rust/src/managed-engine/src/cli.rs index bbd8e70f90..36789fc7b4 100644 --- a/rust/src/managed-engine/src/cli.rs +++ b/rust/src/managed-engine/src/cli.rs @@ -43,6 +43,11 @@ pub struct ManagedEngineArgs { /// Arguments after an explicit `--` are forwarded verbatim. Before `--`, /// `vllm-rs serve` automatically keeps recognized frontend options on /// the Rust side and forwards everything else to Python. + /// + /// The explicit `--` passthrough is a last-resort escape hatch. Rust does + /// not interpret, validate, or de-duplicate those arguments against + /// managed-engine arguments that it appends later; if the same Python flag + /// appears more than once, Python argparse owns the final result. #[arg( last = true, allow_hyphen_values = true, @@ -72,6 +77,7 @@ impl ManagedEngineArgs { model: String, max_model_len: Option, max_logprobs: Option, + reasoning_parser: Option<&str>, language_model_only: bool, disable_log_stats: bool, shutdown_timeout: u64, @@ -87,6 +93,10 @@ impl ManagedEngineArgs { python_args.push("--max-logprobs".to_string()); python_args.push(max_logprobs.to_string()); } + if let Some(reasoning_parser) = reasoning_parser { + python_args.push("--reasoning-parser".to_string()); + python_args.push(reasoning_parser.to_string()); + } if language_model_only { python_args.push("--language-model-only".to_string()); } diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index c5f1e67568..c87be82a4c 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -164,6 +164,7 @@ pub fn lower_sampling_params( logit_bias, allowed_token_ids, bad_words_token_ids: tokenize_bad_words(bad_words.as_deref(), tokenizer)?, + // TODO: Validate structured-output schemas and regexes before submitting requests to engine-core. structured_outputs, logprob_token_ids, skip_reading_prefix_cache,