[Rust Frontend] Correct --reasoning-parser semantics (#46359)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -26,7 +26,7 @@ steps:
|
|||||||
- export VLLM_USE_RUST_FRONTEND=1
|
- export VLLM_USE_RUST_FRONTEND=1
|
||||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
- pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
|
- pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
|
||||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py
|
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex"
|
||||||
# - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"
|
# - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"
|
||||||
|
|
||||||
# - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
|
# - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
|
||||||
|
|||||||
Generated
+1
@@ -5750,6 +5750,7 @@ dependencies = [
|
|||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
"uuid",
|
"uuid",
|
||||||
|
"vllm-chat",
|
||||||
"vllm-engine-core-client",
|
"vllm-engine-core-client",
|
||||||
"vllm-managed-engine",
|
"vllm-managed-engine",
|
||||||
"vllm-server",
|
"vllm-server",
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ tokio-util.workspace = true
|
|||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
tracing-subscriber.workspace = true
|
tracing-subscriber.workspace = true
|
||||||
uuid.workspace = true
|
uuid.workspace = true
|
||||||
|
vllm-chat.workspace = true
|
||||||
vllm-engine-core-client.workspace = true
|
vllm-engine-core-client.workspace = true
|
||||||
vllm-managed-engine.workspace = true
|
vllm-managed-engine.workspace = true
|
||||||
vllm-server.workspace = true
|
vllm-server.workspace = true
|
||||||
|
|||||||
+27
-3
@@ -19,6 +19,7 @@ use serde_json::Value;
|
|||||||
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
||||||
use thiserror_ext::AsReport as _;
|
use thiserror_ext::AsReport as _;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
use vllm_chat::ReasoningParserFactory;
|
||||||
use vllm_engine_core_client::TransportMode;
|
use vllm_engine_core_client::TransportMode;
|
||||||
use vllm_managed_engine::ManagedEngineConfig;
|
use vllm_managed_engine::ManagedEngineConfig;
|
||||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||||
@@ -91,7 +92,12 @@ pub enum Command {
|
|||||||
#[serde(transparent)]
|
#[serde(transparent)]
|
||||||
pub struct JsonStringList(pub Vec<String>);
|
pub struct JsonStringList(pub Vec<String>);
|
||||||
|
|
||||||
/// Runtime arguments shared by the external-engine and managed-engine paths.
|
/// Runtime arguments shared by both paths of the Rust frontend:
|
||||||
|
///
|
||||||
|
/// - External-engine mode: Python-supervised bootstrap, `vllm serve` -> `vllm-rs frontend`.
|
||||||
|
/// Arguments are deserialized from a single JSON object and defaults follow `serde` attrs.
|
||||||
|
/// - Managed-engine mode: Rust-managed Python engine, `vllm-rs serve`.
|
||||||
|
/// Arguments are parsed from CLI flags and defaults follow `clap` attrs.
|
||||||
#[serde_as]
|
#[serde_as]
|
||||||
#[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
|
#[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
|
||||||
#[educe(Debug)]
|
#[educe(Debug)]
|
||||||
@@ -114,12 +120,12 @@ pub struct SharedRuntimeArgs {
|
|||||||
/// Select the tool call parser depending on the model that you're using.
|
/// Select the tool call parser depending on the model that you're using.
|
||||||
/// Use `auto` to infer from the model or `none` to disable parsing.
|
/// Use `auto` to infer from the model or `none` to disable parsing.
|
||||||
#[arg(long, default_value_t)]
|
#[arg(long, default_value_t)]
|
||||||
#[serde(default)]
|
#[serde(default = "default_py_bootstrap_parser_selection")]
|
||||||
pub tool_call_parser: ParserSelection,
|
pub tool_call_parser: ParserSelection,
|
||||||
/// Select the reasoning parser depending on the model that you're using.
|
/// Select the reasoning parser depending on the model that you're using.
|
||||||
/// Use `auto` to infer from the model or `none` to disable parsing.
|
/// Use `auto` to infer from the model or `none` to disable parsing.
|
||||||
#[arg(long, default_value_t)]
|
#[arg(long, default_value_t)]
|
||||||
#[serde(default)]
|
#[serde(default = "default_py_bootstrap_parser_selection")]
|
||||||
pub reasoning_parser: ParserSelection,
|
pub reasoning_parser: ParserSelection,
|
||||||
/// Select the chat renderer implementation.
|
/// Select the chat renderer implementation.
|
||||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||||
@@ -402,6 +408,10 @@ fn default_cors_wildcard() -> JsonStringList {
|
|||||||
JsonStringList(vec!["*".to_string()])
|
JsonStringList(vec!["*".to_string()])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_py_bootstrap_parser_selection() -> ParserSelection {
|
||||||
|
ParserSelection::None
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
|
fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
|
||||||
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
|
serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
|
||||||
}
|
}
|
||||||
@@ -526,10 +536,14 @@ impl ServeArgs {
|
|||||||
/// Build the managed Python-engine spawn configuration with the given
|
/// Build the managed Python-engine spawn configuration with the given
|
||||||
/// handshake port.
|
/// handshake port.
|
||||||
pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
|
pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
|
||||||
|
let reasoning_parser =
|
||||||
|
effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model);
|
||||||
|
|
||||||
self.managed_engine.clone().into_config(
|
self.managed_engine.clone().into_config(
|
||||||
self.runtime.model.clone(),
|
self.runtime.model.clone(),
|
||||||
self.runtime.max_model_len,
|
self.runtime.max_model_len,
|
||||||
self.runtime.max_logprobs,
|
self.runtime.max_logprobs,
|
||||||
|
reasoning_parser.as_deref(),
|
||||||
self.runtime.language_model_only,
|
self.runtime.language_model_only,
|
||||||
self.runtime.disable_log_stats,
|
self.runtime.disable_log_stats,
|
||||||
self.runtime.shutdown_timeout,
|
self.runtime.shutdown_timeout,
|
||||||
@@ -538,6 +552,16 @@ impl ServeArgs {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn effective_engine_reasoning_parser(selection: &ParserSelection, model: &str) -> Option<String> {
|
||||||
|
match selection {
|
||||||
|
ParserSelection::Auto => ReasoningParserFactory::global()
|
||||||
|
.resolve_name_for_model(model)
|
||||||
|
.map(str::to_string),
|
||||||
|
ParserSelection::None => None,
|
||||||
|
ParserSelection::Explicit(name) => Some(name.clone()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Allocate fresh IPC endpoints for one managed frontend instance.
|
/// Allocate fresh IPC endpoints for one managed frontend instance.
|
||||||
fn frontend_ipc_addresses() -> (String, String) {
|
fn frontend_ipc_addresses() -> (String, String) {
|
||||||
let preferred_base_path = std::env::var_os("VLLM_RPC_BASE_PATH")
|
let preferred_base_path = std::env::var_os("VLLM_RPC_BASE_PATH")
|
||||||
|
|||||||
+191
-39
@@ -100,10 +100,13 @@ fn serve_args_auto_forward_python_flags_without_separator() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(
|
expect![[r#"
|
||||||
args.managed_engine.python_args,
|
[
|
||||||
vec!["--quantization", "awq"]
|
"--quantization",
|
||||||
);
|
"awq",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -114,7 +117,12 @@ fn serve_args_auto_forward_enable_lora_to_python() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(args.managed_engine.python_args, vec!["--enable-lora"]);
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--enable-lora",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -134,7 +142,15 @@ fn serve_args_forward_shutdown_timeout_to_managed_engine() {
|
|||||||
assert_eq!(args.runtime.shutdown_timeout, 60);
|
assert_eq!(args.runtime.shutdown_timeout, 60);
|
||||||
|
|
||||||
let config = args.to_managed_engine_config(5555);
|
let config = args.to_managed_engine_config(5555);
|
||||||
assert_eq!(config.python_args, vec!["--shutdown-timeout", "60"]);
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--reasoning-parser",
|
||||||
|
"qwen3",
|
||||||
|
"--shutdown-timeout",
|
||||||
|
"60",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&config.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -148,7 +164,14 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() {
|
|||||||
assert!(args.runtime.disable_log_stats);
|
assert!(args.runtime.disable_log_stats);
|
||||||
|
|
||||||
let config = args.to_managed_engine_config(5555);
|
let config = args.to_managed_engine_config(5555);
|
||||||
assert_eq!(config.python_args, vec!["--disable-log-stats"]);
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--reasoning-parser",
|
||||||
|
"qwen3",
|
||||||
|
"--disable-log-stats",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&config.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -171,7 +194,106 @@ fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
|
|||||||
assert_eq!(frontend_config.max_logprobs, Some(-1));
|
assert_eq!(frontend_config.max_logprobs, Some(-1));
|
||||||
|
|
||||||
let engine_config = args.to_managed_engine_config(5555);
|
let engine_config = args.to_managed_engine_config(5555);
|
||||||
assert_eq!(engine_config.python_args, vec!["--max-logprobs", "-1"]);
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--max-logprobs",
|
||||||
|
"-1",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"qwen3",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&engine_config.python_args);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn serve_args_resolve_auto_reasoning_parser_for_managed_engine() {
|
||||||
|
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
|
||||||
|
|
||||||
|
let Command::Serve(args) = cli.command else {
|
||||||
|
panic!("expected serve args");
|
||||||
|
};
|
||||||
|
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
|
||||||
|
|
||||||
|
let config = args.to_managed_engine_config(5555);
|
||||||
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--reasoning-parser",
|
||||||
|
"qwen3",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&config.python_args);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn serve_args_forward_explicit_reasoning_parser_to_managed_engine() {
|
||||||
|
let cli = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"serve",
|
||||||
|
"Unknown/Model",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"deepseek_r1",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let Command::Serve(args) = cli.command else {
|
||||||
|
panic!("expected serve args");
|
||||||
|
};
|
||||||
|
|
||||||
|
let config = args.to_managed_engine_config(5555);
|
||||||
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--reasoning-parser",
|
||||||
|
"deepseek_r1",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&config.python_args);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn serve_args_do_not_forward_disabled_reasoning_parser_to_managed_engine() {
|
||||||
|
let cli = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"serve",
|
||||||
|
"Qwen/Qwen3-0.6B",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"none",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let Command::Serve(args) = cli.command else {
|
||||||
|
panic!("expected serve args");
|
||||||
|
};
|
||||||
|
|
||||||
|
let config = args.to_managed_engine_config(5555);
|
||||||
|
assert!(config.python_args.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn serve_args_forward_reasoning_parser_even_with_passthrough_reasoning_parser() {
|
||||||
|
let cli = Cli::try_parse_from([
|
||||||
|
"vllm-rs",
|
||||||
|
"serve",
|
||||||
|
"Qwen/Qwen3-0.6B",
|
||||||
|
"--",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"deepseek_r1",
|
||||||
|
])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let Command::Serve(args) = cli.command else {
|
||||||
|
panic!("expected serve args");
|
||||||
|
};
|
||||||
|
|
||||||
|
let config = args.to_managed_engine_config(5555);
|
||||||
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--reasoning-parser",
|
||||||
|
"deepseek_r1",
|
||||||
|
"--reasoning-parser",
|
||||||
|
"qwen3",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&config.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -181,10 +303,13 @@ fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(
|
expect![[r#"
|
||||||
args.managed_engine.python_args,
|
[
|
||||||
vec!["--tensor-parallel-size", "2"]
|
"--tensor-parallel-size",
|
||||||
);
|
"2",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -429,8 +554,8 @@ fn frontend_args_accept_json() {
|
|||||||
runtime: SharedRuntimeArgs {
|
runtime: SharedRuntimeArgs {
|
||||||
model: "Qwen/Qwen3-0.6B",
|
model: "Qwen/Qwen3-0.6B",
|
||||||
engine_ready_timeout_secs: 600,
|
engine_ready_timeout_secs: 600,
|
||||||
tool_call_parser: Auto,
|
tool_call_parser: None,
|
||||||
reasoning_parser: Auto,
|
reasoning_parser: None,
|
||||||
renderer: Auto,
|
renderer: Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
max_model_len: None,
|
max_model_len: None,
|
||||||
@@ -490,8 +615,8 @@ fn frontend_args_json_applies_defaults() {
|
|||||||
};
|
};
|
||||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
|
assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
|
||||||
assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
|
assert_eq!(args.runtime.tool_call_parser, ParserSelection::None);
|
||||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
|
assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
|
||||||
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
||||||
assert_eq!(args.runtime.max_model_len, None);
|
assert_eq!(args.runtime.max_model_len, None);
|
||||||
assert_eq!(args.runtime.max_logprobs, None);
|
assert_eq!(args.runtime.max_logprobs, None);
|
||||||
@@ -784,10 +909,15 @@ fn serve_args_keep_python_passthrough_flags_after_separator() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(
|
expect![[r#"
|
||||||
args.managed_engine.python_args,
|
[
|
||||||
vec!["--tensor-parallel-size", "2", "--dtype", "float16"]
|
"--tensor-parallel-size",
|
||||||
);
|
"2",
|
||||||
|
"--dtype",
|
||||||
|
"float16",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -809,10 +939,15 @@ fn serve_args_keep_python_multi_char_alias_after_separator() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(
|
expect![[r#"
|
||||||
args.managed_engine.python_args,
|
[
|
||||||
vec!["-tp", "2", "--dtype", "float16"]
|
"-tp",
|
||||||
);
|
"2",
|
||||||
|
"--dtype",
|
||||||
|
"float16",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -830,10 +965,13 @@ fn serve_args_keep_frontend_arg_after_separator() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(
|
expect![[r#"
|
||||||
args.managed_engine.python_args,
|
[
|
||||||
vec!["--uds", "/tmp/vllm.sock"]
|
"--uds",
|
||||||
);
|
"/tmp/vllm.sock",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -853,10 +991,15 @@ fn serve_args_keep_python_multi_char_engine_aliases_after_separator() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(
|
expect![[r#"
|
||||||
args.managed_engine.python_args,
|
[
|
||||||
vec!["-dpr", "1", "-dpl", "2"]
|
"-dpr",
|
||||||
);
|
"1",
|
||||||
|
"-dpl",
|
||||||
|
"2",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -866,7 +1009,13 @@ fn serve_args_auto_forward_unknown_flags_without_separator() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(args.managed_engine.python_args, vec!["--foo", "bar"]);
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--foo",
|
||||||
|
"bar",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -883,10 +1032,13 @@ fn serve_args_auto_forward_negative_value_without_separator() {
|
|||||||
let Command::Serve(args) = cli.command else {
|
let Command::Serve(args) = cli.command else {
|
||||||
panic!("expected serve args");
|
panic!("expected serve args");
|
||||||
};
|
};
|
||||||
assert_eq!(
|
expect![[r#"
|
||||||
args.managed_engine.python_args,
|
[
|
||||||
vec!["--num-gpu-blocks-override", "-1"]
|
"--num-gpu-blocks-override",
|
||||||
);
|
"-1",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.managed_engine.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1237,8 +1389,8 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
|||||||
listener_mode: InheritedFd {
|
listener_mode: InheritedFd {
|
||||||
fd: 3,
|
fd: 3,
|
||||||
},
|
},
|
||||||
tool_call_parser: Auto,
|
tool_call_parser: None,
|
||||||
reasoning_parser: Auto,
|
reasoning_parser: None,
|
||||||
renderer: Auto,
|
renderer: Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
chat_template: None,
|
chat_template: None,
|
||||||
|
|||||||
@@ -43,6 +43,11 @@ pub struct ManagedEngineArgs {
|
|||||||
/// Arguments after an explicit `--` are forwarded verbatim. Before `--`,
|
/// Arguments after an explicit `--` are forwarded verbatim. Before `--`,
|
||||||
/// `vllm-rs serve` automatically keeps recognized frontend options on
|
/// `vllm-rs serve` automatically keeps recognized frontend options on
|
||||||
/// the Rust side and forwards everything else to Python.
|
/// the Rust side and forwards everything else to Python.
|
||||||
|
///
|
||||||
|
/// The explicit `--` passthrough is a last-resort escape hatch. Rust does
|
||||||
|
/// not interpret, validate, or de-duplicate those arguments against
|
||||||
|
/// managed-engine arguments that it appends later; if the same Python flag
|
||||||
|
/// appears more than once, Python argparse owns the final result.
|
||||||
#[arg(
|
#[arg(
|
||||||
last = true,
|
last = true,
|
||||||
allow_hyphen_values = true,
|
allow_hyphen_values = true,
|
||||||
@@ -72,6 +77,7 @@ impl ManagedEngineArgs {
|
|||||||
model: String,
|
model: String,
|
||||||
max_model_len: Option<u32>,
|
max_model_len: Option<u32>,
|
||||||
max_logprobs: Option<i32>,
|
max_logprobs: Option<i32>,
|
||||||
|
reasoning_parser: Option<&str>,
|
||||||
language_model_only: bool,
|
language_model_only: bool,
|
||||||
disable_log_stats: bool,
|
disable_log_stats: bool,
|
||||||
shutdown_timeout: u64,
|
shutdown_timeout: u64,
|
||||||
@@ -87,6 +93,10 @@ impl ManagedEngineArgs {
|
|||||||
python_args.push("--max-logprobs".to_string());
|
python_args.push("--max-logprobs".to_string());
|
||||||
python_args.push(max_logprobs.to_string());
|
python_args.push(max_logprobs.to_string());
|
||||||
}
|
}
|
||||||
|
if let Some(reasoning_parser) = reasoning_parser {
|
||||||
|
python_args.push("--reasoning-parser".to_string());
|
||||||
|
python_args.push(reasoning_parser.to_string());
|
||||||
|
}
|
||||||
if language_model_only {
|
if language_model_only {
|
||||||
python_args.push("--language-model-only".to_string());
|
python_args.push("--language-model-only".to_string());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -164,6 +164,7 @@ pub fn lower_sampling_params(
|
|||||||
logit_bias,
|
logit_bias,
|
||||||
allowed_token_ids,
|
allowed_token_ids,
|
||||||
bad_words_token_ids: tokenize_bad_words(bad_words.as_deref(), tokenizer)?,
|
bad_words_token_ids: tokenize_bad_words(bad_words.as_deref(), tokenizer)?,
|
||||||
|
// TODO: Validate structured-output schemas and regexes before submitting requests to engine-core.
|
||||||
structured_outputs,
|
structured_outputs,
|
||||||
logprob_token_ids,
|
logprob_token_ids,
|
||||||
skip_reading_prefix_cache,
|
skip_reading_prefix_cache,
|
||||||
|
|||||||
Reference in New Issue
Block a user