[Rust Frontend] Support --generation-config vllm (#53044)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -22,8 +22,8 @@ use serde_json::Value;
|
||||
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_chat::ReasoningParserFactory;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_chat::{GenerationConfigMode, ReasoningParserFactory};
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||
@@ -186,6 +186,12 @@ pub struct SharedRuntimeArgs {
|
||||
/// public model ID.
|
||||
pub model: String,
|
||||
|
||||
/// The source of generation-config sampling defaults. `"auto"` loads the
|
||||
/// model's defaults, while `"vllm"` uses vLLM's neutral defaults.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub generation_config: GenerationConfigMode,
|
||||
|
||||
/// Maximum time to wait for the expected engines to register on the
|
||||
/// frontend transport.
|
||||
#[arg(
|
||||
@@ -482,6 +488,7 @@ impl SharedRuntimeArgs {
|
||||
None => CoordinatorMode::None,
|
||||
},
|
||||
model: self.model,
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
@@ -535,6 +542,7 @@ impl SharedRuntimeArgs {
|
||||
},
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: self.model,
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
|
||||
use expect_test::expect;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection};
|
||||
use vllm_server::{
|
||||
Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection,
|
||||
};
|
||||
|
||||
use super::{BenchCommand, Cli, Command};
|
||||
|
||||
@@ -88,6 +90,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
uds: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
@@ -732,6 +735,24 @@ fn serve_args_reject_unsupported_flag_arg() {
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_custom_generation_config_source() {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--generation-config",
|
||||
"/tmp/custom-config",
|
||||
])
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value '/tmp/custom-config' for '--generation-config <GENERATION_CONFIG>': generation config source `/tmp/custom-config` is not implemented yet (expected one of: auto, vllm)
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]].assert_eq(&error.to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_reject_unsupported_no_flag_alias() {
|
||||
let error = Cli::try_parse_from([
|
||||
@@ -788,6 +809,7 @@ fn frontend_args_accept_json() {
|
||||
data_parallel_size: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
@@ -858,6 +880,7 @@ fn frontend_args_json_applies_defaults() {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
assert_eq!(args.runtime.generation_config, GenerationConfigMode::Auto);
|
||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
|
||||
assert_eq!(args.runtime.tool_call_parser, ParserSelection::None);
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
|
||||
@@ -901,7 +924,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","generation_config":"vllm","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
@@ -909,6 +932,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 42);
|
||||
assert_eq!(args.runtime.generation_config, GenerationConfigMode::Vllm);
|
||||
assert_eq!(
|
||||
args.runtime.tool_call_parser,
|
||||
ParserSelection::Explicit("hermes".to_string())
|
||||
@@ -1379,6 +1403,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
uds: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
@@ -1523,6 +1548,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
},
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
host: "127.0.0.1",
|
||||
@@ -1608,6 +1634,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
},
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
host: "127.0.0.1",
|
||||
@@ -1715,6 +1742,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
address: "tcp://127.0.0.1:7000",
|
||||
},
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: InheritedFd {
|
||||
fd: 3,
|
||||
|
||||
@@ -240,15 +240,12 @@ pub struct EngineUnsupportedArgs {
|
||||
#[arg(long)]
|
||||
pub hf_overrides: Option<Unsupported>,
|
||||
|
||||
/// The folder path to the generation config. Defaults to `"auto"`, the
|
||||
/// generation config will be loaded from model path. If set to `"vllm"`, no
|
||||
/// generation config is loaded, vLLM defaults will be used. If set to a
|
||||
/// folder path, the generation config will be loaded from the specified
|
||||
/// folder path. If `max_new_tokens` is specified in generation config,
|
||||
/// then it sets a server-wide limit on the number of output tokens for
|
||||
/// all requests.
|
||||
/// Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
|
||||
/// used with `--generation-config auto`, the override parameters will be
|
||||
/// merged with the default config from the model. If used with
|
||||
/// `--generation-config vllm`, only the override parameters are used.
|
||||
#[arg(long)]
|
||||
pub generation_config: Option<Unsupported>,
|
||||
pub override_generation_config: Option<Unsupported>,
|
||||
|
||||
/// IOProcessor plugin name to load at model startup
|
||||
#[arg(long)]
|
||||
|
||||
Reference in New Issue
Block a user