[Rust Frontend] Support max_logprobs validation (#45674)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -127,6 +127,11 @@ pub struct SharedRuntimeArgs {
|
||||
/// `config.json`.
|
||||
#[arg(long)]
|
||||
pub max_model_len: Option<u32>,
|
||||
/// Maximum number of log probabilities to return when `logprobs` is
|
||||
/// specified in sampling parameters. `-1` means no cap.
|
||||
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
|
||||
#[serde(default)]
|
||||
pub max_logprobs: Option<i32>,
|
||||
/// TCP port for the gRPC Generate service. When not set, no gRPC server is
|
||||
/// started.
|
||||
#[arg(long)]
|
||||
@@ -281,6 +286,7 @@ impl SharedRuntimeArgs {
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
@@ -324,6 +330,7 @@ impl SharedRuntimeArgs {
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
api_server_options,
|
||||
api_keys: self.api_key,
|
||||
disable_log_stats: self.disable_log_stats,
|
||||
@@ -467,6 +474,7 @@ impl ServeArgs {
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
self.runtime.max_model_len,
|
||||
self.runtime.max_logprobs,
|
||||
self.runtime.language_model_only,
|
||||
self.runtime.disable_log_stats,
|
||||
self.runtime.shutdown_timeout,
|
||||
|
||||
@@ -38,6 +38,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
max_model_len: Some(
|
||||
512,
|
||||
),
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
@@ -134,6 +135,29 @@ fn serve_args_forward_disable_log_stats_to_managed_engine() {
|
||||
assert_eq!(config.python_args, vec!["--disable-log-stats"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--max-logprobs",
|
||||
"-1",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(args.runtime.max_logprobs, Some(-1));
|
||||
|
||||
let frontend_config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
assert_eq!(frontend_config.max_logprobs, Some(-1));
|
||||
|
||||
let engine_config = args.to_managed_engine_config(5555);
|
||||
assert_eq!(engine_config.python_args, vec!["--max-logprobs", "-1"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
|
||||
let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
|
||||
@@ -388,6 +412,7 @@ fn frontend_args_accept_json() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_model_len: None,
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
@@ -431,6 +456,7 @@ fn frontend_args_json_applies_defaults() {
|
||||
assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::Auto);
|
||||
assert_eq!(args.runtime.max_model_len, None);
|
||||
assert_eq!(args.runtime.max_logprobs, None);
|
||||
assert_eq!(args.runtime.shutdown_timeout, 0);
|
||||
}
|
||||
|
||||
@@ -446,7 +472,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"shutdown_timeout":3}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"max_logprobs":-1,"shutdown_timeout":3}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
@@ -465,6 +491,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
|
||||
assert!(args.runtime.language_model_only);
|
||||
assert_eq!(args.runtime.max_model_len, Some(8192));
|
||||
assert_eq!(args.runtime.max_logprobs, Some(-1));
|
||||
assert_eq!(args.runtime.shutdown_timeout, 3);
|
||||
}
|
||||
|
||||
@@ -792,6 +819,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_model_len: None,
|
||||
max_logprobs: None,
|
||||
grpc_port: None,
|
||||
shutdown_timeout: 0,
|
||||
chat_template: None,
|
||||
@@ -917,6 +945,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -985,6 +1014,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -1068,6 +1098,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
chat_template: None,
|
||||
default_chat_template_kwargs: None,
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
|
||||
@@ -202,14 +202,6 @@ pub struct EngineUnsupportedArgs {
|
||||
#[arg(long)]
|
||||
pub tokenizer_revision: Option<Unsupported>,
|
||||
|
||||
/// Maximum number of log probabilities to return when `logprobs` is
|
||||
/// specified in `SamplingParams`. The default value comes the default for
|
||||
/// the OpenAI Chat Completions API. -1 means no cap, i.e. all
|
||||
/// (output_length * vocab_size) logprobs are allowed to be returned and
|
||||
/// it may cause OOM.
|
||||
#[arg(long)]
|
||||
pub max_logprobs: Option<Unsupported>,
|
||||
|
||||
/// Skip initialization of tokenizer and detokenizer. Expects valid
|
||||
/// `prompt_token_ids` and `None` for prompt from the input. The generated
|
||||
/// output will contain token ids.
|
||||
|
||||
Reference in New Issue
Block a user