diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 454a397ad0..3ae827d286 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -148,11 +148,6 @@ pub struct SharedRuntimeArgs { #[arg(long)] #[serde(default)] pub language_model_only: bool, - /// Override the maximum model context length. When set, the frontend uses - /// this value instead of the model's `max_position_embeddings` from - /// `config.json`. - #[arg(long)] - pub max_model_len: Option, /// Maximum number of log probabilities to return when `logprobs` is /// specified in sampling parameters. `-1` means no cap. #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)] @@ -664,7 +659,6 @@ impl ServeArgs { self.managed_engine.clone().into_config( self.runtime.model.clone(), - self.runtime.max_model_len, self.runtime.max_logprobs, profiler_config, reasoning_parser.as_deref(), diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index d8ec9792b7..b0926fb24a 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -58,9 +58,6 @@ fn serve_args_forward_python_flags_with_separator() { reasoning_parser: Auto, renderer: Auto, language_model_only: false, - max_model_len: Some( - 512, - ), max_logprobs: None, grpc_port: None, shutdown_timeout: 0, @@ -102,6 +99,9 @@ fn serve_args_forward_python_flags_with_separator() { handshake_port: None, data_parallel_size: 1, data_parallel_size_local: None, + max_model_len: Some( + "512", + ), python_args: [ "--dtype", "float16", @@ -756,7 +756,6 @@ fn frontend_args_accept_json() { reasoning_parser: None, renderer: Auto, language_model_only: false, - max_model_len: None, max_logprobs: None, grpc_port: None, shutdown_timeout: 0, @@ -823,11 +822,32 @@ fn frontend_args_json_applies_defaults() { assert_eq!(args.runtime.tool_call_parser, ParserSelection::None); assert_eq!(args.runtime.reasoning_parser, ParserSelection::None); assert_eq!(args.runtime.renderer, RendererSelection::Auto); - assert_eq!(args.runtime.max_model_len, None); assert_eq!(args.runtime.max_logprobs, None); assert_eq!(args.runtime.shutdown_timeout, 0); } +#[test] +fn frontend_args_json_ignores_engine_owned_max_model_len() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "frontend", + "--listen-fd", + "3", + "--input-address", + "ipc:///tmp/input.sock", + "--output-address", + "ipc:///tmp/output.sock", + "--args-json", + r#"{"model_tag":"Qwen/Qwen3-0.6B","max_model_len":-1}"#, + ]) + .unwrap(); + + let Command::Frontend(args) = cli.command else { + panic!("expected frontend args"); + }; + assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B"); +} + #[test] fn frontend_args_json_accepts_supported_non_default_fields() { let cli = Cli::try_parse_from([ @@ -840,7 +860,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() { "--output-address", "ipc:///tmp/output.sock", "--args-json", - r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"max_logprobs":-1,"shutdown_timeout":3}"#, + r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#, ]) .unwrap(); @@ -858,11 +878,38 @@ fn frontend_args_json_accepts_supported_non_default_fields() { ); assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32); assert!(args.runtime.language_model_only); - assert_eq!(args.runtime.max_model_len, Some(8192)); assert_eq!(args.runtime.max_logprobs, Some(-1)); assert_eq!(args.runtime.shutdown_timeout, 3); } +#[test] +fn serve_args_forward_auto_max_model_len_to_managed_engine() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--max-model-len", + "auto", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + assert_eq!(args.managed_engine.max_model_len.as_deref(), Some("auto")); + + let config = args.to_managed_engine_config(5555); + expect![[r#" + [ + "--max-model-len", + "auto", + "--reasoning-parser", + "qwen3", + ] + "#]] + .assert_debug_eq(&config.python_args); +} + #[test] fn serve_args_accept_none_reasoning_parser() { let cli = Cli::try_parse_from([ @@ -1278,7 +1325,6 @@ fn serve_args_accept_handshake_aliases() { reasoning_parser: Auto, renderer: Auto, language_model_only: false, - max_model_len: None, max_logprobs: None, grpc_port: None, shutdown_timeout: 0, @@ -1322,6 +1368,7 @@ fn serve_args_accept_handshake_aliases() { ), data_parallel_size: 4, data_parallel_size_local: None, + max_model_len: None, python_args: [], }, }, diff --git a/rust/src/managed-engine/src/cli.rs b/rust/src/managed-engine/src/cli.rs index 95d61f652c..26a6e56e79 100644 --- a/rust/src/managed-engine/src/cli.rs +++ b/rust/src/managed-engine/src/cli.rs @@ -39,6 +39,12 @@ pub struct ManagedEngineArgs { /// Number of data parallel replicas to run on this node. #[arg(long)] pub data_parallel_size_local: Option, + /// Maximum model context length forwarded to the managed Python engine. + /// + /// Rust leaves validation to Python so values such as `auto` and + /// human-readable integers retain their engine-owned semantics. + #[arg(long)] + pub max_model_len: Option, /// Additional arguments forwarded to `python -m vllm.entrypoints.cli.main /// serve ...`. @@ -78,7 +84,6 @@ impl ManagedEngineArgs { pub fn into_config( self, model: String, - max_model_len: Option, max_logprobs: Option, profiler_config: Option, reasoning_parser: Option<&str>, @@ -89,9 +94,9 @@ impl ManagedEngineArgs { ) -> ManagedEngineConfig { let mut python_args = self.python_args; // Manually forward some args to the Python engine. - if let Some(max_model_len) = max_model_len { + if let Some(max_model_len) = self.max_model_len { python_args.push("--max-model-len".to_string()); - python_args.push(max_model_len.to_string()); + python_args.push(max_model_len); } if let Some(max_logprobs) = max_logprobs { python_args.push("--max-logprobs".to_string());