From 68dcc4fd86b11d1cfa228cfc2c9d2b9706ba7af3 Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Wed, 9 Sep 2026 10:54:54 +0200 Subject: [PATCH] [Rust Frontend] Make --max-model-len optional for the render server (#55328) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Wen Zhou --- rust/src/cmd/src/cli.rs | 5 ++-- rust/src/cmd/src/cli/tests.rs | 14 ++++++++++- rust/src/server/src/render.rs | 11 ++++++--- rust/src/server/src/routes/render.rs | 2 +- rust/src/server/src/routes/tests.rs | 35 +++++++++++++++++++++++++++- 5 files changed, 59 insertions(+), 8 deletions(-) diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 8f8208c016..55374c3f29 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -139,9 +139,10 @@ pub struct RenderArgs { /// How message content is exposed to the chat template. #[arg(long, default_value_t)] chat_template_content_format: ChatTemplateContentFormatOption, - /// Maximum model context length used for request validation. + /// Maximum model context length used for request validation. When not set, + /// prompt-length validation is skipped and the engine enforces its own limit in later stage. #[arg(long)] - max_model_len: u32, + max_model_len: Option, /// Maximum accepted logprobs count; -1 disables the cap. #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)] max_logprobs: Option, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 603c820bfc..0ab6882f34 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -57,7 +57,7 @@ fn render_args_build_config_without_tls() { assert_eq!(config.model, "Qwen/Qwen2.5-0.5B-Instruct"); assert_eq!(config.host, "127.0.0.1"); assert_eq!(config.port, 8080); - assert_eq!(config.max_model_len, 32768); + assert_eq!(config.max_model_len, Some(32768)); assert_eq!(config.served_model_name, ["qwen"]); assert_eq!(config.tool_call_parser, ParserSelection::Auto); assert_eq!(config.reasoning_parser, ParserSelection::Auto); @@ -66,6 +66,18 @@ fn render_args_build_config_without_tls() { assert!(config.tls.is_none()); } +#[test] +fn render_args_allow_omitted_max_model_len() { + let cli = Cli::try_parse_from(["vllm-rs", "render", "Qwen/Qwen2.5-0.5B-Instruct"]).unwrap(); + + let Command::Render(args) = cli.command else { + panic!("expected render args"); + }; + let config = args.into_config(); + + assert_eq!(config.max_model_len, None); +} + #[test] fn render_args_build_config_with_tls() { let cli = Cli::try_parse_from([ diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs index 5f5e8cdc99..6942694afb 100644 --- a/rust/src/server/src/render.rs +++ b/rust/src/server/src/render.rs @@ -33,7 +33,7 @@ pub struct RenderConfig { pub chat_template: Option, pub default_chat_template_kwargs: HashMap, pub chat_template_content_format: ChatTemplateContentFormatOption, - pub max_model_len: u32, + pub max_model_len: Option, pub max_logprobs: Option, pub tls: Option, } @@ -56,6 +56,9 @@ impl RenderConfig { pub(crate) struct RenderState { pub(crate) model: String, pub(crate) served_model_names: Vec, + /// Unlike `text.max_model_len()` this stays `None` when unset, + /// so model cards advertise `null` instead of the `u32::MAX`. + pub(crate) max_model_len: Option, pub(crate) text: TextRequestProcessor, pub(crate) chat: ChatRequestProcessor, } @@ -77,7 +80,8 @@ async fn build_state(config: &RenderConfig) -> Result> { .context("failed to load renderer/tokenizer backends")?; let served_model_names = crate::effective_served_model_names(&config.model, &config.served_model_name); - let text = TextRequestProcessor::new(loaded.text_backend, config.max_model_len) + let max_model_len = config.max_model_len.unwrap_or(u32::MAX); + let text = TextRequestProcessor::new(loaded.text_backend, max_model_len) .with_max_logprobs(config.max_logprobs); let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections( config.tool_call_parser.clone(), @@ -86,6 +90,7 @@ async fn build_state(config: &RenderConfig) -> Result> { Ok(Arc::new(RenderState { model: config.model.clone(), served_model_names, + max_model_len: config.max_model_len, text, chat, })) @@ -154,7 +159,7 @@ mod tests { chat_template: None, default_chat_template_kwargs: HashMap::new(), chat_template_content_format: ChatTemplateContentFormatOption::Auto, - max_model_len: 128, + max_model_len: Some(128), max_logprobs: None, tls: None, }, diff --git a/rust/src/server/src/routes/render.rs b/rust/src/server/src/routes/render.rs index 8833a2d94e..a4ae0298e9 100644 --- a/rust/src/server/src/routes/render.rs +++ b/rust/src/server/src/routes/render.rs @@ -55,7 +55,7 @@ async fn list_models(State(state): State>) -> Json axum::Router { fn test_render_app_with_parser_selections( tool_call_parser: ParserSelection, reasoning_parser: ParserSelection, +) -> axum::Router { + test_render_app_with(Some(128), tool_call_parser, reasoning_parser) +} + +fn test_render_app_with_max_model_len(max_model_len: Option) -> axum::Router { + test_render_app_with(max_model_len, ParserSelection::Auto, ParserSelection::Auto) +} + +fn test_render_app_with( + max_model_len: Option, + tool_call_parser: ParserSelection, + reasoning_parser: ParserSelection, ) -> axum::Router { let backend = Arc::new(FakeChatBackend::new()); build_render_router(Arc::new(RenderState { model: "backend-model".to_string(), served_model_names: vec!["render-model".to_string()], - text: TextRequestProcessor::new(backend.clone(), 128), + max_model_len, + text: TextRequestProcessor::new(backend.clone(), max_model_len.unwrap_or(u32::MAX)), chat: ChatRequestProcessor::render_only(backend) .with_parser_selections(tool_call_parser, reasoning_parser), })) @@ -1414,6 +1427,26 @@ async fn render_completion_returns_generate_request_with_body_request_id() { assert!(json[0].get("mm_features").is_none()); } +#[tokio::test] +async fn render_list_models_reports_configured_max_model_len() { + for (configured, expected) in [(Some(128), json!(128)), (None, serde_json::Value::Null)] { + let mut app = test_render_app_with_max_model_len(configured); + let response = app + .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request")) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + let card = json["data"][0].as_object().expect("card object"); + assert!(card.contains_key("max_model_len")); + assert_eq!( + card["max_model_len"], expected, + "configured: {configured:?}" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn render_chat_response_can_be_submitted_to_generate_unchanged() {