From 16cfe728d8d0bc3cd4a8397db0f392dd52a2c109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:41:06 -0400 Subject: [PATCH] [Bugfix][Rust Frontend] Reject n > 1 in the `/inference/v1/generate` route (#52844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bugen Zhao Signed-off-by: Quentin Gallouédec Signed-off-by: Bugen Zhao --- .../server/src/routes/inference/generate.rs | 2 +- .../src/routes/inference/generate/convert.rs | 6 ++-- .../src/routes/inference/generate/types.rs | 19 +++++++++++- .../src/routes/inference/generate/validate.rs | 30 +++++++++++++++++-- rust/src/server/src/routes/render.rs | 7 +++-- rust/src/server/src/routes/tests.rs | 30 +++++++++++++++++++ 6 files changed, 85 insertions(+), 9 deletions(-) diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 61fed421d3..aa84913f23 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -26,11 +26,11 @@ use vllm_llm::{ }; use self::convert::{ResponseOptions, prepare_generate_request}; -pub(crate) use self::types::GenerateRequest; use self::types::{ GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice, GenerateStreamResponse, }; +pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams}; pub(crate) use self::validate::validate_request_compat; use crate::config::ApiServerOptions; use crate::error::{ApiError, bail_server_error, server_error, text_submit_error}; diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index 5c14107989..09c735238b 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -54,9 +54,9 @@ pub(super) fn prepare_generate_request( .as_ref() .and_then(|options| options.continuous_usage_stats) .unwrap_or(false); - let include_logprobs = request.sampling_params.logprobs.is_some(); - let include_prompt_logprobs = request.sampling_params.prompt_logprobs.is_some(); - let mut sampling_params = request.sampling_params; + let include_logprobs = request.sampling_params.inner.logprobs.is_some(); + let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some(); + let mut sampling_params = request.sampling_params.inner; sampling_params.vllm_xargs = merge_kv_transfer_params( sampling_params.vllm_xargs, request.kv_transfer_params.as_ref(), diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 200209e095..b9220aa7b7 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -11,6 +11,23 @@ use vllm_text::SamplingParams; use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage}; +/// Sampling parameters for the token-in/token-out generate API. +/// +/// Wraps [`SamplingParams`] to additionally capture `n`, which the shared +/// northbound type intentionally omits (parallel sampling is handled by +/// higher layers, and the Rust frontend does not implement it). Capturing it +/// here lets validation reject `n > 1` explicitly instead of silently +/// dropping the key and returning a single choice. +#[serde_with::skip_serializing_none] +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct GenerateSamplingParams { + /// Number of output sequences to generate. Only `1` is supported. + pub n: Option, + /// The supported sampling parameters, lowered to the engine. + #[serde(flatten)] + pub inner: SamplingParams, +} + /// vLLM-compatible request type for the token-in/token-out generate API. #[serde_with::skip_serializing_none] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] @@ -18,7 +35,7 @@ pub struct GenerateRequest { pub request_id: Option, pub model: Option, pub token_ids: Vec, - pub sampling_params: SamplingParams, + pub sampling_params: GenerateSamplingParams, #[serde(default)] pub stream: bool, pub stream_options: Option, diff --git a/rust/src/server/src/routes/inference/generate/validate.rs b/rust/src/server/src/routes/inference/generate/validate.rs index 99e2b23158..9ff8d0460e 100644 --- a/rust/src/server/src/routes/inference/generate/validate.rs +++ b/rust/src/server/src/routes/inference/generate/validate.rs @@ -23,6 +23,10 @@ pub(crate) fn validate_request_compat( ); } + if request.sampling_params.n.unwrap_or(1) != 1 { + bail_invalid_request!(param = "n", "Only n=1 is supported."); + } + if request.token_ids.is_empty() { bail_invalid_request!( param = "token_ids", @@ -30,14 +34,14 @@ pub(crate) fn validate_request_compat( ); } - if request.sampling_params.max_tokens == Some(0) { + if request.sampling_params.inner.max_tokens == Some(0) { bail_invalid_request!( param = "sampling_params", "max_tokens must be greater than 0." ); } - if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs { + if let Some(prompt_logprobs) = request.sampling_params.inner.prompt_logprobs { if prompt_logprobs < 0 && prompt_logprobs != -1 { bail_invalid_request!( param = "sampling_params", @@ -98,6 +102,28 @@ mod tests { assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); } + #[test] + fn validate_request_compat_rejects_parallel_sampling() { + let request: GenerateRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "token_ids": [11, 22], + "sampling_params": {"n": 4} + })) + .expect("parse request"); + assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); + } + + #[test] + fn validate_request_compat_accepts_explicit_n_one() { + let request: GenerateRequest = serde_json::from_value(json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "token_ids": [11, 22], + "sampling_params": {"n": 1} + })) + .expect("parse request"); + assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()); + } + #[test] fn validate_request_compat_rejects_empty_token_ids() { let request = GenerateRequest { diff --git a/rust/src/server/src/routes/render.rs b/rust/src/server/src/routes/render.rs index 1972c02d09..b2d7601ddc 100644 --- a/rust/src/server/src/routes/render.rs +++ b/rust/src/server/src/routes/render.rs @@ -17,7 +17,7 @@ use crate::error::{ApiError, text_submit_error}; use crate::lora::LoraModelResolution; use crate::render::RenderState; use crate::routes::inference::generate::{ - GenerateRequest, validate_request_compat as validate_generate_request, + GenerateRequest, GenerateSamplingParams, validate_request_compat as validate_generate_request, }; use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions}; use crate::routes::openai::utils::validated_json::ValidatedJson; @@ -86,7 +86,10 @@ fn lower_render_request( request_id: Some(text_request.request_id), model: Some(model), token_ids, - sampling_params: text_request.sampling_params, + sampling_params: GenerateSamplingParams { + n: None, + inner: text_request.sampling_params, + }, stream, stream_options, cache_salt: text_request.cache_salt, diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 9ac21ae07f..d57792b0bc 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -4316,6 +4316,36 @@ async fn raw_generate_rejects_empty_token_ids() { assert_eq!(json["error"]["param"], "token_ids"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn raw_generate_rejects_parallel_sampling() { + let mut app = test_app().await; + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/inference/v1/generate") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "Qwen/Qwen1.5-0.5B-Chat", + "token_ids": [11, 22], + "sampling_params": {"n": 4} + }) + .to_string(), + )) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + assert_eq!(json["error"]["param"], "n"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn raw_generate_rejects_streaming_prompt_logprobs() {