[Bugfix][Rust Frontend] Reject n > 1 in the /inference/v1/generate route (#52844)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Quentin Gallouédec <quentin.gallouedec@huggingface.co>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Quentin Gallouédec
2026-08-20 07:41:06 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent d66300a1ba
commit 16cfe728d8
6 changed files with 85 additions and 9 deletions
@@ -26,11 +26,11 @@ use vllm_llm::{
};
use self::convert::{ResponseOptions, prepare_generate_request};
pub(crate) use self::types::GenerateRequest;
use self::types::{
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
GenerateStreamResponse,
};
pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams};
pub(crate) use self::validate::validate_request_compat;
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
@@ -54,9 +54,9 @@ pub(super) fn prepare_generate_request(
.as_ref()
.and_then(|options| options.continuous_usage_stats)
.unwrap_or(false);
let include_logprobs = request.sampling_params.logprobs.is_some();
let include_prompt_logprobs = request.sampling_params.prompt_logprobs.is_some();
let mut sampling_params = request.sampling_params;
let include_logprobs = request.sampling_params.inner.logprobs.is_some();
let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some();
let mut sampling_params = request.sampling_params.inner;
sampling_params.vllm_xargs = merge_kv_transfer_params(
sampling_params.vllm_xargs,
request.kv_transfer_params.as_ref(),
@@ -11,6 +11,23 @@ use vllm_text::SamplingParams;
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage};
/// Sampling parameters for the token-in/token-out generate API.
///
/// Wraps [`SamplingParams`] to additionally capture `n`, which the shared
/// northbound type intentionally omits (parallel sampling is handled by
/// higher layers, and the Rust frontend does not implement it). Capturing it
/// here lets validation reject `n > 1` explicitly instead of silently
/// dropping the key and returning a single choice.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct GenerateSamplingParams {
/// Number of output sequences to generate. Only `1` is supported.
pub n: Option<u32>,
/// The supported sampling parameters, lowered to the engine.
#[serde(flatten)]
pub inner: SamplingParams,
}
/// vLLM-compatible request type for the token-in/token-out generate API.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
@@ -18,7 +35,7 @@ pub struct GenerateRequest {
pub request_id: Option<String>,
pub model: Option<String>,
pub token_ids: Vec<u32>,
pub sampling_params: SamplingParams,
pub sampling_params: GenerateSamplingParams,
#[serde(default)]
pub stream: bool,
pub stream_options: Option<StreamOptions>,
@@ -23,6 +23,10 @@ pub(crate) fn validate_request_compat(
);
}
if request.sampling_params.n.unwrap_or(1) != 1 {
bail_invalid_request!(param = "n", "Only n=1 is supported.");
}
if request.token_ids.is_empty() {
bail_invalid_request!(
param = "token_ids",
@@ -30,14 +34,14 @@ pub(crate) fn validate_request_compat(
);
}
if request.sampling_params.max_tokens == Some(0) {
if request.sampling_params.inner.max_tokens == Some(0) {
bail_invalid_request!(
param = "sampling_params",
"max_tokens must be greater than 0."
);
}
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs {
if let Some(prompt_logprobs) = request.sampling_params.inner.prompt_logprobs {
if prompt_logprobs < 0 && prompt_logprobs != -1 {
bail_invalid_request!(
param = "sampling_params",
@@ -98,6 +102,28 @@ mod tests {
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_parallel_sampling() {
let request: GenerateRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"sampling_params": {"n": 4}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_accepts_explicit_n_one() {
let request: GenerateRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"sampling_params": {"n": 1}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok());
}
#[test]
fn validate_request_compat_rejects_empty_token_ids() {
let request = GenerateRequest {
+5 -2
View File
@@ -17,7 +17,7 @@ use crate::error::{ApiError, text_submit_error};
use crate::lora::LoraModelResolution;
use crate::render::RenderState;
use crate::routes::inference::generate::{
GenerateRequest, validate_request_compat as validate_generate_request,
GenerateRequest, GenerateSamplingParams, validate_request_compat as validate_generate_request,
};
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
use crate::routes::openai::utils::validated_json::ValidatedJson;
@@ -86,7 +86,10 @@ fn lower_render_request(
request_id: Some(text_request.request_id),
model: Some(model),
token_ids,
sampling_params: text_request.sampling_params,
sampling_params: GenerateSamplingParams {
n: None,
inner: text_request.sampling_params,
},
stream,
stream_options,
cache_salt: text_request.cache_salt,
+30
View File
@@ -4316,6 +4316,36 @@ async fn raw_generate_rejects_empty_token_ids() {
assert_eq!(json["error"]["param"], "token_ids");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn raw_generate_rejects_parallel_sampling() {
let mut app = test_app().await;
let response = app
.call(
Request::builder()
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"sampling_params": {"n": 4}
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["error"]["param"], "n");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn raw_generate_rejects_streaming_prompt_logprobs() {