[Bugfix][Rust Frontend] Reject n > 1 in the /inference/v1/generate route (#52844)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Quentin Gallouédec <quentin.gallouedec@huggingface.co> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
d66300a1ba
commit
16cfe728d8
@@ -26,11 +26,11 @@ use vllm_llm::{
|
||||
};
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||
pub(crate) use self::types::GenerateRequest;
|
||||
use self::types::{
|
||||
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
};
|
||||
pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams};
|
||||
pub(crate) use self::validate::validate_request_compat;
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
|
||||
@@ -54,9 +54,9 @@ pub(super) fn prepare_generate_request(
|
||||
.as_ref()
|
||||
.and_then(|options| options.continuous_usage_stats)
|
||||
.unwrap_or(false);
|
||||
let include_logprobs = request.sampling_params.logprobs.is_some();
|
||||
let include_prompt_logprobs = request.sampling_params.prompt_logprobs.is_some();
|
||||
let mut sampling_params = request.sampling_params;
|
||||
let include_logprobs = request.sampling_params.inner.logprobs.is_some();
|
||||
let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some();
|
||||
let mut sampling_params = request.sampling_params.inner;
|
||||
sampling_params.vllm_xargs = merge_kv_transfer_params(
|
||||
sampling_params.vllm_xargs,
|
||||
request.kv_transfer_params.as_ref(),
|
||||
|
||||
@@ -11,6 +11,23 @@ use vllm_text::SamplingParams;
|
||||
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage};
|
||||
|
||||
/// Sampling parameters for the token-in/token-out generate API.
|
||||
///
|
||||
/// Wraps [`SamplingParams`] to additionally capture `n`, which the shared
|
||||
/// northbound type intentionally omits (parallel sampling is handled by
|
||||
/// higher layers, and the Rust frontend does not implement it). Capturing it
|
||||
/// here lets validation reject `n > 1` explicitly instead of silently
|
||||
/// dropping the key and returning a single choice.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct GenerateSamplingParams {
|
||||
/// Number of output sequences to generate. Only `1` is supported.
|
||||
pub n: Option<u32>,
|
||||
/// The supported sampling parameters, lowered to the engine.
|
||||
#[serde(flatten)]
|
||||
pub inner: SamplingParams,
|
||||
}
|
||||
|
||||
/// vLLM-compatible request type for the token-in/token-out generate API.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
||||
@@ -18,7 +35,7 @@ pub struct GenerateRequest {
|
||||
pub request_id: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_params: SamplingParams,
|
||||
pub sampling_params: GenerateSamplingParams,
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
@@ -23,6 +23,10 @@ pub(crate) fn validate_request_compat(
|
||||
);
|
||||
}
|
||||
|
||||
if request.sampling_params.n.unwrap_or(1) != 1 {
|
||||
bail_invalid_request!(param = "n", "Only n=1 is supported.");
|
||||
}
|
||||
|
||||
if request.token_ids.is_empty() {
|
||||
bail_invalid_request!(
|
||||
param = "token_ids",
|
||||
@@ -30,14 +34,14 @@ pub(crate) fn validate_request_compat(
|
||||
);
|
||||
}
|
||||
|
||||
if request.sampling_params.max_tokens == Some(0) {
|
||||
if request.sampling_params.inner.max_tokens == Some(0) {
|
||||
bail_invalid_request!(
|
||||
param = "sampling_params",
|
||||
"max_tokens must be greater than 0."
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs {
|
||||
if let Some(prompt_logprobs) = request.sampling_params.inner.prompt_logprobs {
|
||||
if prompt_logprobs < 0 && prompt_logprobs != -1 {
|
||||
bail_invalid_request!(
|
||||
param = "sampling_params",
|
||||
@@ -98,6 +102,28 @@ mod tests {
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_parallel_sampling() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"token_ids": [11, 22],
|
||||
"sampling_params": {"n": 4}
|
||||
}))
|
||||
.expect("parse request");
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_explicit_n_one() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"token_ids": [11, 22],
|
||||
"sampling_params": {"n": 1}
|
||||
}))
|
||||
.expect("parse request");
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_empty_token_ids() {
|
||||
let request = GenerateRequest {
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::error::{ApiError, text_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
use crate::render::RenderState;
|
||||
use crate::routes::inference::generate::{
|
||||
GenerateRequest, validate_request_compat as validate_generate_request,
|
||||
GenerateRequest, GenerateSamplingParams, validate_request_compat as validate_generate_request,
|
||||
};
|
||||
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
@@ -86,7 +86,10 @@ fn lower_render_request(
|
||||
request_id: Some(text_request.request_id),
|
||||
model: Some(model),
|
||||
token_ids,
|
||||
sampling_params: text_request.sampling_params,
|
||||
sampling_params: GenerateSamplingParams {
|
||||
n: None,
|
||||
inner: text_request.sampling_params,
|
||||
},
|
||||
stream,
|
||||
stream_options,
|
||||
cache_salt: text_request.cache_salt,
|
||||
|
||||
@@ -4316,6 +4316,36 @@ async fn raw_generate_rejects_empty_token_ids() {
|
||||
assert_eq!(json["error"]["param"], "token_ids");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn raw_generate_rejects_parallel_sampling() {
|
||||
let mut app = test_app().await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||
"token_ids": [11, 22],
|
||||
"sampling_params": {"n": 4}
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json["error"]["param"], "n");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn raw_generate_rejects_streaming_prompt_logprobs() {
|
||||
|
||||
Reference in New Issue
Block a user