[Bugfix][Rust Frontend] Reject n > 1 in the /inference/v1/generate route (#52844)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Quentin Gallouédec <quentin.gallouedec@huggingface.co> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
d66300a1ba
commit
16cfe728d8
@@ -26,11 +26,11 @@ use vllm_llm::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||||
pub(crate) use self::types::GenerateRequest;
|
|
||||||
use self::types::{
|
use self::types::{
|
||||||
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
|
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
|
||||||
GenerateStreamResponse,
|
GenerateStreamResponse,
|
||||||
};
|
};
|
||||||
|
pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams};
|
||||||
pub(crate) use self::validate::validate_request_compat;
|
pub(crate) use self::validate::validate_request_compat;
|
||||||
use crate::config::ApiServerOptions;
|
use crate::config::ApiServerOptions;
|
||||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||||
|
|||||||
@@ -54,9 +54,9 @@ pub(super) fn prepare_generate_request(
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|options| options.continuous_usage_stats)
|
.and_then(|options| options.continuous_usage_stats)
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
let include_logprobs = request.sampling_params.logprobs.is_some();
|
let include_logprobs = request.sampling_params.inner.logprobs.is_some();
|
||||||
let include_prompt_logprobs = request.sampling_params.prompt_logprobs.is_some();
|
let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some();
|
||||||
let mut sampling_params = request.sampling_params;
|
let mut sampling_params = request.sampling_params.inner;
|
||||||
sampling_params.vllm_xargs = merge_kv_transfer_params(
|
sampling_params.vllm_xargs = merge_kv_transfer_params(
|
||||||
sampling_params.vllm_xargs,
|
sampling_params.vllm_xargs,
|
||||||
request.kv_transfer_params.as_ref(),
|
request.kv_transfer_params.as_ref(),
|
||||||
|
|||||||
@@ -11,6 +11,23 @@ use vllm_text::SamplingParams;
|
|||||||
|
|
||||||
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage};
|
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage};
|
||||||
|
|
||||||
|
/// Sampling parameters for the token-in/token-out generate API.
|
||||||
|
///
|
||||||
|
/// Wraps [`SamplingParams`] to additionally capture `n`, which the shared
|
||||||
|
/// northbound type intentionally omits (parallel sampling is handled by
|
||||||
|
/// higher layers, and the Rust frontend does not implement it). Capturing it
|
||||||
|
/// here lets validation reject `n > 1` explicitly instead of silently
|
||||||
|
/// dropping the key and returning a single choice.
|
||||||
|
#[serde_with::skip_serializing_none]
|
||||||
|
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||||
|
pub struct GenerateSamplingParams {
|
||||||
|
/// Number of output sequences to generate. Only `1` is supported.
|
||||||
|
pub n: Option<u32>,
|
||||||
|
/// The supported sampling parameters, lowered to the engine.
|
||||||
|
#[serde(flatten)]
|
||||||
|
pub inner: SamplingParams,
|
||||||
|
}
|
||||||
|
|
||||||
/// vLLM-compatible request type for the token-in/token-out generate API.
|
/// vLLM-compatible request type for the token-in/token-out generate API.
|
||||||
#[serde_with::skip_serializing_none]
|
#[serde_with::skip_serializing_none]
|
||||||
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
||||||
@@ -18,7 +35,7 @@ pub struct GenerateRequest {
|
|||||||
pub request_id: Option<String>,
|
pub request_id: Option<String>,
|
||||||
pub model: Option<String>,
|
pub model: Option<String>,
|
||||||
pub token_ids: Vec<u32>,
|
pub token_ids: Vec<u32>,
|
||||||
pub sampling_params: SamplingParams,
|
pub sampling_params: GenerateSamplingParams,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub stream: bool,
|
pub stream: bool,
|
||||||
pub stream_options: Option<StreamOptions>,
|
pub stream_options: Option<StreamOptions>,
|
||||||
|
|||||||
@@ -23,6 +23,10 @@ pub(crate) fn validate_request_compat(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if request.sampling_params.n.unwrap_or(1) != 1 {
|
||||||
|
bail_invalid_request!(param = "n", "Only n=1 is supported.");
|
||||||
|
}
|
||||||
|
|
||||||
if request.token_ids.is_empty() {
|
if request.token_ids.is_empty() {
|
||||||
bail_invalid_request!(
|
bail_invalid_request!(
|
||||||
param = "token_ids",
|
param = "token_ids",
|
||||||
@@ -30,14 +34,14 @@ pub(crate) fn validate_request_compat(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if request.sampling_params.max_tokens == Some(0) {
|
if request.sampling_params.inner.max_tokens == Some(0) {
|
||||||
bail_invalid_request!(
|
bail_invalid_request!(
|
||||||
param = "sampling_params",
|
param = "sampling_params",
|
||||||
"max_tokens must be greater than 0."
|
"max_tokens must be greater than 0."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs {
|
if let Some(prompt_logprobs) = request.sampling_params.inner.prompt_logprobs {
|
||||||
if prompt_logprobs < 0 && prompt_logprobs != -1 {
|
if prompt_logprobs < 0 && prompt_logprobs != -1 {
|
||||||
bail_invalid_request!(
|
bail_invalid_request!(
|
||||||
param = "sampling_params",
|
param = "sampling_params",
|
||||||
@@ -98,6 +102,28 @@ mod tests {
|
|||||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_request_compat_rejects_parallel_sampling() {
|
||||||
|
let request: GenerateRequest = serde_json::from_value(json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"token_ids": [11, 22],
|
||||||
|
"sampling_params": {"n": 4}
|
||||||
|
}))
|
||||||
|
.expect("parse request");
|
||||||
|
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_request_compat_accepts_explicit_n_one() {
|
||||||
|
let request: GenerateRequest = serde_json::from_value(json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"token_ids": [11, 22],
|
||||||
|
"sampling_params": {"n": 1}
|
||||||
|
}))
|
||||||
|
.expect("parse request");
|
||||||
|
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn validate_request_compat_rejects_empty_token_ids() {
|
fn validate_request_compat_rejects_empty_token_ids() {
|
||||||
let request = GenerateRequest {
|
let request = GenerateRequest {
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ use crate::error::{ApiError, text_submit_error};
|
|||||||
use crate::lora::LoraModelResolution;
|
use crate::lora::LoraModelResolution;
|
||||||
use crate::render::RenderState;
|
use crate::render::RenderState;
|
||||||
use crate::routes::inference::generate::{
|
use crate::routes::inference::generate::{
|
||||||
GenerateRequest, validate_request_compat as validate_generate_request,
|
GenerateRequest, GenerateSamplingParams, validate_request_compat as validate_generate_request,
|
||||||
};
|
};
|
||||||
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
|
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
|
||||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||||
@@ -86,7 +86,10 @@ fn lower_render_request(
|
|||||||
request_id: Some(text_request.request_id),
|
request_id: Some(text_request.request_id),
|
||||||
model: Some(model),
|
model: Some(model),
|
||||||
token_ids,
|
token_ids,
|
||||||
sampling_params: text_request.sampling_params,
|
sampling_params: GenerateSamplingParams {
|
||||||
|
n: None,
|
||||||
|
inner: text_request.sampling_params,
|
||||||
|
},
|
||||||
stream,
|
stream,
|
||||||
stream_options,
|
stream_options,
|
||||||
cache_salt: text_request.cache_salt,
|
cache_salt: text_request.cache_salt,
|
||||||
|
|||||||
@@ -4316,6 +4316,36 @@ async fn raw_generate_rejects_empty_token_ids() {
|
|||||||
assert_eq!(json["error"]["param"], "token_ids");
|
assert_eq!(json["error"]["param"], "token_ids");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn raw_generate_rejects_parallel_sampling() {
|
||||||
|
let mut app = test_app().await;
|
||||||
|
|
||||||
|
let response = app
|
||||||
|
.call(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/inference/v1/generate")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from(
|
||||||
|
json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"token_ids": [11, 22],
|
||||||
|
"sampling_params": {"n": 4}
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
))
|
||||||
|
.expect("build request"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("call app");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||||
|
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||||
|
assert_eq!(json["error"]["param"], "n");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn raw_generate_rejects_streaming_prompt_logprobs() {
|
async fn raw_generate_rejects_streaming_prompt_logprobs() {
|
||||||
|
|||||||
Reference in New Issue
Block a user