iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Bugfix][Rust Frontend] Reject prompt_logprobs for streaming generate (#46839)

Browse Source 
Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
Reid
2026-06-30 05:10:07 +00:00
committed by GitHub
parent a16dbd5b85
commit 930f8dc0a1
2 changed files with 103 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/server/src/routes/inference/generate/validate.rs
+64 -8
View File
@@ -34,14 +34,20 @@ pub(super) fn validate_request_compat(
);
}
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs
&& prompt_logprobs < 0
&& prompt_logprobs != -1
{
bail_invalid_request!(
param = "sampling_params",
"`prompt_logprobs` must be a non-negative value or -1."
);
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs {
if prompt_logprobs < 0 && prompt_logprobs != -1 {
bail_invalid_request!(
param = "sampling_params",
"`prompt_logprobs` must be a non-negative value or -1."
);
}
if request.stream {
bail_invalid_request!(
param = "sampling_params",
"`prompt_logprobs` are not available when `stream=true`."
);
}
}
Ok(())
@@ -97,4 +103,54 @@ mod tests {
};
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_streaming_prompt_logprobs() {
let request: GenerateRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"stream": true,
"sampling_params": {
"prompt_logprobs": 0
}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
let request: GenerateRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"stream": true,
"sampling_params": {
"prompt_logprobs": 1
}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
let request: GenerateRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"stream": true,
"sampling_params": {
"prompt_logprobs": -1
}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
let request: GenerateRequest = serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"stream": false,
"sampling_params": {
"prompt_logprobs": 1
}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok());
}
}
rust/src/server/src/routes/tests.rs
+39
View File
@@ -4129,6 +4129,45 @@ async fn raw_generate_rejects_empty_token_ids() {
assert_eq!(json["error"]["param"], "token_ids");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn raw_generate_rejects_streaming_prompt_logprobs() {
let mut app = test_app().await;
for prompt_logprobs in [0, 1] {
let response = app
.call(
Request::builder()
.method("POST")
.uri("/inference/v1/generate")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"token_ids": [11, 22],
"stream": true,
"sampling_params": {
"prompt_logprobs": prompt_logprobs
}
})
.to_string(),
))
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json["error"]["param"], "sampling_params");
assert_eq!(
json["error"]["message"],
"`prompt_logprobs` are not available when `stream=true`."
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn raw_generate_rejects_wrong_model() {