[Bugfix][Rust Frontend] Reject prompt_logprobs for streaming generate (#46839)
Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
@@ -34,14 +34,20 @@ pub(super) fn validate_request_compat(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs
|
if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs {
|
||||||
&& prompt_logprobs < 0
|
if prompt_logprobs < 0 && prompt_logprobs != -1 {
|
||||||
&& prompt_logprobs != -1
|
bail_invalid_request!(
|
||||||
{
|
param = "sampling_params",
|
||||||
bail_invalid_request!(
|
"`prompt_logprobs` must be a non-negative value or -1."
|
||||||
param = "sampling_params",
|
);
|
||||||
"`prompt_logprobs` must be a non-negative value or -1."
|
}
|
||||||
);
|
|
||||||
|
if request.stream {
|
||||||
|
bail_invalid_request!(
|
||||||
|
param = "sampling_params",
|
||||||
|
"`prompt_logprobs` are not available when `stream=true`."
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -97,4 +103,54 @@ mod tests {
|
|||||||
};
|
};
|
||||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_request_compat_rejects_streaming_prompt_logprobs() {
|
||||||
|
let request: GenerateRequest = serde_json::from_value(json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"token_ids": [11, 22],
|
||||||
|
"stream": true,
|
||||||
|
"sampling_params": {
|
||||||
|
"prompt_logprobs": 0
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.expect("parse request");
|
||||||
|
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||||
|
|
||||||
|
let request: GenerateRequest = serde_json::from_value(json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"token_ids": [11, 22],
|
||||||
|
"stream": true,
|
||||||
|
"sampling_params": {
|
||||||
|
"prompt_logprobs": 1
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.expect("parse request");
|
||||||
|
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||||
|
|
||||||
|
let request: GenerateRequest = serde_json::from_value(json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"token_ids": [11, 22],
|
||||||
|
"stream": true,
|
||||||
|
"sampling_params": {
|
||||||
|
"prompt_logprobs": -1
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.expect("parse request");
|
||||||
|
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
|
||||||
|
let request: GenerateRequest = serde_json::from_value(json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"token_ids": [11, 22],
|
||||||
|
"stream": false,
|
||||||
|
"sampling_params": {
|
||||||
|
"prompt_logprobs": 1
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.expect("parse request");
|
||||||
|
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4129,6 +4129,45 @@ async fn raw_generate_rejects_empty_token_ids() {
|
|||||||
assert_eq!(json["error"]["param"], "token_ids");
|
assert_eq!(json["error"]["param"], "token_ids");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
#[serial]
|
||||||
|
async fn raw_generate_rejects_streaming_prompt_logprobs() {
|
||||||
|
let mut app = test_app().await;
|
||||||
|
|
||||||
|
for prompt_logprobs in [0, 1] {
|
||||||
|
let response = app
|
||||||
|
.call(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/inference/v1/generate")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from(
|
||||||
|
json!({
|
||||||
|
"model": "Qwen/Qwen1.5-0.5B-Chat",
|
||||||
|
"token_ids": [11, 22],
|
||||||
|
"stream": true,
|
||||||
|
"sampling_params": {
|
||||||
|
"prompt_logprobs": prompt_logprobs
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.to_string(),
|
||||||
|
))
|
||||||
|
.expect("build request"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("call app");
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||||
|
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||||
|
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||||
|
assert_eq!(json["error"]["param"], "sampling_params");
|
||||||
|
assert_eq!(
|
||||||
|
json["error"]["message"],
|
||||||
|
"`prompt_logprobs` are not available when `stream=true`."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn raw_generate_rejects_wrong_model() {
|
async fn raw_generate_rejects_wrong_model() {
|
||||||
|
|||||||
Reference in New Issue
Block a user