[Rust Frontend] Support logprob_token_ids on OpenAI endpoints (#59549)

Signed-off-by: Wang Zupeng <zupenwang@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Zupeng Wang
2026-10-12 19:12:30 +08:00
committed by GitHub
co-authored by OpenAI Codex Bugen Zhao Claude Opus 5.5
parent ee8f7ea449
commit 21445806bd
8 changed files with 292 additions and 5 deletions
@@ -126,6 +126,13 @@ pub(super) fn prepare_chat_request(
.prompt_logprobs
.or((request.echo && !request.stream).then_some(top_logprobs));
let include_prompt_logprobs = prompt_logprobs.is_some();
let logprob_token_ids = request.logprob_token_ids.filter(|ids| !ids.is_empty());
let logprobs = (request.logprobs && logprob_token_ids.is_none()).then_some(top_logprobs);
let output_top_logprobs = if logprob_token_ids.is_some() {
-1
} else {
top_logprobs
};
let structured_outputs = convert_from_response_format(
request.response_format.as_ref(),
@@ -157,7 +164,7 @@ pub(super) fn prepare_chat_request(
max_tokens: request.max_completion_tokens,
min_tokens: request.min_tokens,
thinking_token_budget: request.thinking_token_budget,
logprobs: request.logprobs.then_some(top_logprobs),
logprobs,
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
@@ -172,7 +179,7 @@ pub(super) fn prepare_chat_request(
allowed_token_ids: request.allowed_token_ids,
bad_words: request.bad_words,
bad_words_token_ids: None,
logprob_token_ids: None,
logprob_token_ids,
structured_outputs,
skip_reading_prefix_cache: None,
vllm_xargs: merge_kv_transfer_params(
@@ -213,7 +220,7 @@ pub(super) fn prepare_chat_request(
include_usage,
include_continuous_usage,
requested_logprobs,
output_top_logprobs: top_logprobs,
output_top_logprobs,
include_prompt_logprobs,
include_reasoning,
echo,
@@ -1556,6 +1563,31 @@ mod tests {
assert!(prepared.options.is_named_tool_choice);
}
#[test]
fn selected_logprobs_override_chat_top_k_unless_empty() {
for ids in [vec![10, 20], vec![]] {
let request = serde_json::from_value(json!({
"messages": [{"role": "user", "content": "hello"}],
"logprobs": true, "top_logprobs": 1, "logprob_token_ids": ids,
}))
.unwrap();
let prepared = prepare_chat_request(
request,
&served(&["test-model"]),
ResolvedRequestContext::default(),
)
.unwrap();
let params = prepared.chat_request.sampling_params;
let selected = !ids.is_empty();
assert_eq!(params.logprob_token_ids, selected.then_some(ids));
assert_eq!(params.logprobs, (!selected).then_some(1));
assert_eq!(
prepared.options.output_top_logprobs,
if selected { -1 } else { 1 }
);
}
}
#[test]
fn prepare_chat_request_lowers_logprobs_fields() {
let request = ChatCompletionRequest {
@@ -171,6 +171,11 @@ pub struct ChatCompletionRequest {
/// Number of prompt logprobs to return
pub prompt_logprobs: Option<i32>,
/// Specific vocab token IDs to return logprobs for at each generated
/// position, in addition to the sampled token. Takes precedence over
/// `top_logprobs`. Requires `logprobs=true`.
pub logprob_token_ids: Option<Vec<u32>>,
/// Restrict output to these token IDs only
pub allowed_token_ids: Option<Vec<u32>>,
@@ -308,6 +313,7 @@ impl Default for ChatCompletionRequest {
truncate_prompt_tokens: None,
truncation_side: None,
prompt_logprobs: None,
logprob_token_ids: None,
allowed_token_ids: None,
bad_words: None,
echo: false,
@@ -35,6 +35,13 @@ pub(super) fn validate_request_compat(
);
}
if request.logprob_token_ids.as_ref().is_some_and(|ids| !ids.is_empty()) && !request.logprobs {
bail_invalid_request!(
param = "logprob_token_ids",
"when using `logprob_token_ids`, `logprobs` must be set to true."
);
}
if let Some(prompt_logprobs) = request.prompt_logprobs {
if prompt_logprobs < 0 && prompt_logprobs != -1 {
bail_invalid_request!(
@@ -100,6 +100,12 @@ pub(super) fn prepare_completion_request(
request.max_tokens
};
let echo = completion_echo_text(&request, tokenizer)?;
let logprob_token_ids = request.logprob_token_ids.filter(|ids| !ids.is_empty());
let logprobs = if logprob_token_ids.is_some() {
None
} else {
logprobs
};
let structured_outputs =
convert_from_response_format_value(&request.response_format, &request.structured_outputs)?;
@@ -137,7 +143,7 @@ pub(super) fn prepare_completion_request(
allowed_token_ids: request.allowed_token_ids,
bad_words: None,
bad_words_token_ids: None,
logprob_token_ids: None,
logprob_token_ids,
structured_outputs,
skip_reading_prefix_cache: None,
vllm_xargs: merge_kv_transfer_params(
@@ -653,6 +659,28 @@ mod tests {
);
}
#[test]
fn selected_logprobs_override_completion_top_k_unless_empty() {
for ids in [vec![10, 20], vec![]] {
let request = serde_json::from_value(json!({
"prompt": "hello", "logprobs": 0, "logprob_token_ids": ids,
}))
.unwrap();
let prepared = prepare_completion_request(
request,
&served(&["test-model"]),
ResolvedRequestContext::default(),
&test_tokenizer(),
)
.unwrap();
let params = prepared.text_request.sampling_params;
let selected = !ids.is_empty();
assert_eq!(params.logprob_token_ids, selected.then_some(ids));
assert_eq!(params.logprobs, (!selected).then_some(0));
assert_eq!(prepared.options.requested_logprobs, Some(0));
}
}
#[test]
fn prepare_completion_request_accepts_logprobs_fields() {
let request: CompletionRequest = serde_json::from_value(json!({
@@ -158,6 +158,11 @@ pub struct CompletionRequest {
/// Number of prompt logprobs to return
pub prompt_logprobs: Option<i32>,
/// Specific vocab token IDs to return logprobs for at each generated
/// position, in addition to the sampled token. Takes precedence over
/// the top-k selected by `logprobs`. Requires `logprobs` to be set.
pub logprob_token_ids: Option<Vec<u32>>,
// -------- Extra vLLM Parameters --------
/// Whether to add special tokens (e.g. BOS) to the prompt
#[serde(default = "default_true")]
@@ -67,6 +67,22 @@ pub(super) fn validate_request_compat(
}
}
if request.logprob_token_ids.as_ref().is_some_and(|ids| !ids.is_empty()) {
if request.logprobs.is_none() {
bail_invalid_request!(
param = "logprob_token_ids",
"when using `logprob_token_ids`, `logprobs` must be set."
);
}
if request.echo && request.max_tokens == Some(0) {
bail_invalid_request!(
param = "logprob_token_ids",
"`logprob_token_ids` is not supported when `echo=True` and \
`max_tokens=0` because no output tokens are generated."
);
}
}
if request.use_beam_search {
bail_invalid_request!(
param = "use_beam_search",
@@ -284,7 +284,9 @@ fn chat_top_logprob_entries(
usize::try_from(top_logprobs).unwrap_or(0)
};
position.entries.iter().take(limit)
// The engine row is the sampled token followed by the top-k or requested
// tokens, so the sampled token may repeat. Keep each token's first entry.
position.entries.iter().unique_by(|entry| entry.token_id).take(limit)
}
fn token_bytes(token: &str) -> Vec<u8> {
@@ -370,6 +372,26 @@ mod tests {
assert_eq!(chat_top_logprobs_len(-1), 3);
}
#[test]
fn chat_logprobs_dedupes_repeated_tokens_before_top_logprobs_cut() {
// Sampled `A` is also the top-1 token: the engine row is `[A, A, B, C]`.
let mut logprobs = sample_logprobs();
let sampled = logprobs.positions[0].entries[0].clone();
logprobs.positions[0].entries.insert(0, sampled);
let top_tokens = |top_logprobs| {
let chat_logprobs =
decoded_logprobs_to_openai_chat(&logprobs, top_logprobs, false).unwrap();
chat_logprobs.content.unwrap()[0]
.top_logprobs
.iter()
.map(|entry| entry.token.clone())
.collect::<Vec<_>>()
};
assert_eq!(top_tokens(2), ["A", "B"]);
assert_eq!(top_tokens(-1), ["A", "B", "C"]);
}
#[test]
fn chat_logprobs_clamps_nan_and_negative_infinity_for_json() {
let mut logprobs = sample_logprobs();
+171
View File
@@ -3012,6 +3012,177 @@ async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn selected_logprobs_invalid_requests_return_client_errors() {
let (mut app, engine_task) = test_app_with_engine_handle().await;
for chat_route in [true, false] {
let base = if chat_route {
json!({"messages": [{"role": "user", "content": "hello"}], "logprobs": true})
} else {
json!({"prompt": "hello", "logprobs": 0})
};
for invalid in [
json!({"logprob_token_ids": [-1]}),
json!({"logprob_token_ids": vec![1; 129]}),
if chat_route {
json!({"logprob_token_ids": [1], "logprobs": false})
} else {
json!({"logprob_token_ids": [1], "logprobs": null})
},
json!({"logprob_token_ids": [1], "use_beam_search": true}),
] {
let mut body = base.clone();
body.as_object_mut().unwrap().extend(invalid.as_object().unwrap().clone());
let path = if chat_route {
"/v1/chat/completions"
} else {
"/v1/completions"
};
let (status, error) = post_json(&mut app, path, body.clone()).await;
assert_eq!(status, StatusCode::BAD_REQUEST, "{body}: {error}");
}
}
let (status, error) = post_json(
&mut app,
"/v1/completions",
json!({"prompt": "hello", "logprobs": 0,
"logprob_token_ids": [1], "echo": true, "max_tokens": 0}),
)
.await;
assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");
engine_task.abort_and_join().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn selected_logprobs_reach_engine_and_both_openai_response_modes() {
for chat_route in [true, false] {
for stream in [true, false] {
let ipc = IpcNamespace::new().unwrap();
let address = ipc.handshake_endpoint();
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
address.clone(),
b"selected-logprobs".to_vec(),
|dealer, push| {
boxed_test_future(async move {
let add = recv_engine_message(dealer).await;
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
let params = request.sampling_params.as_ref().unwrap();
assert_eq!(params.logprob_token_ids, Some(vec![65, 104, 66]));
assert_eq!(params.logprobs, None);
send_outputs(
push,
RequestBatchOutputs {
outputs: vec![request_output_with_logprobs(
&request.request_id,
vec![104],
Some(EngineCoreFinishReason::Stop),
None,
Some(Logprobs {
positions: vec![PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: 104,
logprob: -0.5,
rank: 1,
},
TokenLogprob {
token_id: 65,
logprob: -4.0,
rank: 20,
},
TokenLogprob {
token_id: 104,
logprob: -0.5,
rank: 1,
},
TokenLogprob {
token_id: 66,
logprob: -5.0,
rank: 30,
},
],
}],
}),
None,
)],
..Default::default()
}
.into(),
)
.await;
})
},
));
let client = EngineCoreClient::connect(
EngineCoreClientConfig::new_single(address).with_local_input_output_addresses(
Some(ipc.input_endpoint()),
Some(ipc.output_endpoint()),
),
)
.await
.unwrap();
let chat =
ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
let mut app = build_router(Arc::new(AppState::new(vec!["test-model".into()], chat)));
let mut body = if chat_route {
json!({"messages": [{"role": "user", "content": "hello"}], "logprobs": true, "top_logprobs": 1})
} else {
json!({"prompt": "hello", "logprobs": 0})
};
body["stream"] = json!(stream);
// The sampled token `h` (104) is also requested.
body["logprob_token_ids"] = json!([65, 104, 66]);
let response = app
.call(
Request::builder()
.method("POST")
.uri(if chat_route {
"/v1/chat/completions"
} else {
"/v1/completions"
})
.header("content-type", "application/json")
.body(Body::from(body.to_string()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
engine_task.await.unwrap();
let payloads = if stream {
sse_json_payloads(std::str::from_utf8(&bytes).unwrap())
} else {
vec![serde_json::from_slice(&bytes).unwrap()]
};
let logprobs = payloads
.iter()
.map(|payload| &payload["choices"][0]["logprobs"])
.find(|value| value.is_object())
.unwrap();
if chat_route {
let actual = logprobs["content"][0]["top_logprobs"]
.as_array()
.unwrap()
.iter()
.map(|entry| json!([entry["token"], entry["logprob"]]))
.collect::<Vec<_>>();
assert_eq!(
actual,
[json!(["h", -0.5]), json!(["A", -4.0]), json!(["B", -5.0])]
);
} else {
assert_eq!(
logprobs["top_logprobs"][0],
json!({"h": -0.5, "A": -4.0, "B": -5.0})
);
}
}
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {