[Rust Frontend] Support logprob_token_ids on OpenAI endpoints (#59549)
Signed-off-by: Wang Zupeng <zupenwang@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI Codex
Bugen Zhao
Claude Opus 5.5
parent
ee8f7ea449
commit
21445806bd
@@ -126,6 +126,13 @@ pub(super) fn prepare_chat_request(
|
||||
.prompt_logprobs
|
||||
.or((request.echo && !request.stream).then_some(top_logprobs));
|
||||
let include_prompt_logprobs = prompt_logprobs.is_some();
|
||||
let logprob_token_ids = request.logprob_token_ids.filter(|ids| !ids.is_empty());
|
||||
let logprobs = (request.logprobs && logprob_token_ids.is_none()).then_some(top_logprobs);
|
||||
let output_top_logprobs = if logprob_token_ids.is_some() {
|
||||
-1
|
||||
} else {
|
||||
top_logprobs
|
||||
};
|
||||
|
||||
let structured_outputs = convert_from_response_format(
|
||||
request.response_format.as_ref(),
|
||||
@@ -157,7 +164,7 @@ pub(super) fn prepare_chat_request(
|
||||
max_tokens: request.max_completion_tokens,
|
||||
min_tokens: request.min_tokens,
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs: request.logprobs.then_some(top_logprobs),
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
@@ -172,7 +179,7 @@ pub(super) fn prepare_chat_request(
|
||||
allowed_token_ids: request.allowed_token_ids,
|
||||
bad_words: request.bad_words,
|
||||
bad_words_token_ids: None,
|
||||
logprob_token_ids: None,
|
||||
logprob_token_ids,
|
||||
structured_outputs,
|
||||
skip_reading_prefix_cache: None,
|
||||
vllm_xargs: merge_kv_transfer_params(
|
||||
@@ -213,7 +220,7 @@ pub(super) fn prepare_chat_request(
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
requested_logprobs,
|
||||
output_top_logprobs: top_logprobs,
|
||||
output_top_logprobs,
|
||||
include_prompt_logprobs,
|
||||
include_reasoning,
|
||||
echo,
|
||||
@@ -1556,6 +1563,31 @@ mod tests {
|
||||
assert!(prepared.options.is_named_tool_choice);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selected_logprobs_override_chat_top_k_unless_empty() {
|
||||
for ids in [vec![10, 20], vec![]] {
|
||||
let request = serde_json::from_value(json!({
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"logprobs": true, "top_logprobs": 1, "logprob_token_ids": ids,
|
||||
}))
|
||||
.unwrap();
|
||||
let prepared = prepare_chat_request(
|
||||
request,
|
||||
&served(&["test-model"]),
|
||||
ResolvedRequestContext::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let params = prepared.chat_request.sampling_params;
|
||||
let selected = !ids.is_empty();
|
||||
assert_eq!(params.logprob_token_ids, selected.then_some(ids));
|
||||
assert_eq!(params.logprobs, (!selected).then_some(1));
|
||||
assert_eq!(
|
||||
prepared.options.output_top_logprobs,
|
||||
if selected { -1 } else { 1 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_chat_request_lowers_logprobs_fields() {
|
||||
let request = ChatCompletionRequest {
|
||||
|
||||
@@ -171,6 +171,11 @@ pub struct ChatCompletionRequest {
|
||||
/// Number of prompt logprobs to return
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
|
||||
/// Specific vocab token IDs to return logprobs for at each generated
|
||||
/// position, in addition to the sampled token. Takes precedence over
|
||||
/// `top_logprobs`. Requires `logprobs=true`.
|
||||
pub logprob_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Restrict output to these token IDs only
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
|
||||
@@ -308,6 +313,7 @@ impl Default for ChatCompletionRequest {
|
||||
truncate_prompt_tokens: None,
|
||||
truncation_side: None,
|
||||
prompt_logprobs: None,
|
||||
logprob_token_ids: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words: None,
|
||||
echo: false,
|
||||
|
||||
@@ -35,6 +35,13 @@ pub(super) fn validate_request_compat(
|
||||
);
|
||||
}
|
||||
|
||||
if request.logprob_token_ids.as_ref().is_some_and(|ids| !ids.is_empty()) && !request.logprobs {
|
||||
bail_invalid_request!(
|
||||
param = "logprob_token_ids",
|
||||
"when using `logprob_token_ids`, `logprobs` must be set to true."
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(prompt_logprobs) = request.prompt_logprobs {
|
||||
if prompt_logprobs < 0 && prompt_logprobs != -1 {
|
||||
bail_invalid_request!(
|
||||
|
||||
@@ -100,6 +100,12 @@ pub(super) fn prepare_completion_request(
|
||||
request.max_tokens
|
||||
};
|
||||
let echo = completion_echo_text(&request, tokenizer)?;
|
||||
let logprob_token_ids = request.logprob_token_ids.filter(|ids| !ids.is_empty());
|
||||
let logprobs = if logprob_token_ids.is_some() {
|
||||
None
|
||||
} else {
|
||||
logprobs
|
||||
};
|
||||
|
||||
let structured_outputs =
|
||||
convert_from_response_format_value(&request.response_format, &request.structured_outputs)?;
|
||||
@@ -137,7 +143,7 @@ pub(super) fn prepare_completion_request(
|
||||
allowed_token_ids: request.allowed_token_ids,
|
||||
bad_words: None,
|
||||
bad_words_token_ids: None,
|
||||
logprob_token_ids: None,
|
||||
logprob_token_ids,
|
||||
structured_outputs,
|
||||
skip_reading_prefix_cache: None,
|
||||
vllm_xargs: merge_kv_transfer_params(
|
||||
@@ -653,6 +659,28 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selected_logprobs_override_completion_top_k_unless_empty() {
|
||||
for ids in [vec![10, 20], vec![]] {
|
||||
let request = serde_json::from_value(json!({
|
||||
"prompt": "hello", "logprobs": 0, "logprob_token_ids": ids,
|
||||
}))
|
||||
.unwrap();
|
||||
let prepared = prepare_completion_request(
|
||||
request,
|
||||
&served(&["test-model"]),
|
||||
ResolvedRequestContext::default(),
|
||||
&test_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
let params = prepared.text_request.sampling_params;
|
||||
let selected = !ids.is_empty();
|
||||
assert_eq!(params.logprob_token_ids, selected.then_some(ids));
|
||||
assert_eq!(params.logprobs, (!selected).then_some(0));
|
||||
assert_eq!(prepared.options.requested_logprobs, Some(0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_completion_request_accepts_logprobs_fields() {
|
||||
let request: CompletionRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -158,6 +158,11 @@ pub struct CompletionRequest {
|
||||
/// Number of prompt logprobs to return
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
|
||||
/// Specific vocab token IDs to return logprobs for at each generated
|
||||
/// position, in addition to the sampled token. Takes precedence over
|
||||
/// the top-k selected by `logprobs`. Requires `logprobs` to be set.
|
||||
pub logprob_token_ids: Option<Vec<u32>>,
|
||||
|
||||
// -------- Extra vLLM Parameters --------
|
||||
/// Whether to add special tokens (e.g. BOS) to the prompt
|
||||
#[serde(default = "default_true")]
|
||||
|
||||
@@ -67,6 +67,22 @@ pub(super) fn validate_request_compat(
|
||||
}
|
||||
}
|
||||
|
||||
if request.logprob_token_ids.as_ref().is_some_and(|ids| !ids.is_empty()) {
|
||||
if request.logprobs.is_none() {
|
||||
bail_invalid_request!(
|
||||
param = "logprob_token_ids",
|
||||
"when using `logprob_token_ids`, `logprobs` must be set."
|
||||
);
|
||||
}
|
||||
if request.echo && request.max_tokens == Some(0) {
|
||||
bail_invalid_request!(
|
||||
param = "logprob_token_ids",
|
||||
"`logprob_token_ids` is not supported when `echo=True` and \
|
||||
`max_tokens=0` because no output tokens are generated."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if request.use_beam_search {
|
||||
bail_invalid_request!(
|
||||
param = "use_beam_search",
|
||||
|
||||
@@ -284,7 +284,9 @@ fn chat_top_logprob_entries(
|
||||
usize::try_from(top_logprobs).unwrap_or(0)
|
||||
};
|
||||
|
||||
position.entries.iter().take(limit)
|
||||
// The engine row is the sampled token followed by the top-k or requested
|
||||
// tokens, so the sampled token may repeat. Keep each token's first entry.
|
||||
position.entries.iter().unique_by(|entry| entry.token_id).take(limit)
|
||||
}
|
||||
|
||||
fn token_bytes(token: &str) -> Vec<u8> {
|
||||
@@ -370,6 +372,26 @@ mod tests {
|
||||
assert_eq!(chat_top_logprobs_len(-1), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_logprobs_dedupes_repeated_tokens_before_top_logprobs_cut() {
|
||||
// Sampled `A` is also the top-1 token: the engine row is `[A, A, B, C]`.
|
||||
let mut logprobs = sample_logprobs();
|
||||
let sampled = logprobs.positions[0].entries[0].clone();
|
||||
logprobs.positions[0].entries.insert(0, sampled);
|
||||
|
||||
let top_tokens = |top_logprobs| {
|
||||
let chat_logprobs =
|
||||
decoded_logprobs_to_openai_chat(&logprobs, top_logprobs, false).unwrap();
|
||||
chat_logprobs.content.unwrap()[0]
|
||||
.top_logprobs
|
||||
.iter()
|
||||
.map(|entry| entry.token.clone())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
assert_eq!(top_tokens(2), ["A", "B"]);
|
||||
assert_eq!(top_tokens(-1), ["A", "B", "C"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_logprobs_clamps_nan_and_negative_infinity_for_json() {
|
||||
let mut logprobs = sample_logprobs();
|
||||
|
||||
@@ -3012,6 +3012,177 @@ async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn selected_logprobs_invalid_requests_return_client_errors() {
|
||||
let (mut app, engine_task) = test_app_with_engine_handle().await;
|
||||
for chat_route in [true, false] {
|
||||
let base = if chat_route {
|
||||
json!({"messages": [{"role": "user", "content": "hello"}], "logprobs": true})
|
||||
} else {
|
||||
json!({"prompt": "hello", "logprobs": 0})
|
||||
};
|
||||
for invalid in [
|
||||
json!({"logprob_token_ids": [-1]}),
|
||||
json!({"logprob_token_ids": vec![1; 129]}),
|
||||
if chat_route {
|
||||
json!({"logprob_token_ids": [1], "logprobs": false})
|
||||
} else {
|
||||
json!({"logprob_token_ids": [1], "logprobs": null})
|
||||
},
|
||||
json!({"logprob_token_ids": [1], "use_beam_search": true}),
|
||||
] {
|
||||
let mut body = base.clone();
|
||||
body.as_object_mut().unwrap().extend(invalid.as_object().unwrap().clone());
|
||||
let path = if chat_route {
|
||||
"/v1/chat/completions"
|
||||
} else {
|
||||
"/v1/completions"
|
||||
};
|
||||
let (status, error) = post_json(&mut app, path, body.clone()).await;
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST, "{body}: {error}");
|
||||
}
|
||||
}
|
||||
let (status, error) = post_json(
|
||||
&mut app,
|
||||
"/v1/completions",
|
||||
json!({"prompt": "hello", "logprobs": 0,
|
||||
"logprob_token_ids": [1], "echo": true, "max_tokens": 0}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST, "{error}");
|
||||
engine_task.abort_and_join().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn selected_logprobs_reach_engine_and_both_openai_response_modes() {
|
||||
for chat_route in [true, false] {
|
||||
for stream in [true, false] {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let address = ipc.handshake_endpoint();
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task(
|
||||
address.clone(),
|
||||
b"selected-logprobs".to_vec(),
|
||||
|dealer, push| {
|
||||
boxed_test_future(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
let params = request.sampling_params.as_ref().unwrap();
|
||||
assert_eq!(params.logprob_token_ids, Some(vec![65, 104, 66]));
|
||||
assert_eq!(params.logprobs, None);
|
||||
send_outputs(
|
||||
push,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
vec![104],
|
||||
Some(EngineCoreFinishReason::Stop),
|
||||
None,
|
||||
Some(Logprobs {
|
||||
positions: vec![PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 104,
|
||||
logprob: -0.5,
|
||||
rank: 1,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 65,
|
||||
logprob: -4.0,
|
||||
rank: 20,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 104,
|
||||
logprob: -0.5,
|
||||
rank: 1,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 66,
|
||||
logprob: -5.0,
|
||||
rank: 30,
|
||||
},
|
||||
],
|
||||
}],
|
||||
}),
|
||||
None,
|
||||
)],
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
));
|
||||
let client = EngineCoreClient::connect(
|
||||
EngineCoreClientConfig::new_single(address).with_local_input_output_addresses(
|
||||
Some(ipc.input_endpoint()),
|
||||
Some(ipc.output_endpoint()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let chat =
|
||||
ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
|
||||
let mut app = build_router(Arc::new(AppState::new(vec!["test-model".into()], chat)));
|
||||
let mut body = if chat_route {
|
||||
json!({"messages": [{"role": "user", "content": "hello"}], "logprobs": true, "top_logprobs": 1})
|
||||
} else {
|
||||
json!({"prompt": "hello", "logprobs": 0})
|
||||
};
|
||||
body["stream"] = json!(stream);
|
||||
// The sampled token `h` (104) is also requested.
|
||||
body["logprob_token_ids"] = json!([65, 104, 66]);
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri(if chat_route {
|
||||
"/v1/chat/completions"
|
||||
} else {
|
||||
"/v1/completions"
|
||||
})
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(body.to_string()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
engine_task.await.unwrap();
|
||||
let payloads = if stream {
|
||||
sse_json_payloads(std::str::from_utf8(&bytes).unwrap())
|
||||
} else {
|
||||
vec![serde_json::from_slice(&bytes).unwrap()]
|
||||
};
|
||||
let logprobs = payloads
|
||||
.iter()
|
||||
.map(|payload| &payload["choices"][0]["logprobs"])
|
||||
.find(|value| value.is_object())
|
||||
.unwrap();
|
||||
if chat_route {
|
||||
let actual = logprobs["content"][0]["top_logprobs"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|entry| json!([entry["token"], entry["logprob"]]))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
actual,
|
||||
[json!(["h", -0.5]), json!(["A", -4.0]), json!(["B", -5.0])]
|
||||
);
|
||||
} else {
|
||||
assert_eq!(
|
||||
logprobs["top_logprobs"][0],
|
||||
json!({"h": -0.5, "A": -4.0, "B": -5.0})
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
|
||||
|
||||
Reference in New Issue
Block a user