diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 46aaafbc69..30c4fc8ad2 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -126,6 +126,13 @@ pub(super) fn prepare_chat_request( .prompt_logprobs .or((request.echo && !request.stream).then_some(top_logprobs)); let include_prompt_logprobs = prompt_logprobs.is_some(); + let logprob_token_ids = request.logprob_token_ids.filter(|ids| !ids.is_empty()); + let logprobs = (request.logprobs && logprob_token_ids.is_none()).then_some(top_logprobs); + let output_top_logprobs = if logprob_token_ids.is_some() { + -1 + } else { + top_logprobs + }; let structured_outputs = convert_from_response_format( request.response_format.as_ref(), @@ -157,7 +164,7 @@ pub(super) fn prepare_chat_request( max_tokens: request.max_completion_tokens, min_tokens: request.min_tokens, thinking_token_budget: request.thinking_token_budget, - logprobs: request.logprobs.then_some(top_logprobs), + logprobs, prompt_logprobs, prompt_logprob_token_ids: None, prompt_logprob_start: None, @@ -172,7 +179,7 @@ pub(super) fn prepare_chat_request( allowed_token_ids: request.allowed_token_ids, bad_words: request.bad_words, bad_words_token_ids: None, - logprob_token_ids: None, + logprob_token_ids, structured_outputs, skip_reading_prefix_cache: None, vllm_xargs: merge_kv_transfer_params( @@ -213,7 +220,7 @@ pub(super) fn prepare_chat_request( include_usage, include_continuous_usage, requested_logprobs, - output_top_logprobs: top_logprobs, + output_top_logprobs, include_prompt_logprobs, include_reasoning, echo, @@ -1556,6 +1563,31 @@ mod tests { assert!(prepared.options.is_named_tool_choice); } + #[test] + fn selected_logprobs_override_chat_top_k_unless_empty() { + for ids in [vec![10, 20], vec![]] { + let request = serde_json::from_value(json!({ + "messages": [{"role": "user", "content": "hello"}], + "logprobs": true, "top_logprobs": 1, "logprob_token_ids": ids, + })) + .unwrap(); + let prepared = prepare_chat_request( + request, + &served(&["test-model"]), + ResolvedRequestContext::default(), + ) + .unwrap(); + let params = prepared.chat_request.sampling_params; + let selected = !ids.is_empty(); + assert_eq!(params.logprob_token_ids, selected.then_some(ids)); + assert_eq!(params.logprobs, (!selected).then_some(1)); + assert_eq!( + prepared.options.output_top_logprobs, + if selected { -1 } else { 1 } + ); + } + } + #[test] fn prepare_chat_request_lowers_logprobs_fields() { let request = ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/chat_completions/types.rs b/rust/src/server/src/routes/openai/chat_completions/types.rs index 204f3bab5a..21776dd075 100644 --- a/rust/src/server/src/routes/openai/chat_completions/types.rs +++ b/rust/src/server/src/routes/openai/chat_completions/types.rs @@ -171,6 +171,11 @@ pub struct ChatCompletionRequest { /// Number of prompt logprobs to return pub prompt_logprobs: Option, + /// Specific vocab token IDs to return logprobs for at each generated + /// position, in addition to the sampled token. Takes precedence over + /// `top_logprobs`. Requires `logprobs=true`. + pub logprob_token_ids: Option>, + /// Restrict output to these token IDs only pub allowed_token_ids: Option>, @@ -308,6 +313,7 @@ impl Default for ChatCompletionRequest { truncate_prompt_tokens: None, truncation_side: None, prompt_logprobs: None, + logprob_token_ids: None, allowed_token_ids: None, bad_words: None, echo: false, diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index 579b815ba1..e3701118d9 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -35,6 +35,13 @@ pub(super) fn validate_request_compat( ); } + if request.logprob_token_ids.as_ref().is_some_and(|ids| !ids.is_empty()) && !request.logprobs { + bail_invalid_request!( + param = "logprob_token_ids", + "when using `logprob_token_ids`, `logprobs` must be set to true." + ); + } + if let Some(prompt_logprobs) = request.prompt_logprobs { if prompt_logprobs < 0 && prompt_logprobs != -1 { bail_invalid_request!( diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 33fc6bbe0a..c75cd8da1d 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -100,6 +100,12 @@ pub(super) fn prepare_completion_request( request.max_tokens }; let echo = completion_echo_text(&request, tokenizer)?; + let logprob_token_ids = request.logprob_token_ids.filter(|ids| !ids.is_empty()); + let logprobs = if logprob_token_ids.is_some() { + None + } else { + logprobs + }; let structured_outputs = convert_from_response_format_value(&request.response_format, &request.structured_outputs)?; @@ -137,7 +143,7 @@ pub(super) fn prepare_completion_request( allowed_token_ids: request.allowed_token_ids, bad_words: None, bad_words_token_ids: None, - logprob_token_ids: None, + logprob_token_ids, structured_outputs, skip_reading_prefix_cache: None, vllm_xargs: merge_kv_transfer_params( @@ -653,6 +659,28 @@ mod tests { ); } + #[test] + fn selected_logprobs_override_completion_top_k_unless_empty() { + for ids in [vec![10, 20], vec![]] { + let request = serde_json::from_value(json!({ + "prompt": "hello", "logprobs": 0, "logprob_token_ids": ids, + })) + .unwrap(); + let prepared = prepare_completion_request( + request, + &served(&["test-model"]), + ResolvedRequestContext::default(), + &test_tokenizer(), + ) + .unwrap(); + let params = prepared.text_request.sampling_params; + let selected = !ids.is_empty(); + assert_eq!(params.logprob_token_ids, selected.then_some(ids)); + assert_eq!(params.logprobs, (!selected).then_some(0)); + assert_eq!(prepared.options.requested_logprobs, Some(0)); + } + } + #[test] fn prepare_completion_request_accepts_logprobs_fields() { let request: CompletionRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/openai/completions/types.rs b/rust/src/server/src/routes/openai/completions/types.rs index b10f8a23d7..9f7027511c 100644 --- a/rust/src/server/src/routes/openai/completions/types.rs +++ b/rust/src/server/src/routes/openai/completions/types.rs @@ -158,6 +158,11 @@ pub struct CompletionRequest { /// Number of prompt logprobs to return pub prompt_logprobs: Option, + /// Specific vocab token IDs to return logprobs for at each generated + /// position, in addition to the sampled token. Takes precedence over + /// the top-k selected by `logprobs`. Requires `logprobs` to be set. + pub logprob_token_ids: Option>, + // -------- Extra vLLM Parameters -------- /// Whether to add special tokens (e.g. BOS) to the prompt #[serde(default = "default_true")] diff --git a/rust/src/server/src/routes/openai/completions/validate.rs b/rust/src/server/src/routes/openai/completions/validate.rs index e41ad7a85d..96090cee93 100644 --- a/rust/src/server/src/routes/openai/completions/validate.rs +++ b/rust/src/server/src/routes/openai/completions/validate.rs @@ -67,6 +67,22 @@ pub(super) fn validate_request_compat( } } + if request.logprob_token_ids.as_ref().is_some_and(|ids| !ids.is_empty()) { + if request.logprobs.is_none() { + bail_invalid_request!( + param = "logprob_token_ids", + "when using `logprob_token_ids`, `logprobs` must be set." + ); + } + if request.echo && request.max_tokens == Some(0) { + bail_invalid_request!( + param = "logprob_token_ids", + "`logprob_token_ids` is not supported when `echo=True` and \ + `max_tokens=0` because no output tokens are generated." + ); + } + } + if request.use_beam_search { bail_invalid_request!( param = "use_beam_search", diff --git a/rust/src/server/src/routes/openai/utils/logprobs.rs b/rust/src/server/src/routes/openai/utils/logprobs.rs index 1de2cdc100..3f8b4d9fa7 100644 --- a/rust/src/server/src/routes/openai/utils/logprobs.rs +++ b/rust/src/server/src/routes/openai/utils/logprobs.rs @@ -284,7 +284,9 @@ fn chat_top_logprob_entries( usize::try_from(top_logprobs).unwrap_or(0) }; - position.entries.iter().take(limit) + // The engine row is the sampled token followed by the top-k or requested + // tokens, so the sampled token may repeat. Keep each token's first entry. + position.entries.iter().unique_by(|entry| entry.token_id).take(limit) } fn token_bytes(token: &str) -> Vec { @@ -370,6 +372,26 @@ mod tests { assert_eq!(chat_top_logprobs_len(-1), 3); } + #[test] + fn chat_logprobs_dedupes_repeated_tokens_before_top_logprobs_cut() { + // Sampled `A` is also the top-1 token: the engine row is `[A, A, B, C]`. + let mut logprobs = sample_logprobs(); + let sampled = logprobs.positions[0].entries[0].clone(); + logprobs.positions[0].entries.insert(0, sampled); + + let top_tokens = |top_logprobs| { + let chat_logprobs = + decoded_logprobs_to_openai_chat(&logprobs, top_logprobs, false).unwrap(); + chat_logprobs.content.unwrap()[0] + .top_logprobs + .iter() + .map(|entry| entry.token.clone()) + .collect::>() + }; + assert_eq!(top_tokens(2), ["A", "B"]); + assert_eq!(top_tokens(-1), ["A", "B", "C"]); + } + #[test] fn chat_logprobs_clamps_nan_and_negative_infinity_for_json() { let mut logprobs = sample_logprobs(); diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 9b8378f899..c6c05e63b1 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -3012,6 +3012,177 @@ async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt() ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn selected_logprobs_invalid_requests_return_client_errors() { + let (mut app, engine_task) = test_app_with_engine_handle().await; + for chat_route in [true, false] { + let base = if chat_route { + json!({"messages": [{"role": "user", "content": "hello"}], "logprobs": true}) + } else { + json!({"prompt": "hello", "logprobs": 0}) + }; + for invalid in [ + json!({"logprob_token_ids": [-1]}), + json!({"logprob_token_ids": vec![1; 129]}), + if chat_route { + json!({"logprob_token_ids": [1], "logprobs": false}) + } else { + json!({"logprob_token_ids": [1], "logprobs": null}) + }, + json!({"logprob_token_ids": [1], "use_beam_search": true}), + ] { + let mut body = base.clone(); + body.as_object_mut().unwrap().extend(invalid.as_object().unwrap().clone()); + let path = if chat_route { + "/v1/chat/completions" + } else { + "/v1/completions" + }; + let (status, error) = post_json(&mut app, path, body.clone()).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}: {error}"); + } + } + let (status, error) = post_json( + &mut app, + "/v1/completions", + json!({"prompt": "hello", "logprobs": 0, + "logprob_token_ids": [1], "echo": true, "max_tokens": 0}), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{error}"); + engine_task.abort_and_join().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn selected_logprobs_reach_engine_and_both_openai_response_modes() { + for chat_route in [true, false] { + for stream in [true, false] { + let ipc = IpcNamespace::new().unwrap(); + let address = ipc.handshake_endpoint(); + let engine_task = MockEngineTask::new(spawn_mock_engine_task( + address.clone(), + b"selected-logprobs".to_vec(), + |dealer, push| { + boxed_test_future(async move { + let add = recv_engine_message(dealer).await; + let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap(); + let params = request.sampling_params.as_ref().unwrap(); + assert_eq!(params.logprob_token_ids, Some(vec![65, 104, 66])); + assert_eq!(params.logprobs, None); + send_outputs( + push, + RequestBatchOutputs { + outputs: vec![request_output_with_logprobs( + &request.request_id, + vec![104], + Some(EngineCoreFinishReason::Stop), + None, + Some(Logprobs { + positions: vec![PositionLogprobs { + entries: vec![ + TokenLogprob { + token_id: 104, + logprob: -0.5, + rank: 1, + }, + TokenLogprob { + token_id: 65, + logprob: -4.0, + rank: 20, + }, + TokenLogprob { + token_id: 104, + logprob: -0.5, + rank: 1, + }, + TokenLogprob { + token_id: 66, + logprob: -5.0, + rank: 30, + }, + ], + }], + }), + None, + )], + ..Default::default() + } + .into(), + ) + .await; + }) + }, + )); + let client = EngineCoreClient::connect( + EngineCoreClientConfig::new_single(address).with_local_input_output_addresses( + Some(ipc.input_endpoint()), + Some(ipc.output_endpoint()), + ), + ) + .await + .unwrap(); + let chat = + ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new())); + let mut app = build_router(Arc::new(AppState::new(vec!["test-model".into()], chat))); + let mut body = if chat_route { + json!({"messages": [{"role": "user", "content": "hello"}], "logprobs": true, "top_logprobs": 1}) + } else { + json!({"prompt": "hello", "logprobs": 0}) + }; + body["stream"] = json!(stream); + // The sampled token `h` (104) is also requested. + body["logprob_token_ids"] = json!([65, 104, 66]); + let response = app + .call( + Request::builder() + .method("POST") + .uri(if chat_route { + "/v1/chat/completions" + } else { + "/v1/completions" + }) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + engine_task.await.unwrap(); + let payloads = if stream { + sse_json_payloads(std::str::from_utf8(&bytes).unwrap()) + } else { + vec![serde_json::from_slice(&bytes).unwrap()] + }; + let logprobs = payloads + .iter() + .map(|payload| &payload["choices"][0]["logprobs"]) + .find(|value| value.is_object()) + .unwrap(); + if chat_route { + let actual = logprobs["content"][0]["top_logprobs"] + .as_array() + .unwrap() + .iter() + .map(|entry| json!([entry["token"], entry["logprob"]])) + .collect::>(); + assert_eq!( + actual, + [json!(["h", -0.5]), json!(["A", -4.0]), json!(["B", -5.0])] + ); + } else { + assert_eq!( + logprobs["top_logprobs"][0], + json!({"h": -0.5, "A": -4.0, "B": -5.0}) + ); + } + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {