From b721a4c709da44551b614e2562fcfa7f4cbd1352 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Mon, 28 Sep 2026 22:50:09 +0800 Subject: [PATCH] [Feature] Add fixed-token prefill scoring (#54335) Signed-off-by: aoshen02 Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5 --- docs/training/prompt_token_id_logprobs.md | 64 +++++++ rust/Cargo.lock | 1 + .../engine-core-client/src/protocol/output.rs | 11 ++ .../src/protocol/sampling.rs | 7 + .../engine-core-client/src/tests/client.rs | 3 + .../src/tests/python_compat.py | 1 + rust/src/llm/src/output.rs | 16 ++ rust/src/llm/tests/generate.rs | 2 + rust/src/server/Cargo.toml | 1 + .../server/src/routes/inference/generate.rs | 48 +++++ .../src/routes/inference/generate/types.rs | 2 + .../src/routes/inference/generate/validate.rs | 18 ++ .../routes/openai/chat_completions/convert.rs | 2 + .../src/routes/openai/completions/convert.rs | 2 + rust/src/text/src/lower.rs | 56 ++++++ rust/src/text/src/lower/logprobs.rs | 28 +++ rust/src/text/src/lower/token_ids.rs | 8 + rust/src/text/src/request.rs | 7 + .../token_in_token_out/test_serving_tokens.py | 49 +++++ tests/v1/engine/test_logprobs_processor.py | 23 +++ tests/v1/sample/test_logprobs.py | 179 +++++++++++++++++- tests/v1/sample/test_trace_replay_params.py | 8 + .../scale_out/token_in_token_out/protocol.py | 1 + .../scale_out/token_in_token_out/serving.py | 10 + vllm/outputs.py | 7 + vllm/sampling_params.py | 69 ++++++- vllm/v1/core/sched/scheduler.py | 6 + vllm/v1/engine/__init__.py | 3 + vllm/v1/engine/input_processor.py | 13 ++ vllm/v1/engine/logprobs.py | 12 ++ vllm/v1/engine/output_processor.py | 5 + vllm/v1/outputs.py | 3 + vllm/v1/worker/gpu/async_utils.py | 7 + vllm/v1/worker/gpu/model_runner.py | 13 +- vllm/v1/worker/gpu/sample/logprob.py | 4 +- vllm/v1/worker/gpu/sample/prompt_logprob.py | 87 ++++++++- 36 files changed, 768 insertions(+), 8 deletions(-) create mode 100644 docs/training/prompt_token_id_logprobs.md diff --git a/docs/training/prompt_token_id_logprobs.md b/docs/training/prompt_token_id_logprobs.md new file mode 100644 index 0000000000..940facad4f --- /dev/null +++ b/docs/training/prompt_token_id_logprobs.md @@ -0,0 +1,64 @@ +# Prompt Token ID Logprobs (Teacher Scoring) + +In on-policy distillation, the student records its top-K candidate tokens at +every response position, and the teacher provides the log probabilities of +exactly those candidates as the distillation target. The caller passes the +candidates as `prompt_logprob_token_ids`; the teacher runs a single prefill +over the prompt plus response and gathers their log probabilities on the GPU at +every scored row, starting from `prompt_logprob_start` (typically the prompt +length minus one, so only response positions are scored). The result is a +`[rows, K]` matrix whose row `i` scores prompt token +`prompt_logprob_start + i + 1` and whose columns follow the candidate order, so +it lines up position by position with the student's top-K. + +## Quick start + +```python +from vllm import LLM, SamplingParams +from vllm.inputs import TokensPrompt + +llm = LLM(model) +output = llm.generate( + TokensPrompt(prompt_token_ids=prompt_ids + response_ids), + SamplingParams( + max_tokens=1, + prompt_logprob_token_ids=candidate_ids, + prompt_logprob_start=len(prompt_ids) - 1, + ), +) +scores = output[0].prompt_token_id_logprobs +# scores.shape == (len(response_ids), len(candidate_ids)) +# scores[i, j] = log p(candidate_ids[j] | prompt_ids + response_ids[:i]) +``` + +The same parameters are accepted by the `/inference/v1/generate` HTTP endpoint +(Python and Rust frontends), which returns the matrix as a base64-encoded +`.npy` float32 array: + +```python +import base64, io +import numpy as np + +scores = np.load(io.BytesIO(base64.b64decode(response["prompt_token_id_logprobs"]))) +``` + +The matrix has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows. With a +logits `--logprobs-mode`, it holds logits instead of log probabilities. + +## Requirements + +- The V2 model runner (`VLLM_USE_V2_MODEL_RUNNER=1`). +- `len(prompt_logprob_token_ids)` at most `--max-logprobs`. +- No `--kv-sharing-fast-prefill`. + +The request skips reading the prefix cache (local and KV connector), since +cached rows have no logits to score. + +## Limitations + +- One candidate set per request, shared by every scored row; per-position + candidates are planned ([#56860](https://github.com/vllm-project/vllm/issues/56860)). +- Non-streaming only; `stream=true` requests are rejected. +- Not exposed through the OpenAI-compatible endpoints. +- If a prefill starts past the first scored row (e.g. `skip_reading_prefix_cache=False` + with a cache hit), the result is `None` rather than a partial matrix. diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5b122ea14c..6a5d13f087 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5968,6 +5968,7 @@ dependencies = [ "asynk-strim-attr", "auto_enums", "axum", + "base64 0.22.1", "bytes", "clap", "educe", diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index 0edd0ce788..c33062eeb3 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode}; use crate::protocol::logprobs::MaybeWireLogprobs; use crate::protocol::sampling_mask::MaybeWireSamplingMask; use crate::protocol::stats::{PrefillStats, SchedulerStats}; +use crate::protocol::tensor::WireNdArray; use crate::protocol::{OpaqueValue, decode_msgpack}; /// The stop reason associated with a finished output. @@ -133,6 +134,10 @@ pub struct EngineCoreOutput { /// output when `--per-request-spec-decode-metrics` is enabled. #[serde(default)] pub spec_decode_metrics: Option, + /// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on + /// the first output of a request that asked for them. + #[serde(default)] + pub prompt_token_id_logprobs: Option, } /// Raw per-sequence speculative-decoding accumulator. @@ -180,6 +185,11 @@ impl EngineCoreOutput { self.new_sampling_mask = (self.new_sampling_mask.take()) .map(|value| value.resolve(frames, "new_sampling_mask")) .transpose()?; + if let Some(value) = self.prompt_token_id_logprobs.as_mut() { + value + .resolve_aux_frame(frames) + .map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?; + } Ok(()) } } @@ -531,6 +541,7 @@ mod tests { mm_cache_miss_hashes: None, new_sampling_mask: None, spec_decode_metrics: None, + prompt_token_id_logprobs: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/protocol/sampling.rs b/rust/src/engine-core-client/src/protocol/sampling.rs index 92f354be6f..0cd11b5397 100644 --- a/rust/src/engine-core-client/src/protocol/sampling.rs +++ b/rust/src/engine-core-client/src/protocol/sampling.rs @@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams { /// /// `None` disables prompt logprobs. `-1` requests the full vocabulary. pub prompt_logprobs: Option, + /// Candidate token IDs scored at every scored causal prompt row, where row + /// `i` scores them as predictions of prompt token `i + 1`. + pub prompt_logprob_token_ids: Option>, + /// First causal prompt row to score; `None` scores from the first row. + pub prompt_logprob_start: Option, /// Minimum probability threshold for token sampling. pub min_p: f32, /// Frequency penalty applied by the sampler. @@ -171,6 +176,8 @@ impl EngineCoreSamplingParams { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index bd3847c483..cdc3f718bd 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { mm_cache_miss_hashes: None, new_sampling_mask: None, spec_decode_metrics: None, + prompt_token_id_logprobs: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index 76978328bb..bd8d40dc32 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -116,6 +116,7 @@ class EngineCoreOutput( mm_cache_miss_hashes: list[str] | None = None new_sampling_mask: object | None = None spec_decode_metrics: object | None = None + prompt_token_id_logprobs: object | None = None class ExtendedEngineCoreOutput(EngineCoreOutput): diff --git a/rust/src/llm/src/output.rs b/rust/src/llm/src/output.rs index 34eae25682..4b6d033539 100644 --- a/rust/src/llm/src/output.rs +++ b/rust/src/llm/src/output.rs @@ -15,6 +15,7 @@ use vllm_engine_core_client::protocol::output::{ EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason, }; use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; +use vllm_engine_core_client::protocol::tensor::WireNdArray; use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream}; use crate::error::Result; @@ -38,6 +39,7 @@ pub struct CollectedGenerateOutput { pub request_id: String, pub prompt_token_ids: Vec, pub prompt_logprobs: Option, + pub prompt_token_id_logprobs: Option, pub token_ids: Vec, pub logprobs: Option, pub finish_reason: FinishReason, @@ -62,6 +64,8 @@ pub struct GeneratePromptInfo { /// Prompt logprobs returned by engine-core for scored prompt positions, /// when requested. pub prompt_logprobs: Option, + /// Log probabilities of `prompt_logprob_token_ids`, when requested. + pub prompt_token_id_logprobs: Option, } /// The reason a request finished. @@ -206,6 +210,7 @@ impl GenerateOutput { prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo { prompt_token_ids: ids, prompt_logprobs: None, + prompt_token_id_logprobs: None, }), token_ids, logprobs: None, @@ -247,6 +252,7 @@ impl GenerateOutputStream { pending_prompt_info: Some(GeneratePromptInfo { prompt_token_ids, prompt_logprobs: None, + prompt_token_id_logprobs: None, }), raw_stream, request_metrics, @@ -282,6 +288,11 @@ impl Stream for GenerateOutputStream { info.prompt_logprobs = raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap()); } + if let Some(info) = &mut self.pending_prompt_info + && info.prompt_token_id_logprobs.is_none() + { + info.prompt_token_id_logprobs = raw.prompt_token_id_logprobs; + } let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap()); let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap()); @@ -361,6 +372,7 @@ impl> + Send> T { pin_mut!(stream); let mut prompt_token_ids = None; let mut prompt_logprobs = None; + let mut prompt_token_id_logprobs = None; let mut cached_token_count = 0; let mut collected: Option = None; @@ -374,6 +386,9 @@ impl> + Send> T { if prompt_logprobs.is_none() { prompt_logprobs = info.prompt_logprobs; } + if prompt_token_id_logprobs.is_none() { + prompt_token_id_logprobs = info.prompt_token_id_logprobs; + } } if let Some(existing) = collected.as_mut() { @@ -393,6 +408,7 @@ impl> + Send> T { request_id: output.request_id, prompt_token_ids: prompt_token_ids.take().unwrap_or_default(), prompt_logprobs: prompt_logprobs.take(), + prompt_token_id_logprobs: prompt_token_id_logprobs.take(), token_ids: output.token_ids, logprobs: output.logprobs, finish_reason: FinishReason::Error, diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index 370d5c046f..efd48eba77 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -277,6 +277,7 @@ async fn generate_streams_outputs() { Some(GeneratePromptInfo { prompt_token_ids: vec![11, 22].into(), prompt_logprobs: Some(prompt_logprobs()), + prompt_token_id_logprobs: None, }) ); assert_eq!(first.token_ids, vec![1, 2]); @@ -405,6 +406,7 @@ async fn collect_output_rejects_partial_sampling_mask() { prompt_info: Some(GeneratePromptInfo { prompt_token_ids: Arc::from([11_u32, 22]), prompt_logprobs: None, + prompt_token_id_logprobs: None, }), token_ids: vec![33, 44], logprobs: None, diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index 8990ba2f3b..8b22b6344a 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -9,6 +9,7 @@ anyhow.workspace = true asynk-strim-attr.workspace = true auto_enums.workspace = true axum.workspace = true +base64.workspace = true educe.workspace = true futures.workspace = true http-body.workspace = true diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 79b68622cf..15caedaf17 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -16,12 +16,15 @@ use axum::extract::State; use axum::http::HeaderMap; use axum::response::sse::Event; use axum::response::{IntoResponse, Response}; +use base64::Engine as _; use futures::{Stream, StreamExt as _, pin_mut}; use thiserror_ext::AsReport as _; use tracing::{error, info, trace}; use tracing_futures::Instrument as _; +use vllm_engine_core_client::protocol::dtype::TensorDtype; use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs}; use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec; +use vllm_engine_core_client::protocol::tensor::WireNdArray; use vllm_llm::{ CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage, }; @@ -330,6 +333,11 @@ fn collect_generate( token_ids: collected.token_ids, }], prompt_logprobs, + prompt_token_id_logprobs: collected + .prompt_token_id_logprobs + .as_ref() + .map(npy_base64) + .transpose()?, prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids), mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(), kv_transfer_params: collected.kv_transfer_params, @@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap Result { + let (TensorDtype::F32, Some(data), [rows, cols]) = ( + array.dtype.scalar, + array.data.as_raw_view(), + array.shape.as_slice(), + ) else { + bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix"); + }; + let mut header = + format!("{{'descr': ' String { format!("token_id:{token_id}") } @@ -509,6 +540,7 @@ mod tests { prompt_info: Some(GeneratePromptInfo { prompt_token_ids: Arc::from([11_u32, 22_u32]), prompt_logprobs: None, + prompt_token_id_logprobs: None, }), token_ids: vec![33], logprobs: None, @@ -592,6 +624,7 @@ mod tests { prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo { prompt_token_ids: Arc::from(ids), prompt_logprobs: None, + prompt_token_id_logprobs: None, }), token_ids, logprobs: None, @@ -747,6 +780,7 @@ mod tests { let output = CollectedGenerateOutput { request_id: "raw-1".to_string(), prompt_logprobs: None, + prompt_token_id_logprobs: None, token_ids: vec![30], logprobs: None, finish_reason: FinishReason::stop_eos(), @@ -781,6 +815,17 @@ mod tests { assert!(response.mm_placeholders.is_none()); } + #[test] + fn prompt_token_id_logprobs_match_numpy2base64() { + let scores = + WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap(); + // numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32)) + assert_eq!( + npy_base64(&scores).unwrap(), + "k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA=" + ); + } + #[test] fn collect_generate_returns_spec_decode_metrics() { let metrics = RequestSpecDecodeMetrics { @@ -792,6 +837,7 @@ mod tests { let output = CollectedGenerateOutput { request_id: "raw-metrics".to_string(), prompt_logprobs: None, + prompt_token_id_logprobs: None, token_ids: vec![30], logprobs: None, finish_reason: FinishReason::stop_eos(), @@ -879,6 +925,7 @@ mod tests { let output = CollectedGenerateOutput { request_id: "raw-1".to_string(), prompt_logprobs: None, + prompt_token_id_logprobs: None, token_ids: vec![30], logprobs: None, finish_reason: FinishReason::stop_eos(), @@ -914,6 +961,7 @@ mod tests { let output_without_payload = |prompt_token_ids: Vec| CollectedGenerateOutput { request_id: "raw-1".to_string(), prompt_logprobs: None, + prompt_token_id_logprobs: None, token_ids: vec![3], logprobs: None, finish_reason: FinishReason::stop_eos(), diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 86d8ec00f4..78fe085b83 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -94,6 +94,8 @@ pub(super) struct GenerateResponse { pub request_id: String, pub choices: Vec, pub prompt_logprobs: Option>>>, + /// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it. + pub prompt_token_id_logprobs: Option, pub prompt_token_ids: Option>, pub mm_placeholders: Option, pub kv_transfer_params: Option, diff --git a/rust/src/server/src/routes/inference/generate/validate.rs b/rust/src/server/src/routes/inference/generate/validate.rs index 9ff8d0460e..9b1779d6e9 100644 --- a/rust/src/server/src/routes/inference/generate/validate.rs +++ b/rust/src/server/src/routes/inference/generate/validate.rs @@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat( } } + if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() { + bail_invalid_request!( + param = "sampling_params", + "`prompt_logprob_token_ids` are not available when `stream=true`." + ); + } + Ok(()) } @@ -169,6 +176,17 @@ mod tests { assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); } + #[test] + fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() { + let request: GenerateRequest = serde_json::from_value(json!({ + "token_ids": [11, 22], + "stream": true, + "sampling_params": {"prompt_logprob_token_ids": [1, 2]} + })) + .expect("parse request"); + assert!(validate_request_compat(&request, &served(&["test-model"])).is_err()); + } + #[test] fn validate_request_compat_accepts_non_stream_prompt_logprobs() { let request: GenerateRequest = serde_json::from_value(json!({ diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index d4df614270..418ab23ce8 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request( thinking_token_budget: request.thinking_token_budget, logprobs: request.logprobs.then_some(top_logprobs), prompt_logprobs, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: request.min_p, frequency_penalty: request.frequency_penalty, presence_penalty: request.presence_penalty, diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 53a6ff9906..d05cfffc89 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request( thinking_token_budget: request.thinking_token_budget, logprobs, prompt_logprobs, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: request.min_p, frequency_penalty: request.frequency_penalty, presence_penalty: request.presence_penalty, diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index e1357ed7f9..01a4fb25b8 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -103,6 +103,8 @@ pub fn lower_sampling_params( thinking_token_budget, logprobs, prompt_logprobs, + prompt_logprob_token_ids, + prompt_logprob_start, min_p, frequency_penalty, presence_penalty, @@ -123,6 +125,8 @@ pub fn lower_sampling_params( logprobs, prompt_logprobs, logprob_token_ids.as_deref(), + prompt_logprob_token_ids.as_deref(), + prompt_logprob_start, sampling_limits, )?; validate_repetition_detection(repetition_detection.as_ref())?; @@ -176,6 +180,8 @@ pub fn lower_sampling_params( thinking_token_budget, logprobs, prompt_logprobs, + prompt_logprob_token_ids, + prompt_logprob_start, min_p, frequency_penalty, presence_penalty, @@ -646,6 +652,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -701,6 +709,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -870,6 +880,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -935,6 +947,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -1008,6 +1022,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.1, frequency_penalty: 0.0, presence_penalty: 0.0, @@ -1122,6 +1138,44 @@ mod tests { )); } + #[test] + fn lower_sampling_params_validates_prompt_logprob_token_ids() { + let lower = |ids: Option>, start: Option| { + lower_sampling_params_with_limits( + SamplingParams { + prompt_logprob_token_ids: ids, + prompt_logprob_start: start, + ..Default::default() + }, + sample_sampling_limits(), + ) + }; + let rejects = |ids, start| match lower(ids, start) { + Err(error) => error.to_string(), + Ok(_) => panic!("expected rejection"), + }; + + let params = lower(Some(vec![1, 2]), Some(1)).unwrap(); + assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2])); + assert_eq!(params.prompt_logprob_start, Some(1)); + expect_test::expect![[r#" + [ + "prompt_logprob_token_ids must not be empty.", + "requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20", + "token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000", + "prompt_logprob_token_ids must not contain duplicates.", + "prompt_logprob_start requires prompt_logprob_token_ids.", + ] + "#]] + .assert_debug_eq(&[ + rejects(Some(vec![]), None), + rejects(Some((0..21).collect()), None), + rejects(Some(vec![1000]), None), + rejects(Some(vec![1, 1]), None), + rejects(None, Some(0)), + ]); + } + #[test] fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() { let error = lower_sampling_params_with_limits( @@ -1259,6 +1313,8 @@ mod tests { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: 0.1, frequency_penalty: 0.0, presence_penalty: 0.0, diff --git a/rust/src/text/src/lower/logprobs.rs b/rust/src/text/src/lower/logprobs.rs index b2407f06ed..58ad5d694b 100644 --- a/rust/src/text/src/lower/logprobs.rs +++ b/rust/src/text/src/lower/logprobs.rs @@ -6,6 +6,7 @@ //! `-1` is expanded only for bounds checks. The original request values are //! passed through to engine-core. +use itertools::Itertools as _; use thiserror::Error; use crate::backend::SamplingLimits; @@ -35,6 +36,12 @@ pub enum LogprobsError { len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}." )] TokenIdsMismatch { logprobs: i32, num_token_ids: usize }, + #[error("prompt_logprob_token_ids must not be empty.")] + EmptyPromptLogprobTokenIds, + #[error("prompt_logprob_token_ids must not contain duplicates.")] + DuplicatePromptLogprobTokenIds, + #[error("prompt_logprob_start requires prompt_logprob_token_ids.")] + PromptLogprobStartWithoutTokenIds, } /// Validate logprobs count sampling parameters. @@ -42,6 +49,8 @@ pub(super) fn validate_logprobs( logprobs: Option, prompt_logprobs: Option, logprob_token_ids: Option<&[u32]>, + prompt_logprob_token_ids: Option<&[u32]>, + prompt_logprob_start: Option, sampling_limits: SamplingLimits, ) -> Result<(), LogprobsError> { let vocab_size = sampling_limits.model_vocab_size; @@ -50,9 +59,28 @@ pub(super) fn validate_logprobs( validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?; validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?; + validate_logprobs_count( + prompt_logprob_token_ids.map(|ids| ids.len() as i32), + max_logprobs, + vocab_size, + "prompt_logprob_token_ids", + )?; + validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?; validate_logprob_token_ids(logprobs, logprob_token_ids) } +fn validate_prompt_logprob_token_ids( + token_ids: Option<&[u32]>, + start: Option, +) -> Result<(), LogprobsError> { + match token_ids { + Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds), + Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds), + None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds), + _ => Ok(()), + } +} + fn validate_logprobs_count( requested: Option, max_logprobs: usize, diff --git a/rust/src/text/src/lower/token_ids.rs b/rust/src/text/src/lower/token_ids.rs index 8512a830e6..240b0528e1 100644 --- a/rust/src/text/src/lower/token_ids.rs +++ b/rust/src/text/src/lower/token_ids.rs @@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range( )?; } + if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() { + validate_param( + "prompt_logprob_token_ids", + token_ids.iter().copied(), + limits.model_vocab_size, + )?; + } + if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() { validate_param( "bad_words", diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index 2c768f4855..9c9f22a38e 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -97,6 +97,11 @@ pub struct SamplingParams { /// /// `None` disables prompt logprobs. `-1` requests the full vocabulary. pub prompt_logprobs: Option, + /// Candidate token IDs scored at every scored causal prompt row, where row + /// `i` scores them as predictions of prompt token `i + 1`. + pub prompt_logprob_token_ids: Option>, + /// First causal prompt row to score; `None` scores from the first row. + pub prompt_logprob_start: Option, /// Minimum probability threshold for token sampling. `None` means no /// explicit user override. pub min_p: Option, @@ -156,6 +161,8 @@ impl Default for SamplingParams { thinking_token_budget: None, logprobs: None, prompt_logprobs: None, + prompt_logprob_token_ids: None, + prompt_logprob_start: None, min_p: None, frequency_penalty: None, presence_penalty: None, diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index fe41e651be..18513f83dc 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import io import json import httpx +import numpy as np +import pybase64 as base64 import pytest import pytest_asyncio from transformers import AutoTokenizer @@ -311,6 +314,52 @@ async def test_generate_logprobs(client, logprobs_value): assert len(entry["top_logprobs"]) == max(logprobs_value, 1) +@pytest.mark.asyncio +async def test_generate_prompt_token_id_logprobs(client): + token_ids = [11, 22, 33, 44, 55] + candidates = [22, 33, 44, 55] + sampling_params = { + "max_tokens": 1, + "prompt_logprobs": 0, + "prompt_logprob_token_ids": candidates, + "prompt_logprob_start": 1, + } + payload = { + "model": MODEL_NAME, + "token_ids": token_ids, + "sampling_params": sampling_params, + } + resp = await client.post(GEN_ENDPOINT, json=payload) + resp.raise_for_status() + data = resp.json() + + scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"]))) + assert scores.shape == (len(token_ids) - 2, len(candidates)) + for row, target in enumerate(token_ids[2:]): + expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"] + assert scores[row, candidates.index(target)] == pytest.approx( + expected, abs=1e-3 + ) + + stream_only_ids = {"prompt_logprob_token_ids": candidates} + resp = await client.post( + GEN_ENDPOINT, + json={**payload, "sampling_params": stream_only_ids, "stream": True}, + ) + assert resp.status_code == 400 + assert "prompt_logprob_token_ids" in resp.text + + past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)} + resp = await client.post( + GEN_ENDPOINT, json={**payload, "sampling_params": past_end} + ) + resp.raise_for_status() + empty = np.load( + io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"])) + ) + assert empty.shape == (0, len(candidates)) + + @pytest.mark.asyncio async def test_same_response_as_chat_completions(client, tokenizer, messages): token_ids = tokenizer.apply_chat_template( diff --git a/tests/v1/engine/test_logprobs_processor.py b/tests/v1/engine/test_logprobs_processor.py index edb8cef518..ee404eccfa 100644 --- a/tests/v1/engine/test_logprobs_processor.py +++ b/tests/v1/engine/test_logprobs_processor.py @@ -11,8 +11,10 @@ row), the trailing positions are populated with sentinel values """ import numpy as np +import torch from vllm.logprobs import create_sample_logprobs +from vllm.v1.engine import EngineCoreOutput from vllm.v1.engine.logprobs import LogprobsProcessor from vllm.v1.outputs import LogprobsLists @@ -64,3 +66,24 @@ def test_accepts_exactly_sized_row(): pos = processor.logprobs[0] assert set(pos.keys()) == {7, 11, 13} + + +def test_prompt_token_id_logprobs_are_popped_once(): + """DELTA outputs carry fixed-ID prompt scores exactly once.""" + processor = _make_processor(num_logprobs=1) + assert processor.pop_prompt_token_id_logprobs() is None + + processor.update_from_output( + EngineCoreOutput( + request_id="req-0", + new_token_ids=[], + prompt_token_id_logprobs=torch.tensor( + [[-0.5, -1.5], [-2.5, -3.5]], dtype=torch.float32 + ), + ) + ) + + scores = processor.pop_prompt_token_id_logprobs() + assert scores is not None + assert scores.tolist() == [[-0.5, -1.5], [-2.5, -3.5]] + assert processor.pop_prompt_token_id_logprobs() is None diff --git a/tests/v1/sample/test_logprobs.py b/tests/v1/sample/test_logprobs.py index d2cb99bc26..85551e80b4 100644 --- a/tests/v1/sample/test_logprobs.py +++ b/tests/v1/sample/test_logprobs.py @@ -6,7 +6,9 @@ import math from collections.abc import Generator from types import SimpleNamespace from typing import get_args +from unittest.mock import patch +import numpy as np import pytest import torch @@ -23,6 +25,7 @@ from vllm.config.model import LogprobsMode from vllm.distributed import cleanup_dist_env_and_memory from vllm.exceptions import VLLMValidationError from vllm.platforms import current_platform +from vllm.v1.engine.input_processor import InputProcessor from ...conftest import HfRunner, VllmRunner @@ -81,9 +84,9 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]: yield hf_model -def _model_config(vocab_size: int = 10): +def _model_config(vocab_size: int = 10, max_logprobs: int = 20): return SimpleNamespace( - max_logprobs=20, + max_logprobs=max_logprobs, logits_processors=None, is_diffusion=False, get_vocab_size=lambda: vocab_size, @@ -432,6 +435,57 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]): ) +def test_prompt_logprob_token_ids_bounded_by_max_logprobs(): + """The candidate count is bounded by max_logprobs.""" + model_config = _model_config(vocab_size=100, max_logprobs=4) + + def verify(**kwargs): + SamplingParams(**kwargs).verify( + model_config, + speculative_config=None, + structured_outputs_config=None, + tokenizer=None, + ) + + verify(prompt_logprob_token_ids=[1, 2, 3, 4]) + + # The error names the value to raise max_logprobs to. + with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"): + verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5]) + + # prompt_logprob_start alone is a caller mistake, not a silent no-op. + with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"): + verify(prompt_logprob_start=3) + + +def test_prompt_logprob_token_ids_require_v2_model_runner(): + """Only the V2 runner scores them; the V1 runner would return None, and + kv-sharing fast prefill would score rows the cross-decoder never ran.""" + processor = SimpleNamespace( + model_config=SimpleNamespace( + return_sampling_mask=False, enable_trace_replay=False, is_diffusion=False + ), + vllm_config=SimpleNamespace( + reasoning_config=None, + use_v2_model_runner=False, + cache_config=SimpleNamespace(kv_sharing_fast_prefill=False), + ), + speculative_config=None, + structured_outputs_config=None, + tokenizer=None, + validate_logits_processors_params=lambda params: None, + ) + params = SamplingParams(prompt_logprob_token_ids=[1, 2]) + with patch.object(SamplingParams, "verify"): + with pytest.raises(VLLMValidationError, match="V2 model runner"): + InputProcessor._validate_params(processor, params, ("generate",)) + processor.vllm_config.use_v2_model_runner = True + InputProcessor._validate_params(processor, params, ("generate",)) + processor.vllm_config.cache_config.kv_sharing_fast_prefill = True + with pytest.raises(VLLMValidationError, match="fast-prefill"): + InputProcessor._validate_params(processor, params, ("generate",)) + + def test_none_logprobs(vllm_model, example_prompts): """Engine should return `logprobs` and `prompt_logprobs` as `None`. @@ -1298,6 +1352,127 @@ def test_prompt_logprobs_with_chunking_and_preemption(): print(f"Test passed with {preemptions} preemptions") +def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch): + """Fixed-ID scores stay row-aligned across chunked prefill and preemption.""" + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + + prompts = [ + "The following numbers of the sequence " + + ", ".join(str(i) for i in range(10)) + + " are:", + "In one word, the capital of France is ", + ] + [f"Tell me about the number {i}: " for i in range(32)] + + start = 2 + candidate_ids = [10, 100, 1000, 10000] + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=40, + min_tokens=20, + prompt_logprob_token_ids=candidate_ids, + prompt_logprob_start=start, + ) + + with VllmRunner( + "Qwen/Qwen3-0.6B", + max_model_len=512, + enable_chunked_prefill=True, + max_num_batched_tokens=48, # Force prefill chunking + num_gpu_blocks_override=33, # Force preemptions (32 usable + null block) + disable_log_stats=False, + gpu_memory_utilization=0.25, + ) as vllm_model: + metrics_before = vllm_model.llm.get_metrics() + outputs = vllm_model.llm.generate(prompts, sampling_params) + + for i, output in enumerate(outputs): + scores = output.prompt_token_id_logprobs + assert scores is not None, f"Output {i} missing fixed-ID scores" + expected_shape = ( + len(output.prompt_token_ids) - 1 - start, + len(candidate_ids), + ) + assert scores.shape == expected_shape, ( + f"Output {i} scored {scores.shape}, expected {expected_shape}" + ) + assert math.isfinite(float(scores.min())) + assert float(scores.max()) <= 1e-3, "logprobs must be <= 0" + + metrics_after = vllm_model.llm.get_metrics() + preemptions_before = next( + (m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0 + ) + preemptions_after = next( + (m.value for m in metrics_after if m.name == "vllm:num_preemptions"), 0 + ) + assert preemptions_after - preemptions_before > 0, ( + "Test did not trigger any preemptions" + ) + + # Row alignment is numerical: chunked and preempted scores must match an + # unchunked, unpreempted run of the same requests. Batch composition moves + # bf16 tail logprobs by a few percent; a misaligned row differs by nats. + with VllmRunner( + "Qwen/Qwen3-0.6B", + max_model_len=512, + max_logprobs=64, + gpu_memory_utilization=0.25, + ) as reference: + reference_outputs = reference.llm.generate(prompts, sampling_params) + own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs] + self_scored = reference.llm.generate( + prompts, + [ + SamplingParams( + max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids + ) + for ids in own_ids + ], + ) + for output, ref in zip(outputs, reference_outputs): + np.testing.assert_allclose( + output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1 + ) + for output, ids in zip(self_scored, own_ids): + scores = output.prompt_token_id_logprobs + for row, target in enumerate(output.prompt_token_ids[1:]): + expected = output.prompt_logprobs[row + 1][target].logprob + assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3) + + +def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch): + """A prefill that starts past the first scored row returns no scores. + + Such a prefill (here a prefix-cache hit the caller opted back into) never + computes the leading rows, so emitting the buffer would return values the + model never produced for them. + """ + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + + prompt = "The capital of France is Paris. " * 20 + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=1, + prompt_logprob_token_ids=[10, 100, 1000], + skip_reading_prefix_cache=False, + ) + + with VllmRunner( + "Qwen/Qwen3-0.6B", + max_model_len=512, + enable_prefix_caching=True, + gpu_memory_utilization=0.25, + ) as vllm_model: + first = vllm_model.llm.generate([prompt], sampling_params)[0] + assert first.prompt_token_id_logprobs is not None + assert math.isfinite(float(first.prompt_token_id_logprobs.min())) + + # The re-send is served from the prefix cache, so its prefill starts + # past row 0 and the leading rows are never computed. + second = vllm_model.llm.generate([prompt], sampling_params)[0] + assert second.prompt_token_id_logprobs is None + + @large_gpu_mark(min_gb=24) def test_token_logprobs_large_batch_int64_row_offset(): """Regression: logprob kernel row offset (row * vocab_size) must use int64. diff --git a/tests/v1/sample/test_trace_replay_params.py b/tests/v1/sample/test_trace_replay_params.py index 6e0d726470..d5736f1773 100644 --- a/tests/v1/sample/test_trace_replay_params.py +++ b/tests/v1/sample/test_trace_replay_params.py @@ -41,6 +41,14 @@ def test_sampling_params_trace_field_rejects_empty_list(): ) +def test_sampling_params_trace_field_rejects_prompt_scoring(): + params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3]) + with pytest.raises(ValueError, match="prompt_logprob_token_ids"): + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) + + def test_sampling_params_trace_field_requires_single_output(): params = SamplingParams(n=2, trace_decode_token_ids=[1]) with pytest.raises(ValueError, match="requires n=1"): diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index fb2b938723..ad607648bd 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -381,6 +381,7 @@ class GenerateResponse(BaseModel): choices: list[GenerateResponseChoice] usage: UsageInfo | None = Field(default=None) prompt_logprobs: list[dict[int, Logprob] | None] | None = None + prompt_token_id_logprobs: str | None = None prompt_token_ids: list[int] | None = None mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None metrics: PerRequestMetrics | None = None diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index b480b802ec..8b0fbff99b 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -138,6 +138,11 @@ class ServingTokens(GenerateBaseServing): f"sampling_params.n must be at most the server's max_num_seqs " f"({max_num_seqs}), got {sampling_params.n}." ) + # The stream schema has no field for the scores. + if request.stream and sampling_params.prompt_logprob_token_ids is not None: + return self.create_error_response( + "prompt_logprob_token_ids are not available when stream=true." + ) if self.force_no_detokenize and sampling_params.stop: # SamplingParams rejects stop with detokenize=False at request # validation, but this server forces detokenize=False afterwards, @@ -389,6 +394,11 @@ class ServingTokens(GenerateBaseServing): choices=choices, usage=usage, prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs), + prompt_token_id_logprobs=( + numpy2base64(final_res.prompt_token_id_logprobs) + if final_res.prompt_token_id_logprobs is not None + else None + ), prompt_token_ids=( final_res.prompt_token_ids if request.return_token_ids else None ), diff --git a/vllm/outputs.py b/vllm/outputs.py index e9a08f7c46..049d795f78 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -117,6 +117,10 @@ class RequestOutput: For encoder/decoder models, this is the decoder input prompt token ids. prompt_logprobs: The log probabilities to return per prompt token. + prompt_token_id_logprobs: Logprobs of prompt_logprob_token_ids, shaped + [max(prompt_len - 1 - prompt_logprob_start, 0), + len(prompt_logprob_token_ids)]; row i scores them as predictions of + prompt token prompt_logprob_start + i + 1. outputs: The output sequences of the request. finished: Whether the whole request is finished. metrics: Metrics associated with the request. @@ -148,6 +152,7 @@ class RequestOutput: num_cached_tokens: int | None = None, num_cache_creation_tokens: int | None = None, *, + prompt_token_id_logprobs: np.ndarray | None = None, kv_transfer_params: dict[str, Any] | None = None, ec_transfer_params: dict[str, Any] | None = None, # Forward compatibility, code that uses args added in new release can @@ -162,6 +167,7 @@ class RequestOutput: self.prompt = prompt self.prompt_token_ids = prompt_token_ids self.prompt_logprobs = prompt_logprobs + self.prompt_token_id_logprobs = prompt_token_id_logprobs self.outputs = outputs self.finished = finished self.metrics = metrics @@ -215,6 +221,7 @@ class RequestOutput: f"encoder_prompt={self.encoder_prompt!r}, " f"encoder_prompt_token_ids={self.encoder_prompt_token_ids}, " f"prompt_logprobs={self.prompt_logprobs}, " + f"prompt_token_id_logprobs={self.prompt_token_id_logprobs}, " f"outputs={self.outputs}, " f"finished={self.finished}, " f"metrics={self.metrics}, " diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 88a8978d69..15657997ab 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -289,6 +289,12 @@ class SamplingParams( prompt_logprobs: int | None = None """Number of log probabilities to return per prompt token. When set to -1, return all `vocab_size` log probabilities.""" + prompt_logprob_token_ids: list[int] | None = None + """Token IDs to score at each scored causal prompt row, where row i scores + them as predictions of prompt token i + 1. The last prompt row is excluded, + so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows.""" + prompt_logprob_start: int | None = None + """First causal prompt row to score; defaults to the first row.""" logprob_token_ids: list[int] | None = None """Specific token IDs to return logprobs for. More efficient than logprobs=-1 when you only need logprobs for a small set of tokens. @@ -398,6 +404,8 @@ class SamplingParams( min_tokens: int = 0, logprobs: int | None = None, prompt_logprobs: int | None = None, + prompt_logprob_token_ids: list[int] | None = None, + prompt_logprob_start: int | None = None, detokenize: bool = True, skip_special_tokens: bool = True, spaces_between_special_tokens: bool = True, @@ -464,6 +472,8 @@ class SamplingParams( min_tokens=min_tokens, logprobs=logprobs, prompt_logprobs=prompt_logprobs, + prompt_logprob_token_ids=prompt_logprob_token_ids, + prompt_logprob_start=prompt_logprob_start, logprob_token_ids=logprob_token_ids, detokenize=detokenize, skip_special_tokens=skip_special_tokens, @@ -550,7 +560,10 @@ class SamplingParams( # If prefix caching is enabled, # the output of prompt logprobs may less than n_prompt_tokens, # we need to skip reading cache at this request. - self.skip_reading_prefix_cache = self.prompt_logprobs is not None + self.skip_reading_prefix_cache = ( + self.prompt_logprobs is not None + or self.prompt_logprob_token_ids is not None + ) def _verify_args(self) -> None: _verify_num_sequences(self.n, "n") @@ -902,6 +915,56 @@ class SamplingParams( value=num_prompt_logprobs, ) + # Validate prompt_logprob_token_ids. + if self.prompt_logprob_token_ids is not None: + n = len(self.prompt_logprob_token_ids) + if n == 0: + raise VLLMValidationError( + "prompt_logprob_token_ids must not be empty.", + parameter="prompt_logprob_token_ids", + value=n, + ) + if n > max_logprobs: + raise VLLMValidationError( + f"Requested prompt_logprob_token_ids of length {n}, " + f"which is greater than max allowed: {max_logprobs}. " + f"Set max_logprobs (--max-logprobs) to at least {n}.", + parameter="prompt_logprob_token_ids", + value=n, + ) + vocab_size = model_config.get_vocab_size() + invalid_token_ids = [ + token_id + for token_id in self.prompt_logprob_token_ids + if token_id < 0 or token_id >= vocab_size + ] + if invalid_token_ids: + raise VLLMValidationError( + f"token_id(s) {invalid_token_ids} in " + f"prompt_logprob_token_ids contain out-of-vocab token ids. " + f"Vocabulary size: {vocab_size}", + parameter="prompt_logprob_token_ids", + value=invalid_token_ids, + ) + if len(set(self.prompt_logprob_token_ids)) != n: + raise VLLMValidationError( + "prompt_logprob_token_ids must not contain duplicates.", + parameter="prompt_logprob_token_ids", + value=self.prompt_logprob_token_ids, + ) + if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0: + raise VLLMValidationError( + "prompt_logprob_start must be non-negative.", + parameter="prompt_logprob_start", + value=self.prompt_logprob_start, + ) + elif self.prompt_logprob_start is not None: + raise VLLMValidationError( + "prompt_logprob_start requires prompt_logprob_token_ids.", + parameter="prompt_logprob_start", + value=self.prompt_logprob_start, + ) + def _validate_stop_token_ids(self, model_config: ModelConfig) -> None: """Validate stop_token_ids are within vocabulary range.""" if not self.stop_token_ids: @@ -969,6 +1032,10 @@ class SamplingParams( raise ValueError( "trace_decode_token_ids is not supported with prompt_logprobs." ) + if self.prompt_logprob_token_ids is not None: + raise ValueError( + "trace_decode_token_ids is not supported with prompt_logprob_token_ids." + ) if speculative_config is not None: raise ValueError( "trace_decode_token_ids is not supported with speculative decoding." diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ce832a18f6..8ebedef4ec 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1972,6 +1972,9 @@ class Scheduler(SchedulerInterface): sampled_token_ids = model_runner_output.sampled_token_ids logprobs = model_runner_output.logprobs prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict + prompt_token_id_logprobs_dict = ( + model_runner_output.prompt_token_id_logprobs_dict + ) num_scheduled_tokens = scheduler_output.num_scheduled_tokens pooler_outputs = model_runner_output.pooler_output num_nans_in_logits = model_runner_output.num_nans_in_logits @@ -2194,6 +2197,7 @@ class Scheduler(SchedulerInterface): # Get prompt logprobs for this request. prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id) + prompt_token_id_logprobs = prompt_token_id_logprobs_dict.get(req_id) if should_emit_output: # Add EngineCoreOutput for this Request. outputs[request.client_index].append( @@ -2204,6 +2208,7 @@ class Scheduler(SchedulerInterface): new_logprobs=new_logprobs, new_sampling_mask=new_sampling_mask, new_prompt_logprobs_tensors=prompt_logprobs_tensors, + prompt_token_id_logprobs=prompt_token_id_logprobs, pooling_output=pooler_output, stop_reason=request.stop_reason, events=request.take_events(), @@ -2223,6 +2228,7 @@ class Scheduler(SchedulerInterface): else: # Invariant: EngineCore returns no partial prefill outputs. assert not prompt_logprobs_tensors + assert prompt_token_id_logprobs is None # Remove the stopped requests from the running and waiting queues. if stopped_running_reqs: diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index e5bbb96a9c..effe8d2416 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -236,6 +236,9 @@ class EngineCoreOutput( # Appended last so `array_like` positional serialization stays compatible. spec_decode_metrics: RequestSpecDecodeMetrics | None = None + # [num_scored_rows, num_token_ids]; appended last for array_like compatibility. + prompt_token_id_logprobs: torch.Tensor | None = None + @property def finished(self) -> bool: return self.finish_reason is not None diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 369c31ca27..33d413ba22 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -131,6 +131,19 @@ class InputProcessor: self.structured_outputs_config, self.tokenizer, ) + if params.prompt_logprob_token_ids is not None: + if not self.vllm_config.use_v2_model_runner: + raise VLLMValidationError( + "prompt_logprob_token_ids requires the V2 model runner " + "(VLLM_USE_V2_MODEL_RUNNER=1).", + parameter="prompt_logprob_token_ids", + ) + if self.vllm_config.cache_config.kv_sharing_fast_prefill: + raise VLLMValidationError( + "prompt_logprob_token_ids is incorrect with " + "--kv-sharing-fast-prefill; disable it for scoring.", + parameter="prompt_logprob_token_ids", + ) self.validate_logits_processors_params(params) diff --git a/vllm/v1/engine/logprobs.py b/vllm/v1/engine/logprobs.py index ff483e128e..f8230e5af6 100644 --- a/vllm/v1/engine/logprobs.py +++ b/vllm/v1/engine/logprobs.py @@ -5,6 +5,8 @@ import itertools from collections.abc import Iterable from dataclasses import dataclass +import numpy as np + from vllm.logger import init_logger from vllm.logprobs import ( FlatLogprobs, @@ -38,6 +40,8 @@ class LogprobsProcessor: cumulative_logprob: float | None num_logprobs: int | None num_prompt_logprobs: int | None + # [num_scored_rows, num_token_ids], set once on the final prefill chunk. + prompt_token_id_logprobs: np.ndarray | None = None @classmethod def from_new_request( @@ -204,6 +208,12 @@ class LogprobsProcessor: self.prompt_logprobs = [] return plp + def pop_prompt_token_id_logprobs(self) -> np.ndarray | None: + """Pop and return the fixed-ID prompt scores.""" + scores = self.prompt_token_id_logprobs + self.prompt_token_id_logprobs = None + return scores + @staticmethod def _get_sampled_context_ids( logprobs_source: SampleLogprobs | PromptLogprobs | None, @@ -352,3 +362,5 @@ class LogprobsProcessor: self._update_sample_logprobs(output.new_logprobs) if output.new_prompt_logprobs_tensors is not None: self._update_prompt_logprobs(output.new_prompt_logprobs_tensors) + if output.prompt_token_id_logprobs is not None: + self.prompt_token_id_logprobs = output.prompt_token_id_logprobs.numpy() diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index b4f7338048..5ab5636f81 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -390,8 +390,12 @@ class RequestState: if self.output_kind == RequestOutputKind.DELTA: # Side effect: logprobs processor forgets prompt logprobs prompt_logprobs = self.logprobs_processor.pop_prompt_logprobs() + prompt_token_id_logprobs = ( + self.logprobs_processor.pop_prompt_token_id_logprobs() + ) else: prompt_logprobs = self.logprobs_processor.prompt_logprobs + prompt_token_id_logprobs = self.logprobs_processor.prompt_token_id_logprobs return RequestOutput( request_id=external_req_id, # request_id is what was provided externally @@ -399,6 +403,7 @@ class RequestState: prompt=self.prompt, prompt_token_ids=prompt_token_ids, prompt_logprobs=prompt_logprobs, + prompt_token_id_logprobs=prompt_token_id_logprobs, outputs=cast(list[CompletionOutput], outputs), finished=finished, kv_transfer_params=kv_transfer_params, diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index c38eafa4a4..1f04c2ccf9 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -283,6 +283,9 @@ class ModelRunnerOutput: default_factory=dict ) + # req_id -> [num_scored_rows, num_token_ids] prompt_logprob_token_ids scores. + prompt_token_id_logprobs_dict: dict[str, torch.Tensor] = field(default_factory=dict) + # [num_reqs, hidden_size] pooler_output: list[torch.Tensor | None] | None = None diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index 02592980a8..3c58433318 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -156,6 +156,10 @@ class AsyncOutput(AsyncModelRunnerOutput): k: v.to_cpu_nonblocking() if v is not None else None for k, v in self.model_runner_output.prompt_logprobs_dict.items() } + token_id_logprobs = self.model_runner_output.prompt_token_id_logprobs_dict + self.prompt_token_id_logprobs_dict = { + k: v.to("cpu", non_blocking=True) for k, v in token_id_logprobs.items() + } if self.pending_aux_output is not None: self.pending_aux_output.enqueue_cpu_copy( num_sampled=self.num_sampled_tokens_np, @@ -194,6 +198,9 @@ class AsyncOutput(AsyncModelRunnerOutput): if self.logprobs_tensors is not None: self.model_runner_output.logprobs = self.logprobs_tensors.tolists() self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict + self.model_runner_output.prompt_token_id_logprobs_dict = ( + self.prompt_token_id_logprobs_dict + ) if self.pending_aux_output is not None: self.model_runner_output.aux_output_connector_output = ( self.pending_aux_output.process_output() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 9f25425ab6..3d3dc9b162 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -494,7 +494,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): ), ) self.prompt_logprobs_worker = PromptLogprobsWorker( - self.max_num_reqs, logprobs_mode=self.model_config.logprobs_mode + self.max_num_reqs, + self.device, + logprobs_mode=self.model_config.logprobs_mode, ) self.structured_outputs_worker = StructuredOutputsWorker( max_num_logits=self.max_num_reqs * self.decode_query_len, @@ -2063,6 +2065,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.req_states.num_computed_tokens.gpu, self.req_states.prompt_len.np, ) + prompt_token_id_logprobs_dict = ( + self.prompt_logprobs_worker.compute_prompt_token_id_logprobs( + self.model.compute_logits, + hidden_states, + input_batch, + self.req_states.prompt_len.np, + ) + ) # Prepare the model runner output. model_runner_output = ModelRunnerOutput( @@ -2072,6 +2082,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)}, sampled_token_ids=None, # type: ignore prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type] + prompt_token_id_logprobs_dict=prompt_token_id_logprobs_dict, cudagraph_stats=cudagraph_stats, ) pending_aux_output = None diff --git a/vllm/v1/worker/gpu/sample/logprob.py b/vllm/v1/worker/gpu/sample/logprob.py index 384fcdb9fd..f78db95dea 100644 --- a/vllm/v1/worker/gpu/sample/logprob.py +++ b/vllm/v1/worker/gpu/sample/logprob.py @@ -19,6 +19,7 @@ def _topk_log_softmax_kernel( logits_ptr, logits_stride, topk_ids_ptr, + topk_ids_stride, topk, vocab_size, BLOCK_SIZE: tl.constexpr, @@ -49,7 +50,7 @@ def _topk_log_softmax_kernel( k_offset = j + tl.arange(0, TOPK_BLOCK_SIZE) k_mask = k_offset < topk topk_ids = tl.load( - topk_ids_ptr + req_idx * topk + k_offset, mask=k_mask, other=0 + topk_ids_ptr + req_idx * topk_ids_stride + k_offset, mask=k_mask, other=0 ) logits = tl.load(row_ptr + topk_ids, mask=k_mask) logits = logits.to(tl.float32) @@ -98,6 +99,7 @@ def compute_token_logprobs( logits, logits.stride(0), token_ids, + token_ids.stride(0), num_logprobs, vocab_size, BLOCK_SIZE=1024, # type: ignore diff --git a/vllm/v1/worker/gpu/sample/prompt_logprob.py b/vllm/v1/worker/gpu/sample/prompt_logprob.py index 50baf39ebb..24fdb6188e 100644 --- a/vllm/v1/worker/gpu/sample/prompt_logprob.py +++ b/vllm/v1/worker/gpu/sample/prompt_logprob.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Callable +from dataclasses import dataclass import numpy as np import torch @@ -8,20 +9,41 @@ import torch from vllm.config.model import LogprobsMode from vllm.sampling_params import SamplingParams from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.outputs import LogprobsTensors from vllm.v1.worker.gpu.input_batch import InputBatch -from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores +from vllm.v1.worker.gpu.sample.logprob import ( + compute_token_logprobs, + compute_topk_scores, +) + +CHUNK_SIZE = 1024 + + +@dataclass +class _TokenIdScores: + token_ids: torch.Tensor # [num_ids], on device + start: int # first prompt row to score + scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk class PromptLogprobsWorker: - def __init__(self, max_num_reqs: int, logprobs_mode: LogprobsMode = "raw_logprobs"): + def __init__( + self, + max_num_reqs: int, + device: torch.device, + logprobs_mode: LogprobsMode = "raw_logprobs", + ): self.max_num_reqs = max_num_reqs + self.device = device self.logprobs_mode = logprobs_mode self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool) self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32) # req_idx -> list of in-progress LogprobsTensors self.in_progress_prompt_logprobs: dict[str, list[LogprobsTensors]] = {} + # req_id -> fixed-ID scoring state + self.token_id_scores: dict[str, _TokenIdScores] = {} def add_request(self, req_id: str, req_idx: int, sampling_params: SamplingParams): uses_prompt_logprobs = sampling_params.prompt_logprobs is not None @@ -29,9 +51,69 @@ class PromptLogprobsWorker: self.num_prompt_logprobs[req_idx] = sampling_params.prompt_logprobs or 0 if uses_prompt_logprobs: self.in_progress_prompt_logprobs[req_id] = [] + if sampling_params.prompt_logprob_token_ids is not None: + self.token_id_scores[req_id] = _TokenIdScores( + async_tensor_h2d( + sampling_params.prompt_logprob_token_ids, self.device, torch.int64 + ), + sampling_params.prompt_logprob_start or 0, + ) def remove_request(self, req_id: str) -> None: self.in_progress_prompt_logprobs.pop(req_id, None) + self.token_id_scores.pop(req_id, None) + + def compute_prompt_token_id_logprobs( + self, + logits_fn: Callable[[torch.Tensor], torch.Tensor], + hidden_states: torch.Tensor, + input_batch: InputBatch, + prompt_lens: np.ndarray, + ) -> dict[str, torch.Tensor]: + """Fill scores row (prompt row - req.start) per chunk; emit on the last.""" + if not self.token_id_scores: + return {} + logits_mode = self.logprobs_mode in ("raw_logits", "processed_logits") + out: dict[str, torch.Tensor] = {} + for i, req_id in enumerate(input_batch.req_ids): + req = self.token_id_scores.get(req_id) + if req is None: + continue + prompt_len = int(prompt_lens[input_batch.idx_mapping_np[i]]) + chunk_start = int(input_batch.num_computed_prefill_tokens_np[i]) + chunk_end = chunk_start + int(input_batch.num_scheduled_tokens[i]) + # Skip decode steps and, as compute_prompt_logprobs does, requests + # resumed after preemption: their scores were already emitted. + if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]: + continue + if req.scores is None: + if chunk_start > req.start: + # This prefill starts past the first scored row, so those + # rows will never be written; drop the request instead of + # emitting a buffer with unwritten rows. + self.token_id_scores.pop(req_id, None) + continue + req.scores = hidden_states.new_empty( + (max(prompt_len - 1 - req.start, 0), len(req.token_ids)), + dtype=torch.float32, + ) + # The last prompt row predicts the first decode token; skip it. + lo = max(req.start, chunk_start) + hi = min(chunk_end, prompt_len - 1) + base = int(input_batch.query_start_loc_np[i]) - chunk_start + for a in range(lo, hi, CHUNK_SIZE): + b = min(a + CHUNK_SIZE, hi) + logits = logits_fn(hidden_states[base + a : base + b]) + ids = req.token_ids.expand(b - a, -1) + req.scores[a - req.start : b - req.start] = ( + logits.gather(-1, ids) + if logits_mode + else compute_token_logprobs(logits, ids) + ) + if chunk_end >= prompt_len: + out[req_id] = req.scores + req.scores = None + return out def compute_prompt_logprobs( self, @@ -205,7 +287,6 @@ def compute_prompt_logprobs_with_chunking( ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: # Since materializing the full prompt logits can take too much memory, # we compute it in chunks. - CHUNK_SIZE = 1024 token_ids = [] scores = [] ranks = []