[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
+64
View File
@@ -0,0 +1,64 @@
# Prompt Token ID Logprobs (Teacher Scoring)
In on-policy distillation, the student records its top-K candidate tokens at
every response position, and the teacher provides the log probabilities of
exactly those candidates as the distillation target. The caller passes the
candidates as `prompt_logprob_token_ids`; the teacher runs a single prefill
over the prompt plus response and gathers their log probabilities on the GPU at
every scored row, starting from `prompt_logprob_start` (typically the prompt
length minus one, so only response positions are scored). The result is a
`[rows, K]` matrix whose row `i` scores prompt token
`prompt_logprob_start + i + 1` and whose columns follow the candidate order, so
it lines up position by position with the student's top-K.
## Quick start
```python
from vllm import LLM, SamplingParams
from vllm.inputs import TokensPrompt
llm = LLM(model)
output = llm.generate(
TokensPrompt(prompt_token_ids=prompt_ids + response_ids),
SamplingParams(
max_tokens=1,
prompt_logprob_token_ids=candidate_ids,
prompt_logprob_start=len(prompt_ids) - 1,
),
)
scores = output[0].prompt_token_id_logprobs
# scores.shape == (len(response_ids), len(candidate_ids))
# scores[i, j] = log p(candidate_ids[j] | prompt_ids + response_ids[:i])
```
The same parameters are accepted by the `/inference/v1/generate` HTTP endpoint
(Python and Rust frontends), which returns the matrix as a base64-encoded
`.npy` float32 array:
```python
import base64, io
import numpy as np
scores = np.load(io.BytesIO(base64.b64decode(response["prompt_token_id_logprobs"])))
```
The matrix has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows. With a
logits `--logprobs-mode`, it holds logits instead of log probabilities.
## Requirements
- The V2 model runner (`VLLM_USE_V2_MODEL_RUNNER=1`).
- `len(prompt_logprob_token_ids)` at most `--max-logprobs`.
- No `--kv-sharing-fast-prefill`.
The request skips reading the prefix cache (local and KV connector), since
cached rows have no logits to score.
## Limitations
- One candidate set per request, shared by every scored row; per-position
candidates are planned ([#56860](https://github.com/vllm-project/vllm/issues/56860)).
- Non-streaming only; `stream=true` requests are rejected.
- Not exposed through the OpenAI-compatible endpoints.
- If a prefill starts past the first scored row (e.g. `skip_reading_prefix_cache=False`
with a cache hit), the result is `None` rather than a partial matrix.
+1
View File
@@ -5968,6 +5968,7 @@ dependencies = [
"asynk-strim-attr",
"auto_enums",
"axum",
"base64 0.22.1",
"bytes",
"clap",
"educe",
@@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
use crate::protocol::tensor::WireNdArray;
use crate::protocol::{OpaqueValue, decode_msgpack};
/// The stop reason associated with a finished output.
@@ -133,6 +134,10 @@ pub struct EngineCoreOutput {
/// output when `--per-request-spec-decode-metrics` is enabled.
#[serde(default)]
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
/// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on
/// the first output of a request that asked for them.
#[serde(default)]
pub prompt_token_id_logprobs: Option<WireNdArray>,
}
/// Raw per-sequence speculative-decoding accumulator.
@@ -180,6 +185,11 @@ impl EngineCoreOutput {
self.new_sampling_mask = (self.new_sampling_mask.take())
.map(|value| value.resolve(frames, "new_sampling_mask"))
.transpose()?;
if let Some(value) = self.prompt_token_id_logprobs.as_mut() {
value
.resolve_aux_frame(frames)
.map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?;
}
Ok(())
}
}
@@ -531,6 +541,7 @@ mod tests {
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
prompt_token_id_logprobs: None,
},
],
scheduler_stats: None,
@@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling.
pub min_p: f32,
/// Frequency penalty applied by the sampler.
@@ -171,6 +176,8 @@ impl EngineCoreSamplingParams {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
prompt_token_id_logprobs: None,
},
],
scheduler_stats: None,
@@ -116,6 +116,7 @@ class EngineCoreOutput(
mm_cache_miss_hashes: list[str] | None = None
new_sampling_mask: object | None = None
spec_decode_metrics: object | None = None
prompt_token_id_logprobs: object | None = None
class ExtendedEngineCoreOutput(EngineCoreOutput):
+16
View File
@@ -15,6 +15,7 @@ use vllm_engine_core_client::protocol::output::{
EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason,
};
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
use crate::error::Result;
@@ -38,6 +39,7 @@ pub struct CollectedGenerateOutput {
pub request_id: String,
pub prompt_token_ids: Vec<u32>,
pub prompt_logprobs: Option<Logprobs>,
pub prompt_token_id_logprobs: Option<WireNdArray>,
pub token_ids: Vec<u32>,
pub logprobs: Option<Logprobs>,
pub finish_reason: FinishReason,
@@ -62,6 +64,8 @@ pub struct GeneratePromptInfo {
/// Prompt logprobs returned by engine-core for scored prompt positions,
/// when requested.
pub prompt_logprobs: Option<Logprobs>,
/// Log probabilities of `prompt_logprob_token_ids`, when requested.
pub prompt_token_id_logprobs: Option<WireNdArray>,
}
/// The reason a request finished.
@@ -206,6 +210,7 @@ impl GenerateOutput {
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: ids,
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids,
logprobs: None,
@@ -247,6 +252,7 @@ impl GenerateOutputStream {
pending_prompt_info: Some(GeneratePromptInfo {
prompt_token_ids,
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
raw_stream,
request_metrics,
@@ -282,6 +288,11 @@ impl Stream for GenerateOutputStream {
info.prompt_logprobs =
raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap());
}
if let Some(info) = &mut self.pending_prompt_info
&& info.prompt_token_id_logprobs.is_none()
{
info.prompt_token_id_logprobs = raw.prompt_token_id_logprobs;
}
let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap());
let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap());
@@ -361,6 +372,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
pin_mut!(stream);
let mut prompt_token_ids = None;
let mut prompt_logprobs = None;
let mut prompt_token_id_logprobs = None;
let mut cached_token_count = 0;
let mut collected: Option<CollectedGenerateOutput> = None;
@@ -374,6 +386,9 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
if prompt_logprobs.is_none() {
prompt_logprobs = info.prompt_logprobs;
}
if prompt_token_id_logprobs.is_none() {
prompt_token_id_logprobs = info.prompt_token_id_logprobs;
}
}
if let Some(existing) = collected.as_mut() {
@@ -393,6 +408,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
request_id: output.request_id,
prompt_token_ids: prompt_token_ids.take().unwrap_or_default(),
prompt_logprobs: prompt_logprobs.take(),
prompt_token_id_logprobs: prompt_token_id_logprobs.take(),
token_ids: output.token_ids,
logprobs: output.logprobs,
finish_reason: FinishReason::Error,
+2
View File
@@ -277,6 +277,7 @@ async fn generate_streams_outputs() {
Some(GeneratePromptInfo {
prompt_token_ids: vec![11, 22].into(),
prompt_logprobs: Some(prompt_logprobs()),
prompt_token_id_logprobs: None,
})
);
assert_eq!(first.token_ids, vec![1, 2]);
@@ -405,6 +406,7 @@ async fn collect_output_rejects_partial_sampling_mask() {
prompt_info: Some(GeneratePromptInfo {
prompt_token_ids: Arc::from([11_u32, 22]),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids: vec![33, 44],
logprobs: None,
+1
View File
@@ -9,6 +9,7 @@ anyhow.workspace = true
asynk-strim-attr.workspace = true
auto_enums.workspace = true
axum.workspace = true
base64.workspace = true
educe.workspace = true
futures.workspace = true
http-body.workspace = true
@@ -16,12 +16,15 @@ use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::Event;
use axum::response::{IntoResponse, Response};
use base64::Engine as _;
use futures::{Stream, StreamExt as _, pin_mut};
use thiserror_ext::AsReport as _;
use tracing::{error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::dtype::TensorDtype;
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_llm::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
};
@@ -330,6 +333,11 @@ fn collect_generate(
token_ids: collected.token_ids,
}],
prompt_logprobs,
prompt_token_id_logprobs: collected
.prompt_token_id_logprobs
.as_ref()
.map(npy_base64)
.transpose()?,
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
kv_transfer_params: collected.kv_transfer_params,
@@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, Generate
.collect()
}
/// Encode a float32 matrix as Python `numpy2base64` does.
fn npy_base64(array: &WireNdArray) -> Result<String, ApiError> {
let (TensorDtype::F32, Some(data), [rows, cols]) = (
array.dtype.scalar,
array.data.as_raw_view(),
array.shape.as_slice(),
) else {
bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix");
};
let mut header =
format!("{{'descr': '<f4', 'fortran_order': False, 'shape': ({rows}, {cols}), }}");
header.push_str(&" ".repeat(21 - rows.to_string().len()));
header.push_str(&" ".repeat(64 - (11 + header.len()) % 64));
header.push('\n');
let mut npy = Vec::with_capacity(10 + header.len() + data.len());
npy.extend_from_slice(b"\x93NUMPY\x01\x00");
npy.extend_from_slice(&(header.len() as u16).to_le_bytes());
npy.extend_from_slice(header.as_bytes());
npy.extend_from_slice(data);
Ok(base64::engine::general_purpose::STANDARD.encode(npy))
}
fn format_token_id(token_id: u32) -> String {
format!("token_id:{token_id}")
}
@@ -509,6 +540,7 @@ mod tests {
prompt_info: Some(GeneratePromptInfo {
prompt_token_ids: Arc::from([11_u32, 22_u32]),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids: vec![33],
logprobs: None,
@@ -592,6 +624,7 @@ mod tests {
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: Arc::from(ids),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids,
logprobs: None,
@@ -747,6 +780,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -781,6 +815,17 @@ mod tests {
assert!(response.mm_placeholders.is_none());
}
#[test]
fn prompt_token_id_logprobs_match_numpy2base64() {
let scores =
WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap();
// numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32))
assert_eq!(
npy_base64(&scores).unwrap(),
"k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA="
);
}
#[test]
fn collect_generate_returns_spec_decode_metrics() {
let metrics = RequestSpecDecodeMetrics {
@@ -792,6 +837,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-metrics".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -879,6 +925,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -914,6 +961,7 @@ mod tests {
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![3],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -94,6 +94,8 @@ pub(super) struct GenerateResponse {
pub request_id: String,
pub choices: Vec<GenerateResponseChoice>,
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
/// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it.
pub prompt_token_id_logprobs: Option<String>,
pub prompt_token_ids: Option<Vec<u32>>,
pub mm_placeholders: Option<MultiModalPlaceholders>,
pub kv_transfer_params: Option<Value>,
@@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat(
}
}
if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() {
bail_invalid_request!(
param = "sampling_params",
"`prompt_logprob_token_ids` are not available when `stream=true`."
);
}
Ok(())
}
@@ -169,6 +176,17 @@ mod tests {
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() {
let request: GenerateRequest = serde_json::from_value(json!({
"token_ids": [11, 22],
"stream": true,
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
}
#[test]
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
let request: GenerateRequest = serde_json::from_value(json!({
@@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request(
thinking_token_budget: request.thinking_token_budget,
logprobs: request.logprobs.then_some(top_logprobs),
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
@@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request(
thinking_token_budget: request.thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
+56
View File
@@ -103,6 +103,8 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_start,
min_p,
frequency_penalty,
presence_penalty,
@@ -123,6 +125,8 @@ pub fn lower_sampling_params(
logprobs,
prompt_logprobs,
logprob_token_ids.as_deref(),
prompt_logprob_token_ids.as_deref(),
prompt_logprob_start,
sampling_limits,
)?;
validate_repetition_detection(repetition_detection.as_ref())?;
@@ -176,6 +180,8 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_start,
min_p,
frequency_penalty,
presence_penalty,
@@ -646,6 +652,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -701,6 +709,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -870,6 +880,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -935,6 +947,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -1008,6 +1022,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -1122,6 +1138,44 @@ mod tests {
));
}
#[test]
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
lower_sampling_params_with_limits(
SamplingParams {
prompt_logprob_token_ids: ids,
prompt_logprob_start: start,
..Default::default()
},
sample_sampling_limits(),
)
};
let rejects = |ids, start| match lower(ids, start) {
Err(error) => error.to_string(),
Ok(_) => panic!("expected rejection"),
};
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
assert_eq!(params.prompt_logprob_start, Some(1));
expect_test::expect![[r#"
[
"prompt_logprob_token_ids must not be empty.",
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
"prompt_logprob_token_ids must not contain duplicates.",
"prompt_logprob_start requires prompt_logprob_token_ids.",
]
"#]]
.assert_debug_eq(&[
rejects(Some(vec![]), None),
rejects(Some((0..21).collect()), None),
rejects(Some(vec![1000]), None),
rejects(Some(vec![1, 1]), None),
rejects(None, Some(0)),
]);
}
#[test]
fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() {
let error = lower_sampling_params_with_limits(
@@ -1259,6 +1313,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
+28
View File
@@ -6,6 +6,7 @@
//! `-1` is expanded only for bounds checks. The original request values are
//! passed through to engine-core.
use itertools::Itertools as _;
use thiserror::Error;
use crate::backend::SamplingLimits;
@@ -35,6 +36,12 @@ pub enum LogprobsError {
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
)]
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
#[error("prompt_logprob_token_ids must not be empty.")]
EmptyPromptLogprobTokenIds,
#[error("prompt_logprob_token_ids must not contain duplicates.")]
DuplicatePromptLogprobTokenIds,
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
PromptLogprobStartWithoutTokenIds,
}
/// Validate logprobs count sampling parameters.
@@ -42,6 +49,8 @@ pub(super) fn validate_logprobs(
logprobs: Option<i32>,
prompt_logprobs: Option<i32>,
logprob_token_ids: Option<&[u32]>,
prompt_logprob_token_ids: Option<&[u32]>,
prompt_logprob_start: Option<u32>,
sampling_limits: SamplingLimits,
) -> Result<(), LogprobsError> {
let vocab_size = sampling_limits.model_vocab_size;
@@ -50,9 +59,28 @@ pub(super) fn validate_logprobs(
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
validate_logprobs_count(
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
max_logprobs,
vocab_size,
"prompt_logprob_token_ids",
)?;
validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?;
validate_logprob_token_ids(logprobs, logprob_token_ids)
}
fn validate_prompt_logprob_token_ids(
token_ids: Option<&[u32]>,
start: Option<u32>,
) -> Result<(), LogprobsError> {
match token_ids {
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
_ => Ok(()),
}
}
fn validate_logprobs_count(
requested: Option<i32>,
max_logprobs: usize,
+8
View File
@@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range(
)?;
}
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
validate_param(
"prompt_logprob_token_ids",
token_ids.iter().copied(),
limits.model_vocab_size,
)?;
}
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
validate_param(
"bad_words",
+7
View File
@@ -97,6 +97,11 @@ pub struct SamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling. `None` means no
/// explicit user override.
pub min_p: Option<f32>,
@@ -156,6 +161,8 @@ impl Default for SamplingParams {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: None,
frequency_penalty: None,
presence_penalty: None,
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import json
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from transformers import AutoTokenizer
@@ -311,6 +314,52 @@ async def test_generate_logprobs(client, logprobs_value):
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
candidates = [22, 33, 44, 55]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": candidates,
"prompt_logprob_start": 1,
}
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": sampling_params,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (len(token_ids) - 2, len(candidates))
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, candidates.index(target)] == pytest.approx(
expected, abs=1e-3
)
stream_only_ids = {"prompt_logprob_token_ids": candidates}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
)
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
)
resp.raise_for_status()
empty = np.load(
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
)
assert empty.shape == (0, len(candidates))
@pytest.mark.asyncio
async def test_same_response_as_chat_completions(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
@@ -11,8 +11,10 @@ row), the trailing positions are populated with sentinel values
"""
import numpy as np
import torch
from vllm.logprobs import create_sample_logprobs
from vllm.v1.engine import EngineCoreOutput
from vllm.v1.engine.logprobs import LogprobsProcessor
from vllm.v1.outputs import LogprobsLists
@@ -64,3 +66,24 @@ def test_accepts_exactly_sized_row():
pos = processor.logprobs[0]
assert set(pos.keys()) == {7, 11, 13}
def test_prompt_token_id_logprobs_are_popped_once():
"""DELTA outputs carry fixed-ID prompt scores exactly once."""
processor = _make_processor(num_logprobs=1)
assert processor.pop_prompt_token_id_logprobs() is None
processor.update_from_output(
EngineCoreOutput(
request_id="req-0",
new_token_ids=[],
prompt_token_id_logprobs=torch.tensor(
[[-0.5, -1.5], [-2.5, -3.5]], dtype=torch.float32
),
)
)
scores = processor.pop_prompt_token_id_logprobs()
assert scores is not None
assert scores.tolist() == [[-0.5, -1.5], [-2.5, -3.5]]
assert processor.pop_prompt_token_id_logprobs() is None
+177 -2
View File
@@ -6,7 +6,9 @@ import math
from collections.abc import Generator
from types import SimpleNamespace
from typing import get_args
from unittest.mock import patch
import numpy as np
import pytest
import torch
@@ -23,6 +25,7 @@ from vllm.config.model import LogprobsMode
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
from vllm.platforms import current_platform
from vllm.v1.engine.input_processor import InputProcessor
from ...conftest import HfRunner, VllmRunner
@@ -81,9 +84,9 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
yield hf_model
def _model_config(vocab_size: int = 10):
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
return SimpleNamespace(
max_logprobs=20,
max_logprobs=max_logprobs,
logits_processors=None,
is_diffusion=False,
get_vocab_size=lambda: vocab_size,
@@ -432,6 +435,57 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
)
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
"""The candidate count is bounded by max_logprobs."""
model_config = _model_config(vocab_size=100, max_logprobs=4)
def verify(**kwargs):
SamplingParams(**kwargs).verify(
model_config,
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
)
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
# The error names the value to raise max_logprobs to.
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
verify(prompt_logprob_start=3)
def test_prompt_logprob_token_ids_require_v2_model_runner():
"""Only the V2 runner scores them; the V1 runner would return None, and
kv-sharing fast prefill would score rows the cross-decoder never ran."""
processor = SimpleNamespace(
model_config=SimpleNamespace(
return_sampling_mask=False, enable_trace_replay=False, is_diffusion=False
),
vllm_config=SimpleNamespace(
reasoning_config=None,
use_v2_model_runner=False,
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False),
),
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
validate_logits_processors_params=lambda params: None,
)
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
with patch.object(SamplingParams, "verify"):
with pytest.raises(VLLMValidationError, match="V2 model runner"):
InputProcessor._validate_params(processor, params, ("generate",))
processor.vllm_config.use_v2_model_runner = True
InputProcessor._validate_params(processor, params, ("generate",))
processor.vllm_config.cache_config.kv_sharing_fast_prefill = True
with pytest.raises(VLLMValidationError, match="fast-prefill"):
InputProcessor._validate_params(processor, params, ("generate",))
def test_none_logprobs(vllm_model, example_prompts):
"""Engine should return `logprobs` and `prompt_logprobs` as `None`.
@@ -1298,6 +1352,127 @@ def test_prompt_logprobs_with_chunking_and_preemption():
print(f"Test passed with {preemptions} preemptions")
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = [
"The following numbers of the sequence "
+ ", ".join(str(i) for i in range(10))
+ " are:",
"In one word, the capital of France is ",
] + [f"Tell me about the number {i}: " for i in range(32)]
start = 2
candidate_ids = [10, 100, 1000, 10000]
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=40,
min_tokens=20,
prompt_logprob_token_ids=candidate_ids,
prompt_logprob_start=start,
)
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
enable_chunked_prefill=True,
max_num_batched_tokens=48, # Force prefill chunking
num_gpu_blocks_override=33, # Force preemptions (32 usable + null block)
disable_log_stats=False,
gpu_memory_utilization=0.25,
) as vllm_model:
metrics_before = vllm_model.llm.get_metrics()
outputs = vllm_model.llm.generate(prompts, sampling_params)
for i, output in enumerate(outputs):
scores = output.prompt_token_id_logprobs
assert scores is not None, f"Output {i} missing fixed-ID scores"
expected_shape = (
len(output.prompt_token_ids) - 1 - start,
len(candidate_ids),
)
assert scores.shape == expected_shape, (
f"Output {i} scored {scores.shape}, expected {expected_shape}"
)
assert math.isfinite(float(scores.min()))
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
metrics_after = vllm_model.llm.get_metrics()
preemptions_before = next(
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
)
preemptions_after = next(
(m.value for m in metrics_after if m.name == "vllm:num_preemptions"), 0
)
assert preemptions_after - preemptions_before > 0, (
"Test did not trigger any preemptions"
)
# Row alignment is numerical: chunked and preempted scores must match an
# unchunked, unpreempted run of the same requests. Batch composition moves
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
max_logprobs=64,
gpu_memory_utilization=0.25,
) as reference:
reference_outputs = reference.llm.generate(prompts, sampling_params)
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
self_scored = reference.llm.generate(
prompts,
[
SamplingParams(
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
)
for ids in own_ids
],
)
for output, ref in zip(outputs, reference_outputs):
np.testing.assert_allclose(
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
)
for output, ids in zip(self_scored, own_ids):
scores = output.prompt_token_id_logprobs
for row, target in enumerate(output.prompt_token_ids[1:]):
expected = output.prompt_logprobs[row + 1][target].logprob
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
"""A prefill that starts past the first scored row returns no scores.
Such a prefill (here a prefix-cache hit the caller opted back into) never
computes the leading rows, so emitting the buffer would return values the
model never produced for them.
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompt = "The capital of France is Paris. " * 20
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
prompt_logprob_token_ids=[10, 100, 1000],
skip_reading_prefix_cache=False,
)
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
enable_prefix_caching=True,
gpu_memory_utilization=0.25,
) as vllm_model:
first = vllm_model.llm.generate([prompt], sampling_params)[0]
assert first.prompt_token_id_logprobs is not None
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
# The re-send is served from the prefix cache, so its prefill starts
# past row 0 and the leading rows are never computed.
second = vllm_model.llm.generate([prompt], sampling_params)[0]
assert second.prompt_token_id_logprobs is None
@large_gpu_mark(min_gb=24)
def test_token_logprobs_large_batch_int64_row_offset():
"""Regression: logprob kernel row offset (row * vocab_size) must use int64.
@@ -41,6 +41,14 @@ def test_sampling_params_trace_field_rejects_empty_list():
)
def test_sampling_params_trace_field_rejects_prompt_scoring():
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
def test_sampling_params_trace_field_requires_single_output():
params = SamplingParams(n=2, trace_decode_token_ids=[1])
with pytest.raises(ValueError, match="requires n=1"):
@@ -381,6 +381,7 @@ class GenerateResponse(BaseModel):
choices: list[GenerateResponseChoice]
usage: UsageInfo | None = Field(default=None)
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
prompt_token_id_logprobs: str | None = None
prompt_token_ids: list[int] | None = None
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
metrics: PerRequestMetrics | None = None
@@ -138,6 +138,11 @@ class ServingTokens(GenerateBaseServing):
f"sampling_params.n must be at most the server's max_num_seqs "
f"({max_num_seqs}), got {sampling_params.n}."
)
# The stream schema has no field for the scores.
if request.stream and sampling_params.prompt_logprob_token_ids is not None:
return self.create_error_response(
"prompt_logprob_token_ids are not available when stream=true."
)
if self.force_no_detokenize and sampling_params.stop:
# SamplingParams rejects stop with detokenize=False at request
# validation, but this server forces detokenize=False afterwards,
@@ -389,6 +394,11 @@ class ServingTokens(GenerateBaseServing):
choices=choices,
usage=usage,
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
prompt_token_id_logprobs=(
numpy2base64(final_res.prompt_token_id_logprobs)
if final_res.prompt_token_id_logprobs is not None
else None
),
prompt_token_ids=(
final_res.prompt_token_ids if request.return_token_ids else None
),
+7
View File
@@ -117,6 +117,10 @@ class RequestOutput:
For encoder/decoder models, this is the
decoder input prompt token ids.
prompt_logprobs: The log probabilities to return per prompt token.
prompt_token_id_logprobs: Logprobs of prompt_logprob_token_ids, shaped
[max(prompt_len - 1 - prompt_logprob_start, 0),
len(prompt_logprob_token_ids)]; row i scores them as predictions of
prompt token prompt_logprob_start + i + 1.
outputs: The output sequences of the request.
finished: Whether the whole request is finished.
metrics: Metrics associated with the request.
@@ -148,6 +152,7 @@ class RequestOutput:
num_cached_tokens: int | None = None,
num_cache_creation_tokens: int | None = None,
*,
prompt_token_id_logprobs: np.ndarray | None = None,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
# Forward compatibility, code that uses args added in new release can
@@ -162,6 +167,7 @@ class RequestOutput:
self.prompt = prompt
self.prompt_token_ids = prompt_token_ids
self.prompt_logprobs = prompt_logprobs
self.prompt_token_id_logprobs = prompt_token_id_logprobs
self.outputs = outputs
self.finished = finished
self.metrics = metrics
@@ -215,6 +221,7 @@ class RequestOutput:
f"encoder_prompt={self.encoder_prompt!r}, "
f"encoder_prompt_token_ids={self.encoder_prompt_token_ids}, "
f"prompt_logprobs={self.prompt_logprobs}, "
f"prompt_token_id_logprobs={self.prompt_token_id_logprobs}, "
f"outputs={self.outputs}, "
f"finished={self.finished}, "
f"metrics={self.metrics}, "
+68 -1
View File
@@ -289,6 +289,12 @@ class SamplingParams(
prompt_logprobs: int | None = None
"""Number of log probabilities to return per prompt token.
When set to -1, return all `vocab_size` log probabilities."""
prompt_logprob_token_ids: list[int] | None = None
"""Token IDs to score at each scored causal prompt row, where row i scores
them as predictions of prompt token i + 1. The last prompt row is excluded,
so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows."""
prompt_logprob_start: int | None = None
"""First causal prompt row to score; defaults to the first row."""
logprob_token_ids: list[int] | None = None
"""Specific token IDs to return logprobs for. More efficient than
logprobs=-1 when you only need logprobs for a small set of tokens.
@@ -398,6 +404,8 @@ class SamplingParams(
min_tokens: int = 0,
logprobs: int | None = None,
prompt_logprobs: int | None = None,
prompt_logprob_token_ids: list[int] | None = None,
prompt_logprob_start: int | None = None,
detokenize: bool = True,
skip_special_tokens: bool = True,
spaces_between_special_tokens: bool = True,
@@ -464,6 +472,8 @@ class SamplingParams(
min_tokens=min_tokens,
logprobs=logprobs,
prompt_logprobs=prompt_logprobs,
prompt_logprob_token_ids=prompt_logprob_token_ids,
prompt_logprob_start=prompt_logprob_start,
logprob_token_ids=logprob_token_ids,
detokenize=detokenize,
skip_special_tokens=skip_special_tokens,
@@ -550,7 +560,10 @@ class SamplingParams(
# If prefix caching is enabled,
# the output of prompt logprobs may less than n_prompt_tokens,
# we need to skip reading cache at this request.
self.skip_reading_prefix_cache = self.prompt_logprobs is not None
self.skip_reading_prefix_cache = (
self.prompt_logprobs is not None
or self.prompt_logprob_token_ids is not None
)
def _verify_args(self) -> None:
_verify_num_sequences(self.n, "n")
@@ -902,6 +915,56 @@ class SamplingParams(
value=num_prompt_logprobs,
)
# Validate prompt_logprob_token_ids.
if self.prompt_logprob_token_ids is not None:
n = len(self.prompt_logprob_token_ids)
if n == 0:
raise VLLMValidationError(
"prompt_logprob_token_ids must not be empty.",
parameter="prompt_logprob_token_ids",
value=n,
)
if n > max_logprobs:
raise VLLMValidationError(
f"Requested prompt_logprob_token_ids of length {n}, "
f"which is greater than max allowed: {max_logprobs}. "
f"Set max_logprobs (--max-logprobs) to at least {n}.",
parameter="prompt_logprob_token_ids",
value=n,
)
vocab_size = model_config.get_vocab_size()
invalid_token_ids = [
token_id
for token_id in self.prompt_logprob_token_ids
if token_id < 0 or token_id >= vocab_size
]
if invalid_token_ids:
raise VLLMValidationError(
f"token_id(s) {invalid_token_ids} in "
f"prompt_logprob_token_ids contain out-of-vocab token ids. "
f"Vocabulary size: {vocab_size}",
parameter="prompt_logprob_token_ids",
value=invalid_token_ids,
)
if len(set(self.prompt_logprob_token_ids)) != n:
raise VLLMValidationError(
"prompt_logprob_token_ids must not contain duplicates.",
parameter="prompt_logprob_token_ids",
value=self.prompt_logprob_token_ids,
)
if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0:
raise VLLMValidationError(
"prompt_logprob_start must be non-negative.",
parameter="prompt_logprob_start",
value=self.prompt_logprob_start,
)
elif self.prompt_logprob_start is not None:
raise VLLMValidationError(
"prompt_logprob_start requires prompt_logprob_token_ids.",
parameter="prompt_logprob_start",
value=self.prompt_logprob_start,
)
def _validate_stop_token_ids(self, model_config: ModelConfig) -> None:
"""Validate stop_token_ids are within vocabulary range."""
if not self.stop_token_ids:
@@ -969,6 +1032,10 @@ class SamplingParams(
raise ValueError(
"trace_decode_token_ids is not supported with prompt_logprobs."
)
if self.prompt_logprob_token_ids is not None:
raise ValueError(
"trace_decode_token_ids is not supported with prompt_logprob_token_ids."
)
if speculative_config is not None:
raise ValueError(
"trace_decode_token_ids is not supported with speculative decoding."
+6
View File
@@ -1972,6 +1972,9 @@ class Scheduler(SchedulerInterface):
sampled_token_ids = model_runner_output.sampled_token_ids
logprobs = model_runner_output.logprobs
prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict
prompt_token_id_logprobs_dict = (
model_runner_output.prompt_token_id_logprobs_dict
)
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
pooler_outputs = model_runner_output.pooler_output
num_nans_in_logits = model_runner_output.num_nans_in_logits
@@ -2194,6 +2197,7 @@ class Scheduler(SchedulerInterface):
# Get prompt logprobs for this request.
prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
prompt_token_id_logprobs = prompt_token_id_logprobs_dict.get(req_id)
if should_emit_output:
# Add EngineCoreOutput for this Request.
outputs[request.client_index].append(
@@ -2204,6 +2208,7 @@ class Scheduler(SchedulerInterface):
new_logprobs=new_logprobs,
new_sampling_mask=new_sampling_mask,
new_prompt_logprobs_tensors=prompt_logprobs_tensors,
prompt_token_id_logprobs=prompt_token_id_logprobs,
pooling_output=pooler_output,
stop_reason=request.stop_reason,
events=request.take_events(),
@@ -2223,6 +2228,7 @@ class Scheduler(SchedulerInterface):
else:
# Invariant: EngineCore returns no partial prefill outputs.
assert not prompt_logprobs_tensors
assert prompt_token_id_logprobs is None
# Remove the stopped requests from the running and waiting queues.
if stopped_running_reqs:
+3
View File
@@ -236,6 +236,9 @@ class EngineCoreOutput(
# Appended last so `array_like` positional serialization stays compatible.
spec_decode_metrics: RequestSpecDecodeMetrics | None = None
# [num_scored_rows, num_token_ids]; appended last for array_like compatibility.
prompt_token_id_logprobs: torch.Tensor | None = None
@property
def finished(self) -> bool:
return self.finish_reason is not None
+13
View File
@@ -131,6 +131,19 @@ class InputProcessor:
self.structured_outputs_config,
self.tokenizer,
)
if params.prompt_logprob_token_ids is not None:
if not self.vllm_config.use_v2_model_runner:
raise VLLMValidationError(
"prompt_logprob_token_ids requires the V2 model runner "
"(VLLM_USE_V2_MODEL_RUNNER=1).",
parameter="prompt_logprob_token_ids",
)
if self.vllm_config.cache_config.kv_sharing_fast_prefill:
raise VLLMValidationError(
"prompt_logprob_token_ids is incorrect with "
"--kv-sharing-fast-prefill; disable it for scoring.",
parameter="prompt_logprob_token_ids",
)
self.validate_logits_processors_params(params)
+12
View File
@@ -5,6 +5,8 @@ import itertools
from collections.abc import Iterable
from dataclasses import dataclass
import numpy as np
from vllm.logger import init_logger
from vllm.logprobs import (
FlatLogprobs,
@@ -38,6 +40,8 @@ class LogprobsProcessor:
cumulative_logprob: float | None
num_logprobs: int | None
num_prompt_logprobs: int | None
# [num_scored_rows, num_token_ids], set once on the final prefill chunk.
prompt_token_id_logprobs: np.ndarray | None = None
@classmethod
def from_new_request(
@@ -204,6 +208,12 @@ class LogprobsProcessor:
self.prompt_logprobs = []
return plp
def pop_prompt_token_id_logprobs(self) -> np.ndarray | None:
"""Pop and return the fixed-ID prompt scores."""
scores = self.prompt_token_id_logprobs
self.prompt_token_id_logprobs = None
return scores
@staticmethod
def _get_sampled_context_ids(
logprobs_source: SampleLogprobs | PromptLogprobs | None,
@@ -352,3 +362,5 @@ class LogprobsProcessor:
self._update_sample_logprobs(output.new_logprobs)
if output.new_prompt_logprobs_tensors is not None:
self._update_prompt_logprobs(output.new_prompt_logprobs_tensors)
if output.prompt_token_id_logprobs is not None:
self.prompt_token_id_logprobs = output.prompt_token_id_logprobs.numpy()
+5
View File
@@ -390,8 +390,12 @@ class RequestState:
if self.output_kind == RequestOutputKind.DELTA:
# Side effect: logprobs processor forgets prompt logprobs
prompt_logprobs = self.logprobs_processor.pop_prompt_logprobs()
prompt_token_id_logprobs = (
self.logprobs_processor.pop_prompt_token_id_logprobs()
)
else:
prompt_logprobs = self.logprobs_processor.prompt_logprobs
prompt_token_id_logprobs = self.logprobs_processor.prompt_token_id_logprobs
return RequestOutput(
request_id=external_req_id, # request_id is what was provided externally
@@ -399,6 +403,7 @@ class RequestState:
prompt=self.prompt,
prompt_token_ids=prompt_token_ids,
prompt_logprobs=prompt_logprobs,
prompt_token_id_logprobs=prompt_token_id_logprobs,
outputs=cast(list[CompletionOutput], outputs),
finished=finished,
kv_transfer_params=kv_transfer_params,
+3
View File
@@ -283,6 +283,9 @@ class ModelRunnerOutput:
default_factory=dict
)
# req_id -> [num_scored_rows, num_token_ids] prompt_logprob_token_ids scores.
prompt_token_id_logprobs_dict: dict[str, torch.Tensor] = field(default_factory=dict)
# [num_reqs, hidden_size]
pooler_output: list[torch.Tensor | None] | None = None
+7
View File
@@ -156,6 +156,10 @@ class AsyncOutput(AsyncModelRunnerOutput):
k: v.to_cpu_nonblocking() if v is not None else None
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
}
token_id_logprobs = self.model_runner_output.prompt_token_id_logprobs_dict
self.prompt_token_id_logprobs_dict = {
k: v.to("cpu", non_blocking=True) for k, v in token_id_logprobs.items()
}
if self.pending_aux_output is not None:
self.pending_aux_output.enqueue_cpu_copy(
num_sampled=self.num_sampled_tokens_np,
@@ -194,6 +198,9 @@ class AsyncOutput(AsyncModelRunnerOutput):
if self.logprobs_tensors is not None:
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
self.model_runner_output.prompt_token_id_logprobs_dict = (
self.prompt_token_id_logprobs_dict
)
if self.pending_aux_output is not None:
self.model_runner_output.aux_output_connector_output = (
self.pending_aux_output.process_output()
+12 -1
View File
@@ -494,7 +494,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
),
)
self.prompt_logprobs_worker = PromptLogprobsWorker(
self.max_num_reqs, logprobs_mode=self.model_config.logprobs_mode
self.max_num_reqs,
self.device,
logprobs_mode=self.model_config.logprobs_mode,
)
self.structured_outputs_worker = StructuredOutputsWorker(
max_num_logits=self.max_num_reqs * self.decode_query_len,
@@ -2063,6 +2065,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.num_computed_tokens.gpu,
self.req_states.prompt_len.np,
)
prompt_token_id_logprobs_dict = (
self.prompt_logprobs_worker.compute_prompt_token_id_logprobs(
self.model.compute_logits,
hidden_states,
input_batch,
self.req_states.prompt_len.np,
)
)
# Prepare the model runner output.
model_runner_output = ModelRunnerOutput(
@@ -2072,6 +2082,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
sampled_token_ids=None, # type: ignore
prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
prompt_token_id_logprobs_dict=prompt_token_id_logprobs_dict,
cudagraph_stats=cudagraph_stats,
)
pending_aux_output = None
+3 -1
View File
@@ -19,6 +19,7 @@ def _topk_log_softmax_kernel(
logits_ptr,
logits_stride,
topk_ids_ptr,
topk_ids_stride,
topk,
vocab_size,
BLOCK_SIZE: tl.constexpr,
@@ -49,7 +50,7 @@ def _topk_log_softmax_kernel(
k_offset = j + tl.arange(0, TOPK_BLOCK_SIZE)
k_mask = k_offset < topk
topk_ids = tl.load(
topk_ids_ptr + req_idx * topk + k_offset, mask=k_mask, other=0
topk_ids_ptr + req_idx * topk_ids_stride + k_offset, mask=k_mask, other=0
)
logits = tl.load(row_ptr + topk_ids, mask=k_mask)
logits = logits.to(tl.float32)
@@ -98,6 +99,7 @@ def compute_token_logprobs(
logits,
logits.stride(0),
token_ids,
token_ids.stride(0),
num_logprobs,
vocab_size,
BLOCK_SIZE=1024, # type: ignore
+84 -3
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass
import numpy as np
import torch
@@ -8,20 +9,41 @@ import torch
from vllm.config.model import LogprobsMode
from vllm.sampling_params import SamplingParams
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
from vllm.v1.worker.gpu.sample.logprob import (
compute_token_logprobs,
compute_topk_scores,
)
CHUNK_SIZE = 1024
@dataclass
class _TokenIdScores:
token_ids: torch.Tensor # [num_ids], on device
start: int # first prompt row to score
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
class PromptLogprobsWorker:
def __init__(self, max_num_reqs: int, logprobs_mode: LogprobsMode = "raw_logprobs"):
def __init__(
self,
max_num_reqs: int,
device: torch.device,
logprobs_mode: LogprobsMode = "raw_logprobs",
):
self.max_num_reqs = max_num_reqs
self.device = device
self.logprobs_mode = logprobs_mode
self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool)
self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32)
# req_idx -> list of in-progress LogprobsTensors
self.in_progress_prompt_logprobs: dict[str, list[LogprobsTensors]] = {}
# req_id -> fixed-ID scoring state
self.token_id_scores: dict[str, _TokenIdScores] = {}
def add_request(self, req_id: str, req_idx: int, sampling_params: SamplingParams):
uses_prompt_logprobs = sampling_params.prompt_logprobs is not None
@@ -29,9 +51,69 @@ class PromptLogprobsWorker:
self.num_prompt_logprobs[req_idx] = sampling_params.prompt_logprobs or 0
if uses_prompt_logprobs:
self.in_progress_prompt_logprobs[req_id] = []
if sampling_params.prompt_logprob_token_ids is not None:
self.token_id_scores[req_id] = _TokenIdScores(
async_tensor_h2d(
sampling_params.prompt_logprob_token_ids, self.device, torch.int64
),
sampling_params.prompt_logprob_start or 0,
)
def remove_request(self, req_id: str) -> None:
self.in_progress_prompt_logprobs.pop(req_id, None)
self.token_id_scores.pop(req_id, None)
def compute_prompt_token_id_logprobs(
self,
logits_fn: Callable[[torch.Tensor], torch.Tensor],
hidden_states: torch.Tensor,
input_batch: InputBatch,
prompt_lens: np.ndarray,
) -> dict[str, torch.Tensor]:
"""Fill scores row (prompt row - req.start) per chunk; emit on the last."""
if not self.token_id_scores:
return {}
logits_mode = self.logprobs_mode in ("raw_logits", "processed_logits")
out: dict[str, torch.Tensor] = {}
for i, req_id in enumerate(input_batch.req_ids):
req = self.token_id_scores.get(req_id)
if req is None:
continue
prompt_len = int(prompt_lens[input_batch.idx_mapping_np[i]])
chunk_start = int(input_batch.num_computed_prefill_tokens_np[i])
chunk_end = chunk_start + int(input_batch.num_scheduled_tokens[i])
# Skip decode steps and, as compute_prompt_logprobs does, requests
# resumed after preemption: their scores were already emitted.
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
continue
if req.scores is None:
if chunk_start > req.start:
# This prefill starts past the first scored row, so those
# rows will never be written; drop the request instead of
# emitting a buffer with unwritten rows.
self.token_id_scores.pop(req_id, None)
continue
req.scores = hidden_states.new_empty(
(max(prompt_len - 1 - req.start, 0), len(req.token_ids)),
dtype=torch.float32,
)
# The last prompt row predicts the first decode token; skip it.
lo = max(req.start, chunk_start)
hi = min(chunk_end, prompt_len - 1)
base = int(input_batch.query_start_loc_np[i]) - chunk_start
for a in range(lo, hi, CHUNK_SIZE):
b = min(a + CHUNK_SIZE, hi)
logits = logits_fn(hidden_states[base + a : base + b])
ids = req.token_ids.expand(b - a, -1)
req.scores[a - req.start : b - req.start] = (
logits.gather(-1, ids)
if logits_mode
else compute_token_logprobs(logits, ids)
)
if chunk_end >= prompt_len:
out[req_id] = req.scores
req.scores = None
return out
def compute_prompt_logprobs(
self,
@@ -205,7 +287,6 @@ def compute_prompt_logprobs_with_chunking(
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# Since materializing the full prompt logits can take too much memory,
# we compute it in chunks.
CHUNK_SIZE = 1024
token_ids = []
scores = []
ranks = []