[Feature] Per-row candidate IDs for prefill token scoring (M2 of #56860) (#56984)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-03 14:17:09 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent bc21cba967
commit e319f86f15
17 changed files with 366 additions and 162 deletions
+18 -16
View File
@@ -2,14 +2,14 @@
In on-policy distillation, the student records its top-K candidate tokens at
every response position, and the teacher provides the log probabilities of
exactly those candidates as the distillation target. The caller passes the
candidates as `prompt_logprob_token_ids`; the teacher runs a single prefill
over the prompt plus response and gathers their log probabilities on the GPU at
every scored row, starting from `prompt_logprob_start` (typically the prompt
length minus one, so only response positions are scored). The result is a
`[rows, K]` matrix whose row `i` scores prompt token
`prompt_logprob_start + i + 1` and whose columns follow the candidate order, so
it lines up position by position with the student's top-K.
exactly those candidates as the distillation target. The caller passes one
candidate list per scored row as `prompt_logprob_token_ids`; the teacher runs a
single prefill over the prompt plus response and gathers their log
probabilities on the GPU, starting from `prompt_logprob_start` (typically the
prompt length minus one, so only response positions are scored). The result is
a `[rows, K]` matrix whose row `i` scores prompt token
`prompt_logprob_start + i + 1` and whose columns follow that row's candidate
order, so it lines up position by position with the student's top-K.
## Quick start
@@ -22,13 +22,13 @@ output = llm.generate(
TokensPrompt(prompt_token_ids=prompt_ids + response_ids),
SamplingParams(
max_tokens=1,
prompt_logprob_token_ids=candidate_ids,
prompt_logprob_token_ids=student_topk_ids, # [len(response_ids), K]
prompt_logprob_start=len(prompt_ids) - 1,
),
)
scores = output[0].prompt_token_id_logprobs
# scores.shape == (len(response_ids), len(candidate_ids))
# scores[i, j] = log p(candidate_ids[j] | prompt_ids + response_ids[:i])
# scores.shape == (len(response_ids), K)
# scores[i, j] = log p(student_topk_ids[i][j] | prompt_ids + response_ids[:i])
```
The same parameters are accepted by the `/inference/v1/generate` HTTP endpoint
@@ -42,13 +42,17 @@ import numpy as np
scores = np.load(io.BytesIO(base64.b64decode(response["prompt_token_id_logprobs"])))
```
The matrix has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows. With a
logits `--logprobs-mode`, it holds logits instead of log probabilities.
`prompt_logprob_token_ids` is an integer array or a list of lists with exactly
`prompt_len - 1 - prompt_logprob_start` rows; shorter rows are padded with `-1`,
and every `-1` entry scores `-inf`. Pass an `int32` NumPy array, such as the
student's top-K IDs, which is sent to the engine without conversion; nested
lists of Python ints are much slower to copy and serialize. With a logits
`--logprobs-mode`, the matrix holds logits instead of log probabilities.
## Requirements
- The V2 model runner (`VLLM_USE_V2_MODEL_RUNNER=1`).
- `len(prompt_logprob_token_ids)` at most `--max-logprobs`.
- The longest row at most `--max-logprobs` candidates.
- No `--kv-sharing-fast-prefill`.
The request skips reading the prefix cache (local and KV connector), since
@@ -56,8 +60,6 @@ cached rows have no logits to score.
## Limitations
- One candidate set per request, shared by every scored row; per-position
candidates are planned ([#56860](https://github.com/vllm-project/vllm/issues/56860)).
- Non-streaming only; `stream=true` requests are rejected.
- Not exposed through the OpenAI-compatible endpoints.
- If a prefill starts past the first scored row (e.g. `skip_reading_prefix_cache=False`
@@ -152,6 +152,11 @@ impl EngineCoreRequest {
feature.extract_aux_frames(&mut aux_frames, threshold);
}
}
if let Some(params) = &mut self.sampling_params
&& let Some(ids) = &mut params.prompt_logprob_token_ids
{
ids.extract_aux_frame(&mut aux_frames, threshold);
}
aux_frames
}
}
@@ -168,7 +173,7 @@ mod tests {
PlaceholderRange,
};
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::tensor::{WireArrayData, WireTensor};
use crate::protocol::tensor::{WireArrayData, WireNdArray, WireTensor};
use crate::protocol::{decode_value, encode_msgpack};
const AUX_FRAME_THRESHOLD: usize = 256;
@@ -294,4 +299,27 @@ mod tests {
);
assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty());
}
#[test]
fn engine_core_request_extracts_large_prompt_logprob_token_ids() {
let ids = WireNdArray::from_i32(vec![AUX_FRAME_THRESHOLD, 1], vec![5; AUX_FRAME_THRESHOLD])
.unwrap();
let mut request = EngineCoreRequest {
sampling_params: Some(EngineCoreSamplingParams {
prompt_logprob_token_ids: Some(ids),
..EngineCoreSamplingParams::for_test()
}),
..EngineCoreRequest::default()
};
let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD);
assert_eq!(aux_frames.len(), 1);
assert_eq!(aux_frames[0].len(), AUX_FRAME_THRESHOLD * 4);
let params = request.sampling_params.unwrap();
assert_eq!(
params.prompt_logprob_token_ids.unwrap().data,
WireArrayData::AuxIndex(1)
);
}
}
@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use crate::protocol::structured_outputs::StructuredOutputsParams;
use crate::protocol::tensor::WireNdArray;
fn default_top_p() -> f32 {
1.0
@@ -101,9 +102,9 @@ pub struct EngineCoreSamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// `[num_rows, num_ids]` candidate token IDs per scored causal prompt row,
/// `-1` padding shorter rows.
pub prompt_logprob_token_ids: Option<WireNdArray>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling.
@@ -121,6 +121,18 @@ impl<Dtype: From<TensorDtype> + Serialize + DeserializeOwned> WireArray<Dtype> {
))
}
/// Build an int32 tensor/ndarray backed by native-endian raw-view bytes.
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_i32(shape: Vec<usize>, data: Vec<i32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self::from_raw_bytes(
TensorDtype::I32,
shape,
bytes_from_pod_vec(data),
))
}
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
///
/// Takes ownership of the backing buffer without copying its data.
@@ -181,7 +181,7 @@ mod tests {
let request: GenerateRequest = serde_json::from_value(json!({
"token_ids": [11, 22],
"stream": true,
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
"sampling_params": {"prompt_logprob_token_ids": [[1, 2]]}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
+33 -16
View File
@@ -7,7 +7,7 @@ pub(crate) mod logprobs;
pub(crate) mod sampling;
pub(crate) mod token_ids;
use logprobs::validate_logprobs;
use logprobs::{lower_prompt_logprob_token_ids, validate_logprobs};
use sampling::validate_resolved_sampling_params;
use token_ids::{validate_prompt_token_ids, validate_vocab_range};
use vllm_engine_core_client::protocol::sampling::{
@@ -50,7 +50,11 @@ pub fn lower_text_request(
// multimodal tensor payloads.
mm_features: request.mm_features.take(),
sampling_params: lower_sampling_params(
request.sampling_params.clone(),
SamplingParams {
// Move the candidate table rather than cloning it; only lowering reads it.
prompt_logprob_token_ids: request.sampling_params.prompt_logprob_token_ids.take(),
..request.sampling_params.clone()
},
sampling_hints,
sampling_limits,
prompt_len,
@@ -180,7 +184,12 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_token_ids: lower_prompt_logprob_token_ids(
prompt_logprob_token_ids,
prompt_logprob_start,
prompt_len,
sampling_limits.model_vocab_size,
)?,
prompt_logprob_start,
min_p,
frequency_penalty,
@@ -332,6 +341,7 @@ mod tests {
use vllm_engine_core_client::protocol::multimodal::{
MmFeatureSpec, MmModality, PlaceholderRange,
};
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
@@ -1140,7 +1150,7 @@ mod tests {
#[test]
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
let lower = |ids: Option<Vec<Vec<i32>>>, start: Option<u32>| {
lower_sampling_params_with_limits(
SamplingParams {
prompt_logprob_token_ids: ids,
@@ -1155,23 +1165,30 @@ mod tests {
Ok(_) => panic!("expected rejection"),
};
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
assert_eq!(params.prompt_logprob_start, Some(1));
let params = lower(Some(vec![vec![1, 2], vec![3]]), None).unwrap();
assert_eq!(
params.prompt_logprob_token_ids,
Some(WireNdArray::from_i32(vec![2, 2], vec![1, 2, 3, -1]).unwrap())
);
assert_eq!(
lower(Some(vec![vec![5]]), Some(1)).unwrap().prompt_logprob_start,
Some(1)
);
expect_test::expect![[r#"
[
"prompt_logprob_token_ids must not be empty.",
"prompt_logprob_token_ids must be a non-empty integer array of shape [num_rows, num_ids].",
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
"prompt_logprob_token_ids must not contain duplicates.",
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000",
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000",
"prompt_logprob_token_ids has 1 rows, but the prompt has 2 scored rows (prompt_len - 1 - prompt_logprob_start).",
"prompt_logprob_start requires prompt_logprob_token_ids.",
]
"#]]
.assert_debug_eq(&[
rejects(Some(vec![]), None),
rejects(Some((0..21).collect()), None),
rejects(Some(vec![1000]), None),
rejects(Some(vec![1, 1]), None),
"#]].assert_debug_eq(&[
rejects(Some(vec![vec![]]), None),
rejects(Some(vec![(0..21).collect(), vec![0]]), None),
rejects(Some(vec![vec![1000], vec![0]]), None),
rejects(Some(vec![vec![-2], vec![0]]), None),
rejects(Some(vec![vec![1]]), None),
rejects(None, Some(0)),
]);
}
+55 -12
View File
@@ -3,11 +3,11 @@
//! Python-compatible validation for logprobs sampling params.
//!
//! `-1` is expanded only for bounds checks. The original request values are
//! passed through to engine-core.
//! `-1` counts are expanded only for bounds checks and passed to engine-core
//! unchanged; `prompt_logprob_token_ids` rows are padded into an array.
use itertools::Itertools as _;
use thiserror::Error;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use crate::backend::SamplingLimits;
@@ -36,10 +36,21 @@ pub enum LogprobsError {
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
)]
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
#[error("prompt_logprob_token_ids must not be empty.")]
#[error(
"prompt_logprob_token_ids must be a non-empty integer array of shape \
[num_rows, num_ids]."
)]
EmptyPromptLogprobTokenIds,
#[error("prompt_logprob_token_ids must not contain duplicates.")]
DuplicatePromptLogprobTokenIds,
#[error(
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). \
Vocabulary size: {vocab_size}"
)]
PromptLogprobTokenIdsOutOfVocab { vocab_size: usize },
#[error(
"prompt_logprob_token_ids has {rows} rows, but the prompt has {expected} scored \
rows (prompt_len - 1 - prompt_logprob_start)."
)]
PromptLogprobRowsMismatch { rows: usize, expected: usize },
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
PromptLogprobStartWithoutTokenIds,
}
@@ -49,7 +60,7 @@ pub(super) fn validate_logprobs(
logprobs: Option<i32>,
prompt_logprobs: Option<i32>,
logprob_token_ids: Option<&[u32]>,
prompt_logprob_token_ids: Option<&[u32]>,
prompt_logprob_token_ids: Option<&[Vec<i32>]>,
prompt_logprob_start: Option<u32>,
sampling_limits: SamplingLimits,
) -> Result<(), LogprobsError> {
@@ -60,7 +71,7 @@ pub(super) fn validate_logprobs(
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
validate_logprobs_count(
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
prompt_logprob_token_ids.map(|rows| rows.iter().map(Vec::len).max().unwrap_or(0) as i32),
max_logprobs,
vocab_size,
"prompt_logprob_token_ids",
@@ -70,17 +81,49 @@ pub(super) fn validate_logprobs(
}
fn validate_prompt_logprob_token_ids(
token_ids: Option<&[u32]>,
rows: Option<&[Vec<i32>]>,
start: Option<u32>,
) -> Result<(), LogprobsError> {
match token_ids {
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
match rows {
Some(rows) if rows.iter().all(Vec::is_empty) => {
Err(LogprobsError::EmptyPromptLogprobTokenIds)
}
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
_ => Ok(()),
}
}
/// Check per-row candidates against the vocabulary and the prompt as Python
/// does, and pad them with `-1` into the engine's `[num_rows, num_ids]` array.
pub(super) fn lower_prompt_logprob_token_ids(
rows: Option<Vec<Vec<i32>>>,
start: Option<u32>,
prompt_len: u32,
vocab_size: usize,
) -> Result<Option<WireNdArray>, LogprobsError> {
let Some(rows) = rows else {
return Ok(None);
};
if rows.iter().flatten().any(|&id| id < -1 || i64::from(id) >= vocab_size as i64) {
return Err(LogprobsError::PromptLogprobTokenIdsOutOfVocab { vocab_size });
}
let expected = (prompt_len as usize).saturating_sub(1 + start.unwrap_or(0) as usize);
if rows.len() != expected {
return Err(LogprobsError::PromptLogprobRowsMismatch {
rows: rows.len(),
expected,
});
}
let width = rows.iter().map(Vec::len).max().unwrap_or(0);
let data = rows
.iter()
.flat_map(|row| row.iter().copied().chain(std::iter::repeat_n(-1, width - row.len())))
.collect();
Ok(Some(
WireNdArray::from_i32(vec![rows.len(), width], data).expect("padded rows are rectangular"),
))
}
fn validate_logprobs_count(
requested: Option<i32>,
max_logprobs: usize,
-8
View File
@@ -95,14 +95,6 @@ pub(crate) fn validate_vocab_range(
)?;
}
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
validate_param(
"prompt_logprob_token_ids",
token_ids.iter().copied(),
limits.model_vocab_size,
)?;
}
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
validate_param(
"bad_words",
+3 -3
View File
@@ -97,9 +97,9 @@ pub struct SamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// Candidate token IDs per scored causal prompt row, where row `i` scores
/// its IDs as predictions of prompt token `prompt_logprob_start + i + 1`.
pub prompt_logprob_token_ids: Option<Vec<Vec<i32>>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling. `None` means no
@@ -318,11 +318,11 @@ async def test_generate_logprobs(client, logprobs_value):
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
candidates = [22, 33, 44, 55]
rows = [[33], [44, 100], [55]]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": candidates,
"prompt_logprob_token_ids": rows,
"prompt_logprob_start": 1,
}
payload = {
@@ -335,14 +335,13 @@ async def test_generate_prompt_token_id_logprobs(client):
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (len(token_ids) - 2, len(candidates))
assert scores.shape == (3, 2)
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, candidates.index(target)] == pytest.approx(
expected, abs=1e-3
)
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
assert np.isneginf(scores[[0, 2], 1]).all()
stream_only_ids = {"prompt_logprob_token_ids": candidates}
stream_only_ids = {"prompt_logprob_token_ids": rows}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
@@ -350,15 +349,12 @@ async def test_generate_prompt_token_id_logprobs(client):
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
)
resp.raise_for_status()
empty = np.load(
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
)
assert empty.shape == (0, len(candidates))
assert resp.status_code == 400
assert "scored rows" in resp.text
@pytest.mark.asyncio
+115 -40
View File
@@ -11,6 +11,7 @@ from unittest.mock import patch
import numpy as np
import pytest
import torch
from pydantic import TypeAdapter, ValidationError
from tests.utils import large_gpu_mark
from tests.v1.sample.utils import (
@@ -20,7 +21,7 @@ from tests.v1.sample.utils import (
compute_correct_cumulative_logprob,
get_test_batch,
)
from vllm import SamplingParams
from vllm import SamplingParams, TokensPrompt
from vllm.config.model import LogprobsMode
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
@@ -88,6 +89,7 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
return SimpleNamespace(
max_logprobs=max_logprobs,
max_model_len=16,
logits_processors=None,
is_diffusion=False,
get_vocab_size=lambda: vocab_size,
@@ -443,24 +445,58 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
)
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
"""The candidate count is bounded by max_logprobs."""
def test_prompt_logprob_token_ids_validation():
"""Rows may be ragged (-1 padded); the width is bounded by max_logprobs."""
model_config = _model_config(vocab_size=100, max_logprobs=4)
def verify(**kwargs):
SamplingParams(**kwargs).verify(
params = SamplingParams(**kwargs)
params.verify(
model_config,
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
)
return params.prompt_logprob_token_ids
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
assert verify(prompt_logprob_token_ids=[[], [1], [4, 3, 2, 1]]).tolist() == [
[-1] * 4,
[1, -1, -1, -1],
[4, 3, 2, 1],
]
# Any integer layout is normalized to what the worker uploads:
# C-contiguous native int32.
odd = np.asfortranarray(np.array([[1, 2], [3, -1]], dtype=">i8"))
ids = verify(prompt_logprob_token_ids=odd)
assert ids.dtype == np.int32 and ids.flags.c_contiguous
assert ids.tolist() == [[1, 2], [3, -1]]
# The pydantic path (token-in/token-out API, /docs) takes strict integer lists.
adapter = TypeAdapter(SamplingParams)
for bad in ("[[1.5]]", '[["3"]]', "[[true]]", "[1, 2]"):
with pytest.raises(ValidationError):
adapter.validate_json(f'{{"prompt_logprob_token_ids": {bad}}}')
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
verify(prompt_logprob_token_ids=np.array([1, 2]))
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
verify(prompt_logprob_token_ids=[1, 2])
for bad_ids in ([[None]], [[2**63]], [[[1]]], [[[1], 2]]):
with pytest.raises(VLLMValidationError, match="integer token ids"):
verify(prompt_logprob_token_ids=bad_ids)
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=np.array([[2**64 - 1]], dtype=np.uint64))
# The error names the value to raise max_logprobs to.
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
verify(prompt_logprob_token_ids=[[1], [1, 2, 3, 4, 5]])
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=[[1], [100]])
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=[[1], [-2]])
with pytest.raises(VLLMValidationError, match="non-empty"):
verify(prompt_logprob_token_ids=[[], []])
# Rows are bounded by max_model_len before the table is padded.
with pytest.raises(VLLMValidationError, match="at most 15 scored rows"):
verify(prompt_logprob_token_ids=[[]] * 15 + [[1]])
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
verify(prompt_logprob_start=3)
@@ -483,7 +519,7 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
tokenizer=None,
validate_logits_processors_params=lambda params: None,
)
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
params = SamplingParams(prompt_logprob_token_ids=[[1, 2]])
with patch.object(SamplingParams, "verify"):
with pytest.raises(VLLMValidationError, match="V2 model runner"):
InputProcessor._validate_params(processor, params, ("generate",))
@@ -1369,7 +1405,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
"""Per-row scores stay row-aligned across chunked prefill and preemption."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = [
@@ -1380,14 +1416,9 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
] + [f"Tell me about the number {i}: " for i in range(32)]
start = 2
candidate_ids = [10, 100, 1000, 10000]
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=40,
min_tokens=20,
prompt_logprob_token_ids=candidate_ids,
prompt_logprob_start=start,
)
def candidate_ids(j):
return [10 + j, 100 + j, 1000 + j, 10000 + j]
with VllmRunner(
"Qwen/Qwen3-0.6B",
@@ -1398,15 +1429,38 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
disable_log_stats=False,
gpu_memory_utilization=0.25,
) as vllm_model:
tokenizer = vllm_model.llm.get_tokenizer()
token_prompts = [
TokensPrompt(prompt_token_ids=tokenizer(p).input_ids) for p in prompts
]
def make_params(row_ids):
return [
SamplingParams(
temperature=0.0,
max_tokens=40,
min_tokens=20,
prompt_logprob_token_ids=[
row_ids(j)
for j in range(len(p["prompt_token_ids"]) - 1 - start)
],
prompt_logprob_start=start,
)
for p in token_prompts
]
metrics_before = vllm_model.llm.get_metrics()
outputs = vllm_model.llm.generate(prompts, sampling_params)
full_params = make_params(candidate_ids)
for params in full_params: # the array form, as a top-k tensor would be
params.prompt_logprob_token_ids = np.array(params.prompt_logprob_token_ids)
outputs = vllm_model.llm.generate(token_prompts, full_params)
for i, output in enumerate(outputs):
scores = output.prompt_token_id_logprobs
assert scores is not None, f"Output {i} missing fixed-ID scores"
assert scores is not None, f"Output {i} missing per-row scores"
expected_shape = (
len(output.prompt_token_ids) - 1 - start,
len(candidate_ids),
4,
)
assert scores.shape == expected_shape, (
f"Output {i} scored {scores.shape}, expected {expected_shape}"
@@ -1414,6 +1468,23 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
assert math.isfinite(float(scores.min()))
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
# Ragged rows: each row keeps a prefix of the candidates and must
# reproduce that prefix of the full scores, with -inf padding.
ragged_params = make_params(lambda j: candidate_ids(j)[: 1 + j % 4])
ragged_outputs = vllm_model.llm.generate(token_prompts, ragged_params)
for output, ragged_output in zip(outputs, ragged_outputs):
expected = np.full_like(output.prompt_token_id_logprobs, -np.inf)
for j in range(len(expected)):
n = 1 + j % 4
expected[j, :n] = output.prompt_token_id_logprobs[j, :n]
np.testing.assert_allclose(
ragged_output.prompt_token_id_logprobs, expected, atol=1e-2
)
# The row count is checked against the prompt at admission.
with pytest.raises(VLLMValidationError, match="scored rows"):
vllm_model.llm.generate(token_prompts[0], ragged_params[1])
metrics_after = vllm_model.llm.get_metrics()
preemptions_before = next(
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
@@ -1429,31 +1500,32 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
# unchunked, unpreempted run of the same requests. Batch composition moves
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
max_logprobs=64,
gpu_memory_utilization=0.25,
"Qwen/Qwen3-0.6B", max_model_len=512, gpu_memory_utilization=0.25
) as reference:
reference_outputs = reference.llm.generate(prompts, sampling_params)
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
reference_outputs = reference.llm.generate(token_prompts, full_params)
self_scored = reference.llm.generate(
prompts,
token_prompts,
[
SamplingParams(
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
max_tokens=1,
prompt_logprobs=0,
prompt_logprob_token_ids=[[t] for t in p["prompt_token_ids"][1:]],
)
for ids in own_ids
for p in token_prompts
],
)
for output, ref in zip(outputs, reference_outputs):
np.testing.assert_allclose(
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
output.prompt_token_id_logprobs,
ref.prompt_token_id_logprobs,
rtol=0.1,
atol=0.2,
)
for output, ids in zip(self_scored, own_ids):
scores = output.prompt_token_id_logprobs
for output in self_scored:
scores = output.prompt_token_id_logprobs[:, 0]
for row, target in enumerate(output.prompt_token_ids[1:]):
expected = output.prompt_logprobs[row + 1][target].logprob
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
assert scores[row] == pytest.approx(expected, abs=1e-3)
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
@@ -1465,13 +1537,7 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompt = "The capital of France is Paris. " * 20
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
prompt_logprob_token_ids=[10, 100, 1000],
skip_reading_prefix_cache=False,
)
text = "The capital of France is Paris. " * 20
with VllmRunner(
"Qwen/Qwen3-0.6B",
@@ -1479,6 +1545,15 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
enable_prefix_caching=True,
gpu_memory_utilization=0.25,
) as vllm_model:
token_ids = vllm_model.llm.get_tokenizer()(text).input_ids
prompt = TokensPrompt(prompt_token_ids=token_ids)
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
prompt_logprob_token_ids=np.tile([10, 100, 1000], (len(token_ids) - 1, 1)),
skip_reading_prefix_cache=False,
)
first = vllm_model.llm.generate([prompt], sampling_params)[0]
assert first.prompt_token_id_logprobs is not None
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
+3 -1
View File
@@ -42,7 +42,9 @@ def test_sampling_params_trace_field_rejects_empty_list():
def test_sampling_params_trace_field_rejects_prompt_scoring():
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
params = SamplingParams(
trace_decode_token_ids=[1], prompt_logprob_token_ids=[[2, 3]]
)
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
+3 -3
View File
@@ -127,9 +127,9 @@ class RequestOutput:
decoder input prompt token ids.
prompt_logprobs: The log probabilities to return per prompt token.
prompt_token_id_logprobs: Logprobs of prompt_logprob_token_ids, shaped
[max(prompt_len - 1 - prompt_logprob_start, 0),
len(prompt_logprob_token_ids)]; row i scores them as predictions of
prompt token prompt_logprob_start + i + 1.
[prompt_len - 1 - prompt_logprob_start, num_ids]; row i holds the
logprobs of its own IDs as predictions of prompt token
prompt_logprob_start + i + 1, with -inf in padded columns.
outputs: The output sequences of the request.
finished: Whether the whole request is finished.
metrics: Metrics associated with the request.
+54 -32
View File
@@ -11,7 +11,8 @@ from functools import cached_property
from typing import Annotated, Any
import msgspec
from pydantic import BeforeValidator
import numpy as np
from pydantic import BeforeValidator, GetPydanticSchema, StrictInt
from pydantic.dataclasses import dataclass
import vllm.envs as envs
@@ -20,6 +21,7 @@ from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.utils.mistral import is_mistral_tokenizer
from vllm.utils.torch_utils import make_ndarray_with_pad
from vllm.v1.serial_utils import PydanticMsgspecMixin
logger = init_logger(__name__)
@@ -289,10 +291,14 @@ class SamplingParams(
prompt_logprobs: int | None = None
"""Number of log probabilities to return per prompt token.
When set to -1, return all `vocab_size` log probabilities."""
prompt_logprob_token_ids: list[int] | None = None
"""Token IDs to score at each scored causal prompt row, where row i scores
them as predictions of prompt token i + 1. The last prompt row is excluded,
so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows."""
prompt_logprob_token_ids: (
Annotated[np.ndarray, GetPydanticSchema(lambda _, h: h(list[list[StrictInt]]))]
| None
) = None
"""Token IDs to score per causal prompt row: an integer array of shape
[num_rows, num_ids] or nested lists; -1 entries (padding) score -inf.
Row i scores its IDs as predictions of prompt token
prompt_logprob_start + i + 1, so the last prompt row is excluded."""
prompt_logprob_start: int | None = None
"""First causal prompt row to score; defaults to the first row."""
logprob_token_ids: list[int] | None = None
@@ -404,7 +410,7 @@ class SamplingParams(
min_tokens: int = 0,
logprobs: int | None = None,
prompt_logprobs: int | None = None,
prompt_logprob_token_ids: list[int] | None = None,
prompt_logprob_token_ids: np.ndarray | list[list[int]] | None = None,
prompt_logprob_start: int | None = None,
detokenize: bool = True,
skip_special_tokens: bool = True,
@@ -916,42 +922,58 @@ class SamplingParams(
)
# Validate prompt_logprob_token_ids.
if self.prompt_logprob_token_ids is not None:
n = len(self.prompt_logprob_token_ids)
if n == 0:
ids = self.prompt_logprob_token_ids
if ids is not None:
shape: tuple[int, ...] = ()
if isinstance(ids, list) and all(isinstance(row, list) for row in ids):
shape = (len(ids), max(map(len, ids), default=0))
elif isinstance(ids, np.ndarray) and ids.dtype.kind in "iu":
shape = ids.shape
if len(shape) != 2 or 0 in shape:
raise VLLMValidationError(
"prompt_logprob_token_ids must not be empty.",
"prompt_logprob_token_ids must be a non-empty integer array "
"of shape [num_rows, num_ids].",
parameter="prompt_logprob_token_ids",
value=n,
value=getattr(ids, "shape", type(ids).__name__),
)
if n > max_logprobs:
max_rows = model_config.max_model_len - 1
if shape[0] > max_rows:
raise VLLMValidationError(
f"Requested prompt_logprob_token_ids of length {n}, "
f"which is greater than max allowed: {max_logprobs}. "
f"Set max_logprobs (--max-logprobs) to at least {n}.",
f"prompt_logprob_token_ids has {shape[0]} rows, but a prompt "
f"has at most {max_rows} scored rows (max_model_len - 1).",
parameter="prompt_logprob_token_ids",
value=n,
value=shape[0],
)
num_ids = shape[1]
if num_ids > max_logprobs:
raise VLLMValidationError(
f"Requested {num_ids} token ids per row in "
f"prompt_logprob_token_ids, which is greater than max allowed: "
f"{max_logprobs}. "
f"Set max_logprobs (--max-logprobs) to at least {num_ids}.",
parameter="prompt_logprob_token_ids",
value=num_ids,
)
if isinstance(ids, list):
# Pad only after the checks above bound the allocation.
try:
ids = make_ndarray_with_pad(ids, -1, np.int64, max_len=num_ids)
except (TypeError, ValueError, OverflowError) as e:
raise VLLMValidationError(
"prompt_logprob_token_ids must contain integer token ids.",
parameter="prompt_logprob_token_ids",
) from e
vocab_size = model_config.get_vocab_size()
invalid_token_ids = [
token_id
for token_id in self.prompt_logprob_token_ids
if token_id < 0 or token_id >= vocab_size
]
if invalid_token_ids:
lo, hi = int(ids.min()), int(ids.max())
if lo < -1 or hi >= vocab_size:
raise VLLMValidationError(
f"token_id(s) {invalid_token_ids} in "
f"prompt_logprob_token_ids contain out-of-vocab token ids. "
f"Vocabulary size: {vocab_size}",
"prompt_logprob_token_ids contain out-of-vocab token ids "
f"(-1 pads a row). Vocabulary size: {vocab_size}",
parameter="prompt_logprob_token_ids",
value=invalid_token_ids,
)
if len(set(self.prompt_logprob_token_ids)) != n:
raise VLLMValidationError(
"prompt_logprob_token_ids must not contain duplicates.",
parameter="prompt_logprob_token_ids",
value=self.prompt_logprob_token_ids,
value=[lo, hi],
)
# In-vocab IDs fit int32, like the runner's other token ID buffers.
self.prompt_logprob_token_ids = np.ascontiguousarray(ids, dtype=np.int32)
if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0:
raise VLLMValidationError(
"prompt_logprob_start must be non-negative.",
+12
View File
@@ -423,6 +423,18 @@ class InputProcessor:
parameter="routed_experts_prompt_start",
value=sampling_params.routed_experts_prompt_start,
)
rows = sampling_params.prompt_logprob_token_ids
if rows is not None:
start = sampling_params.prompt_logprob_start or 0
num_rows = max(prompt_len - 1 - start, 0)
if len(rows) != num_rows:
raise VLLMValidationError(
f"prompt_logprob_token_ids has {len(rows)} rows, but the "
f"prompt has {num_rows} scored rows "
f"(prompt_len - 1 - prompt_logprob_start).",
parameter="prompt_logprob_token_ids",
value=len(rows),
)
# If unset max tokens, then generate up to the max_model_len.
if sampling_params.max_tokens is None:
sampling_params.max_tokens = (
+1 -1
View File
@@ -532,7 +532,7 @@ class PydanticMsgspecMixin:
called on every validation.
"""
msgspec_fields = {f.name: f for f in msgspec.structs.fields(source_type)}
type_hints = get_type_hints(source_type)
type_hints = get_type_hints(source_type, include_extras=True)
# Build the Pydantic typed_dict_field for each msgspec field
fields = {}
+13 -11
View File
@@ -22,7 +22,8 @@ CHUNK_SIZE = 1024
@dataclass
class _TokenIdScores:
token_ids: torch.Tensor # [num_ids], on device
token_ids: torch.Tensor # [num_rows, num_ids], on device
pad: torch.Tensor # [num_rows, num_ids] bool, True at padded columns
start: int # first prompt row to score
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
@@ -52,11 +53,13 @@ class PromptLogprobsWorker:
if uses_prompt_logprobs:
self.in_progress_prompt_logprobs[req_id] = []
if sampling_params.prompt_logprob_token_ids is not None:
ids = sampling_params.prompt_logprob_token_ids
# Decoded arrays may be read-only; torch.from_numpy warns on those.
if not ids.flags.writeable:
ids = ids.copy()
ids = async_tensor_h2d(ids, self.device, torch.int64)
self.token_id_scores[req_id] = _TokenIdScores(
async_tensor_h2d(
sampling_params.prompt_logprob_token_ids, self.device, torch.int64
),
sampling_params.prompt_logprob_start or 0,
ids.clamp_min(0), ids < 0, sampling_params.prompt_logprob_start or 0
)
def remove_request(self, req_id: str) -> None:
@@ -85,6 +88,7 @@ class PromptLogprobsWorker:
# Skip decode steps and, as compute_prompt_logprobs does, requests
# resumed after preemption: their scores were already emitted.
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
self.token_id_scores.pop(req_id, None)
continue
if req.scores is None:
if chunk_start > req.start:
@@ -93,10 +97,7 @@ class PromptLogprobsWorker:
# emitting a buffer with unwritten rows.
self.token_id_scores.pop(req_id, None)
continue
req.scores = hidden_states.new_empty(
(max(prompt_len - 1 - req.start, 0), len(req.token_ids)),
dtype=torch.float32,
)
req.scores = torch.empty_like(req.token_ids, dtype=torch.float32)
# The last prompt row predicts the first decode token; skip it.
lo = max(req.start, chunk_start)
hi = min(chunk_end, prompt_len - 1)
@@ -104,15 +105,16 @@ class PromptLogprobsWorker:
for a in range(lo, hi, CHUNK_SIZE):
b = min(a + CHUNK_SIZE, hi)
logits = logits_fn(hidden_states[base + a : base + b])
ids = req.token_ids.expand(b - a, -1)
ids = req.token_ids[a - req.start : b - req.start]
req.scores[a - req.start : b - req.start] = (
logits.gather(-1, ids)
if logits_mode
else compute_token_logprobs(logits, ids)
)
if chunk_end >= prompt_len:
req.scores.masked_fill_(req.pad, float("-inf"))
out[req_id] = req.scores
req.scores = None
self.token_id_scores.pop(req_id, None)
return out
def compute_prompt_logprobs(