[Feature] Per-row candidate IDs for prefill token scoring (M2 of #56860) (#56984)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-03 14:17:09 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent bc21cba967
commit e319f86f15
17 changed files with 366 additions and 162 deletions
@@ -152,6 +152,11 @@ impl EngineCoreRequest {
feature.extract_aux_frames(&mut aux_frames, threshold);
}
}
if let Some(params) = &mut self.sampling_params
&& let Some(ids) = &mut params.prompt_logprob_token_ids
{
ids.extract_aux_frame(&mut aux_frames, threshold);
}
aux_frames
}
}
@@ -168,7 +173,7 @@ mod tests {
PlaceholderRange,
};
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::tensor::{WireArrayData, WireTensor};
use crate::protocol::tensor::{WireArrayData, WireNdArray, WireTensor};
use crate::protocol::{decode_value, encode_msgpack};
const AUX_FRAME_THRESHOLD: usize = 256;
@@ -294,4 +299,27 @@ mod tests {
);
assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty());
}
#[test]
fn engine_core_request_extracts_large_prompt_logprob_token_ids() {
let ids = WireNdArray::from_i32(vec![AUX_FRAME_THRESHOLD, 1], vec![5; AUX_FRAME_THRESHOLD])
.unwrap();
let mut request = EngineCoreRequest {
sampling_params: Some(EngineCoreSamplingParams {
prompt_logprob_token_ids: Some(ids),
..EngineCoreSamplingParams::for_test()
}),
..EngineCoreRequest::default()
};
let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD);
assert_eq!(aux_frames.len(), 1);
assert_eq!(aux_frames[0].len(), AUX_FRAME_THRESHOLD * 4);
let params = request.sampling_params.unwrap();
assert_eq!(
params.prompt_logprob_token_ids.unwrap().data,
WireArrayData::AuxIndex(1)
);
}
}
@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use crate::protocol::structured_outputs::StructuredOutputsParams;
use crate::protocol::tensor::WireNdArray;
fn default_top_p() -> f32 {
1.0
@@ -101,9 +102,9 @@ pub struct EngineCoreSamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// `[num_rows, num_ids]` candidate token IDs per scored causal prompt row,
/// `-1` padding shorter rows.
pub prompt_logprob_token_ids: Option<WireNdArray>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling.
@@ -121,6 +121,18 @@ impl<Dtype: From<TensorDtype> + Serialize + DeserializeOwned> WireArray<Dtype> {
))
}
/// Build an int32 tensor/ndarray backed by native-endian raw-view bytes.
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_i32(shape: Vec<usize>, data: Vec<i32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self::from_raw_bytes(
TensorDtype::I32,
shape,
bytes_from_pod_vec(data),
))
}
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
///
/// Takes ownership of the backing buffer without copying its data.
@@ -181,7 +181,7 @@ mod tests {
let request: GenerateRequest = serde_json::from_value(json!({
"token_ids": [11, 22],
"stream": true,
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
"sampling_params": {"prompt_logprob_token_ids": [[1, 2]]}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
+33 -16
View File
@@ -7,7 +7,7 @@ pub(crate) mod logprobs;
pub(crate) mod sampling;
pub(crate) mod token_ids;
use logprobs::validate_logprobs;
use logprobs::{lower_prompt_logprob_token_ids, validate_logprobs};
use sampling::validate_resolved_sampling_params;
use token_ids::{validate_prompt_token_ids, validate_vocab_range};
use vllm_engine_core_client::protocol::sampling::{
@@ -50,7 +50,11 @@ pub fn lower_text_request(
// multimodal tensor payloads.
mm_features: request.mm_features.take(),
sampling_params: lower_sampling_params(
request.sampling_params.clone(),
SamplingParams {
// Move the candidate table rather than cloning it; only lowering reads it.
prompt_logprob_token_ids: request.sampling_params.prompt_logprob_token_ids.take(),
..request.sampling_params.clone()
},
sampling_hints,
sampling_limits,
prompt_len,
@@ -180,7 +184,12 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_token_ids: lower_prompt_logprob_token_ids(
prompt_logprob_token_ids,
prompt_logprob_start,
prompt_len,
sampling_limits.model_vocab_size,
)?,
prompt_logprob_start,
min_p,
frequency_penalty,
@@ -332,6 +341,7 @@ mod tests {
use vllm_engine_core_client::protocol::multimodal::{
MmFeatureSpec, MmModality, PlaceholderRange,
};
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
@@ -1140,7 +1150,7 @@ mod tests {
#[test]
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
let lower = |ids: Option<Vec<Vec<i32>>>, start: Option<u32>| {
lower_sampling_params_with_limits(
SamplingParams {
prompt_logprob_token_ids: ids,
@@ -1155,23 +1165,30 @@ mod tests {
Ok(_) => panic!("expected rejection"),
};
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
assert_eq!(params.prompt_logprob_start, Some(1));
let params = lower(Some(vec![vec![1, 2], vec![3]]), None).unwrap();
assert_eq!(
params.prompt_logprob_token_ids,
Some(WireNdArray::from_i32(vec![2, 2], vec![1, 2, 3, -1]).unwrap())
);
assert_eq!(
lower(Some(vec![vec![5]]), Some(1)).unwrap().prompt_logprob_start,
Some(1)
);
expect_test::expect![[r#"
[
"prompt_logprob_token_ids must not be empty.",
"prompt_logprob_token_ids must be a non-empty integer array of shape [num_rows, num_ids].",
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
"prompt_logprob_token_ids must not contain duplicates.",
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000",
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000",
"prompt_logprob_token_ids has 1 rows, but the prompt has 2 scored rows (prompt_len - 1 - prompt_logprob_start).",
"prompt_logprob_start requires prompt_logprob_token_ids.",
]
"#]]
.assert_debug_eq(&[
rejects(Some(vec![]), None),
rejects(Some((0..21).collect()), None),
rejects(Some(vec![1000]), None),
rejects(Some(vec![1, 1]), None),
"#]].assert_debug_eq(&[
rejects(Some(vec![vec![]]), None),
rejects(Some(vec![(0..21).collect(), vec![0]]), None),
rejects(Some(vec![vec![1000], vec![0]]), None),
rejects(Some(vec![vec![-2], vec![0]]), None),
rejects(Some(vec![vec![1]]), None),
rejects(None, Some(0)),
]);
}
+55 -12
View File
@@ -3,11 +3,11 @@
//! Python-compatible validation for logprobs sampling params.
//!
//! `-1` is expanded only for bounds checks. The original request values are
//! passed through to engine-core.
//! `-1` counts are expanded only for bounds checks and passed to engine-core
//! unchanged; `prompt_logprob_token_ids` rows are padded into an array.
use itertools::Itertools as _;
use thiserror::Error;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use crate::backend::SamplingLimits;
@@ -36,10 +36,21 @@ pub enum LogprobsError {
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
)]
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
#[error("prompt_logprob_token_ids must not be empty.")]
#[error(
"prompt_logprob_token_ids must be a non-empty integer array of shape \
[num_rows, num_ids]."
)]
EmptyPromptLogprobTokenIds,
#[error("prompt_logprob_token_ids must not contain duplicates.")]
DuplicatePromptLogprobTokenIds,
#[error(
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). \
Vocabulary size: {vocab_size}"
)]
PromptLogprobTokenIdsOutOfVocab { vocab_size: usize },
#[error(
"prompt_logprob_token_ids has {rows} rows, but the prompt has {expected} scored \
rows (prompt_len - 1 - prompt_logprob_start)."
)]
PromptLogprobRowsMismatch { rows: usize, expected: usize },
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
PromptLogprobStartWithoutTokenIds,
}
@@ -49,7 +60,7 @@ pub(super) fn validate_logprobs(
logprobs: Option<i32>,
prompt_logprobs: Option<i32>,
logprob_token_ids: Option<&[u32]>,
prompt_logprob_token_ids: Option<&[u32]>,
prompt_logprob_token_ids: Option<&[Vec<i32>]>,
prompt_logprob_start: Option<u32>,
sampling_limits: SamplingLimits,
) -> Result<(), LogprobsError> {
@@ -60,7 +71,7 @@ pub(super) fn validate_logprobs(
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
validate_logprobs_count(
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
prompt_logprob_token_ids.map(|rows| rows.iter().map(Vec::len).max().unwrap_or(0) as i32),
max_logprobs,
vocab_size,
"prompt_logprob_token_ids",
@@ -70,17 +81,49 @@ pub(super) fn validate_logprobs(
}
fn validate_prompt_logprob_token_ids(
token_ids: Option<&[u32]>,
rows: Option<&[Vec<i32>]>,
start: Option<u32>,
) -> Result<(), LogprobsError> {
match token_ids {
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
match rows {
Some(rows) if rows.iter().all(Vec::is_empty) => {
Err(LogprobsError::EmptyPromptLogprobTokenIds)
}
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
_ => Ok(()),
}
}
/// Check per-row candidates against the vocabulary and the prompt as Python
/// does, and pad them with `-1` into the engine's `[num_rows, num_ids]` array.
pub(super) fn lower_prompt_logprob_token_ids(
rows: Option<Vec<Vec<i32>>>,
start: Option<u32>,
prompt_len: u32,
vocab_size: usize,
) -> Result<Option<WireNdArray>, LogprobsError> {
let Some(rows) = rows else {
return Ok(None);
};
if rows.iter().flatten().any(|&id| id < -1 || i64::from(id) >= vocab_size as i64) {
return Err(LogprobsError::PromptLogprobTokenIdsOutOfVocab { vocab_size });
}
let expected = (prompt_len as usize).saturating_sub(1 + start.unwrap_or(0) as usize);
if rows.len() != expected {
return Err(LogprobsError::PromptLogprobRowsMismatch {
rows: rows.len(),
expected,
});
}
let width = rows.iter().map(Vec::len).max().unwrap_or(0);
let data = rows
.iter()
.flat_map(|row| row.iter().copied().chain(std::iter::repeat_n(-1, width - row.len())))
.collect();
Ok(Some(
WireNdArray::from_i32(vec![rows.len(), width], data).expect("padded rows are rectangular"),
))
}
fn validate_logprobs_count(
requested: Option<i32>,
max_logprobs: usize,
-8
View File
@@ -95,14 +95,6 @@ pub(crate) fn validate_vocab_range(
)?;
}
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
validate_param(
"prompt_logprob_token_ids",
token_ids.iter().copied(),
limits.model_vocab_size,
)?;
}
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
validate_param(
"bad_words",
+3 -3
View File
@@ -97,9 +97,9 @@ pub struct SamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// Candidate token IDs per scored causal prompt row, where row `i` scores
/// its IDs as predictions of prompt token `prompt_logprob_start + i + 1`.
pub prompt_logprob_token_ids: Option<Vec<Vec<i32>>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling. `None` means no