[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
@@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
use crate::protocol::tensor::WireNdArray;
use crate::protocol::{OpaqueValue, decode_msgpack};
/// The stop reason associated with a finished output.
@@ -133,6 +134,10 @@ pub struct EngineCoreOutput {
/// output when `--per-request-spec-decode-metrics` is enabled.
#[serde(default)]
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
/// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on
/// the first output of a request that asked for them.
#[serde(default)]
pub prompt_token_id_logprobs: Option<WireNdArray>,
}
/// Raw per-sequence speculative-decoding accumulator.
@@ -180,6 +185,11 @@ impl EngineCoreOutput {
self.new_sampling_mask = (self.new_sampling_mask.take())
.map(|value| value.resolve(frames, "new_sampling_mask"))
.transpose()?;
if let Some(value) = self.prompt_token_id_logprobs.as_mut() {
value
.resolve_aux_frame(frames)
.map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?;
}
Ok(())
}
}
@@ -531,6 +541,7 @@ mod tests {
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
prompt_token_id_logprobs: None,
},
],
scheduler_stats: None,
@@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling.
pub min_p: f32,
/// Frequency penalty applied by the sampler.
@@ -171,6 +176,8 @@ impl EngineCoreSamplingParams {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
prompt_token_id_logprobs: None,
},
],
scheduler_stats: None,
@@ -116,6 +116,7 @@ class EngineCoreOutput(
mm_cache_miss_hashes: list[str] | None = None
new_sampling_mask: object | None = None
spec_decode_metrics: object | None = None
prompt_token_id_logprobs: object | None = None
class ExtendedEngineCoreOutput(EngineCoreOutput):
+16
View File
@@ -15,6 +15,7 @@ use vllm_engine_core_client::protocol::output::{
EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason,
};
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
use crate::error::Result;
@@ -38,6 +39,7 @@ pub struct CollectedGenerateOutput {
pub request_id: String,
pub prompt_token_ids: Vec<u32>,
pub prompt_logprobs: Option<Logprobs>,
pub prompt_token_id_logprobs: Option<WireNdArray>,
pub token_ids: Vec<u32>,
pub logprobs: Option<Logprobs>,
pub finish_reason: FinishReason,
@@ -62,6 +64,8 @@ pub struct GeneratePromptInfo {
/// Prompt logprobs returned by engine-core for scored prompt positions,
/// when requested.
pub prompt_logprobs: Option<Logprobs>,
/// Log probabilities of `prompt_logprob_token_ids`, when requested.
pub prompt_token_id_logprobs: Option<WireNdArray>,
}
/// The reason a request finished.
@@ -206,6 +210,7 @@ impl GenerateOutput {
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: ids,
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids,
logprobs: None,
@@ -247,6 +252,7 @@ impl GenerateOutputStream {
pending_prompt_info: Some(GeneratePromptInfo {
prompt_token_ids,
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
raw_stream,
request_metrics,
@@ -282,6 +288,11 @@ impl Stream for GenerateOutputStream {
info.prompt_logprobs =
raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap());
}
if let Some(info) = &mut self.pending_prompt_info
&& info.prompt_token_id_logprobs.is_none()
{
info.prompt_token_id_logprobs = raw.prompt_token_id_logprobs;
}
let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap());
let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap());
@@ -361,6 +372,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
pin_mut!(stream);
let mut prompt_token_ids = None;
let mut prompt_logprobs = None;
let mut prompt_token_id_logprobs = None;
let mut cached_token_count = 0;
let mut collected: Option<CollectedGenerateOutput> = None;
@@ -374,6 +386,9 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
if prompt_logprobs.is_none() {
prompt_logprobs = info.prompt_logprobs;
}
if prompt_token_id_logprobs.is_none() {
prompt_token_id_logprobs = info.prompt_token_id_logprobs;
}
}
if let Some(existing) = collected.as_mut() {
@@ -393,6 +408,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
request_id: output.request_id,
prompt_token_ids: prompt_token_ids.take().unwrap_or_default(),
prompt_logprobs: prompt_logprobs.take(),
prompt_token_id_logprobs: prompt_token_id_logprobs.take(),
token_ids: output.token_ids,
logprobs: output.logprobs,
finish_reason: FinishReason::Error,
+2
View File
@@ -277,6 +277,7 @@ async fn generate_streams_outputs() {
Some(GeneratePromptInfo {
prompt_token_ids: vec![11, 22].into(),
prompt_logprobs: Some(prompt_logprobs()),
prompt_token_id_logprobs: None,
})
);
assert_eq!(first.token_ids, vec![1, 2]);
@@ -405,6 +406,7 @@ async fn collect_output_rejects_partial_sampling_mask() {
prompt_info: Some(GeneratePromptInfo {
prompt_token_ids: Arc::from([11_u32, 22]),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids: vec![33, 44],
logprobs: None,
+1
View File
@@ -9,6 +9,7 @@ anyhow.workspace = true
asynk-strim-attr.workspace = true
auto_enums.workspace = true
axum.workspace = true
base64.workspace = true
educe.workspace = true
futures.workspace = true
http-body.workspace = true
@@ -16,12 +16,15 @@ use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::Event;
use axum::response::{IntoResponse, Response};
use base64::Engine as _;
use futures::{Stream, StreamExt as _, pin_mut};
use thiserror_ext::AsReport as _;
use tracing::{error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::dtype::TensorDtype;
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
use vllm_engine_core_client::protocol::tensor::WireNdArray;
use vllm_llm::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
};
@@ -330,6 +333,11 @@ fn collect_generate(
token_ids: collected.token_ids,
}],
prompt_logprobs,
prompt_token_id_logprobs: collected
.prompt_token_id_logprobs
.as_ref()
.map(npy_base64)
.transpose()?,
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
kv_transfer_params: collected.kv_transfer_params,
@@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, Generate
.collect()
}
/// Encode a float32 matrix as Python `numpy2base64` does.
fn npy_base64(array: &WireNdArray) -> Result<String, ApiError> {
let (TensorDtype::F32, Some(data), [rows, cols]) = (
array.dtype.scalar,
array.data.as_raw_view(),
array.shape.as_slice(),
) else {
bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix");
};
let mut header =
format!("{{'descr': '<f4', 'fortran_order': False, 'shape': ({rows}, {cols}), }}");
header.push_str(&" ".repeat(21 - rows.to_string().len()));
header.push_str(&" ".repeat(64 - (11 + header.len()) % 64));
header.push('\n');
let mut npy = Vec::with_capacity(10 + header.len() + data.len());
npy.extend_from_slice(b"\x93NUMPY\x01\x00");
npy.extend_from_slice(&(header.len() as u16).to_le_bytes());
npy.extend_from_slice(header.as_bytes());
npy.extend_from_slice(data);
Ok(base64::engine::general_purpose::STANDARD.encode(npy))
}
fn format_token_id(token_id: u32) -> String {
format!("token_id:{token_id}")
}
@@ -509,6 +540,7 @@ mod tests {
prompt_info: Some(GeneratePromptInfo {
prompt_token_ids: Arc::from([11_u32, 22_u32]),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids: vec![33],
logprobs: None,
@@ -592,6 +624,7 @@ mod tests {
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: Arc::from(ids),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
}),
token_ids,
logprobs: None,
@@ -747,6 +780,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -781,6 +815,17 @@ mod tests {
assert!(response.mm_placeholders.is_none());
}
#[test]
fn prompt_token_id_logprobs_match_numpy2base64() {
let scores =
WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap();
// numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32))
assert_eq!(
npy_base64(&scores).unwrap(),
"k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA="
);
}
#[test]
fn collect_generate_returns_spec_decode_metrics() {
let metrics = RequestSpecDecodeMetrics {
@@ -792,6 +837,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-metrics".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -879,6 +925,7 @@ mod tests {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -914,6 +961,7 @@ mod tests {
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![3],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
@@ -94,6 +94,8 @@ pub(super) struct GenerateResponse {
pub request_id: String,
pub choices: Vec<GenerateResponseChoice>,
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
/// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it.
pub prompt_token_id_logprobs: Option<String>,
pub prompt_token_ids: Option<Vec<u32>>,
pub mm_placeholders: Option<MultiModalPlaceholders>,
pub kv_transfer_params: Option<Value>,
@@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat(
}
}
if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() {
bail_invalid_request!(
param = "sampling_params",
"`prompt_logprob_token_ids` are not available when `stream=true`."
);
}
Ok(())
}
@@ -169,6 +176,17 @@ mod tests {
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
}
#[test]
fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() {
let request: GenerateRequest = serde_json::from_value(json!({
"token_ids": [11, 22],
"stream": true,
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
}))
.expect("parse request");
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
}
#[test]
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
let request: GenerateRequest = serde_json::from_value(json!({
@@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request(
thinking_token_budget: request.thinking_token_budget,
logprobs: request.logprobs.then_some(top_logprobs),
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
@@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request(
thinking_token_budget: request.thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: request.min_p,
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
+56
View File
@@ -103,6 +103,8 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_start,
min_p,
frequency_penalty,
presence_penalty,
@@ -123,6 +125,8 @@ pub fn lower_sampling_params(
logprobs,
prompt_logprobs,
logprob_token_ids.as_deref(),
prompt_logprob_token_ids.as_deref(),
prompt_logprob_start,
sampling_limits,
)?;
validate_repetition_detection(repetition_detection.as_ref())?;
@@ -176,6 +180,8 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_start,
min_p,
frequency_penalty,
presence_penalty,
@@ -646,6 +652,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -701,6 +709,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -870,6 +880,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -935,6 +947,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -1008,6 +1022,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -1122,6 +1138,44 @@ mod tests {
));
}
#[test]
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
lower_sampling_params_with_limits(
SamplingParams {
prompt_logprob_token_ids: ids,
prompt_logprob_start: start,
..Default::default()
},
sample_sampling_limits(),
)
};
let rejects = |ids, start| match lower(ids, start) {
Err(error) => error.to_string(),
Ok(_) => panic!("expected rejection"),
};
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
assert_eq!(params.prompt_logprob_start, Some(1));
expect_test::expect![[r#"
[
"prompt_logprob_token_ids must not be empty.",
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
"prompt_logprob_token_ids must not contain duplicates.",
"prompt_logprob_start requires prompt_logprob_token_ids.",
]
"#]]
.assert_debug_eq(&[
rejects(Some(vec![]), None),
rejects(Some((0..21).collect()), None),
rejects(Some(vec![1000]), None),
rejects(Some(vec![1, 1]), None),
rejects(None, Some(0)),
]);
}
#[test]
fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() {
let error = lower_sampling_params_with_limits(
@@ -1259,6 +1313,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
+28
View File
@@ -6,6 +6,7 @@
//! `-1` is expanded only for bounds checks. The original request values are
//! passed through to engine-core.
use itertools::Itertools as _;
use thiserror::Error;
use crate::backend::SamplingLimits;
@@ -35,6 +36,12 @@ pub enum LogprobsError {
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
)]
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
#[error("prompt_logprob_token_ids must not be empty.")]
EmptyPromptLogprobTokenIds,
#[error("prompt_logprob_token_ids must not contain duplicates.")]
DuplicatePromptLogprobTokenIds,
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
PromptLogprobStartWithoutTokenIds,
}
/// Validate logprobs count sampling parameters.
@@ -42,6 +49,8 @@ pub(super) fn validate_logprobs(
logprobs: Option<i32>,
prompt_logprobs: Option<i32>,
logprob_token_ids: Option<&[u32]>,
prompt_logprob_token_ids: Option<&[u32]>,
prompt_logprob_start: Option<u32>,
sampling_limits: SamplingLimits,
) -> Result<(), LogprobsError> {
let vocab_size = sampling_limits.model_vocab_size;
@@ -50,9 +59,28 @@ pub(super) fn validate_logprobs(
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
validate_logprobs_count(
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
max_logprobs,
vocab_size,
"prompt_logprob_token_ids",
)?;
validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?;
validate_logprob_token_ids(logprobs, logprob_token_ids)
}
fn validate_prompt_logprob_token_ids(
token_ids: Option<&[u32]>,
start: Option<u32>,
) -> Result<(), LogprobsError> {
match token_ids {
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
_ => Ok(()),
}
}
fn validate_logprobs_count(
requested: Option<i32>,
max_logprobs: usize,
+8
View File
@@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range(
)?;
}
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
validate_param(
"prompt_logprob_token_ids",
token_ids.iter().copied(),
limits.model_vocab_size,
)?;
}
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
validate_param(
"bad_words",
+7
View File
@@ -97,6 +97,11 @@ pub struct SamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling. `None` means no
/// explicit user override.
pub min_p: Option<f32>,
@@ -156,6 +161,8 @@ impl Default for SamplingParams {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: None,
frequency_penalty: None,
presence_penalty: None,