[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
+56
View File
@@ -103,6 +103,8 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_start,
min_p,
frequency_penalty,
presence_penalty,
@@ -123,6 +125,8 @@ pub fn lower_sampling_params(
logprobs,
prompt_logprobs,
logprob_token_ids.as_deref(),
prompt_logprob_token_ids.as_deref(),
prompt_logprob_start,
sampling_limits,
)?;
validate_repetition_detection(repetition_detection.as_ref())?;
@@ -176,6 +180,8 @@ pub fn lower_sampling_params(
thinking_token_budget,
logprobs,
prompt_logprobs,
prompt_logprob_token_ids,
prompt_logprob_start,
min_p,
frequency_penalty,
presence_penalty,
@@ -646,6 +652,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -701,6 +709,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -870,6 +880,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -935,6 +947,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -1008,6 +1022,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -1122,6 +1138,44 @@ mod tests {
));
}
#[test]
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
lower_sampling_params_with_limits(
SamplingParams {
prompt_logprob_token_ids: ids,
prompt_logprob_start: start,
..Default::default()
},
sample_sampling_limits(),
)
};
let rejects = |ids, start| match lower(ids, start) {
Err(error) => error.to_string(),
Ok(_) => panic!("expected rejection"),
};
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
assert_eq!(params.prompt_logprob_start, Some(1));
expect_test::expect![[r#"
[
"prompt_logprob_token_ids must not be empty.",
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
"prompt_logprob_token_ids must not contain duplicates.",
"prompt_logprob_start requires prompt_logprob_token_ids.",
]
"#]]
.assert_debug_eq(&[
rejects(Some(vec![]), None),
rejects(Some((0..21).collect()), None),
rejects(Some(vec![1000]), None),
rejects(Some(vec![1, 1]), None),
rejects(None, Some(0)),
]);
}
#[test]
fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() {
let error = lower_sampling_params_with_limits(
@@ -1259,6 +1313,8 @@ mod tests {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.1,
frequency_penalty: 0.0,
presence_penalty: 0.0,
+28
View File
@@ -6,6 +6,7 @@
//! `-1` is expanded only for bounds checks. The original request values are
//! passed through to engine-core.
use itertools::Itertools as _;
use thiserror::Error;
use crate::backend::SamplingLimits;
@@ -35,6 +36,12 @@ pub enum LogprobsError {
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
)]
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
#[error("prompt_logprob_token_ids must not be empty.")]
EmptyPromptLogprobTokenIds,
#[error("prompt_logprob_token_ids must not contain duplicates.")]
DuplicatePromptLogprobTokenIds,
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
PromptLogprobStartWithoutTokenIds,
}
/// Validate logprobs count sampling parameters.
@@ -42,6 +49,8 @@ pub(super) fn validate_logprobs(
logprobs: Option<i32>,
prompt_logprobs: Option<i32>,
logprob_token_ids: Option<&[u32]>,
prompt_logprob_token_ids: Option<&[u32]>,
prompt_logprob_start: Option<u32>,
sampling_limits: SamplingLimits,
) -> Result<(), LogprobsError> {
let vocab_size = sampling_limits.model_vocab_size;
@@ -50,9 +59,28 @@ pub(super) fn validate_logprobs(
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
validate_logprobs_count(
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
max_logprobs,
vocab_size,
"prompt_logprob_token_ids",
)?;
validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?;
validate_logprob_token_ids(logprobs, logprob_token_ids)
}
fn validate_prompt_logprob_token_ids(
token_ids: Option<&[u32]>,
start: Option<u32>,
) -> Result<(), LogprobsError> {
match token_ids {
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
_ => Ok(()),
}
}
fn validate_logprobs_count(
requested: Option<i32>,
max_logprobs: usize,
+8
View File
@@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range(
)?;
}
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
validate_param(
"prompt_logprob_token_ids",
token_ids.iter().copied(),
limits.model_vocab_size,
)?;
}
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
validate_param(
"bad_words",
+7
View File
@@ -97,6 +97,11 @@ pub struct SamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling. `None` means no
/// explicit user override.
pub min_p: Option<f32>,
@@ -156,6 +161,8 @@ impl Default for SamplingParams {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: None,
frequency_penalty: None,
presence_penalty: None,