[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -103,6 +103,8 @@ pub fn lower_sampling_params(
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_start,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
@@ -123,6 +125,8 @@ pub fn lower_sampling_params(
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
logprob_token_ids.as_deref(),
|
||||
prompt_logprob_token_ids.as_deref(),
|
||||
prompt_logprob_start,
|
||||
sampling_limits,
|
||||
)?;
|
||||
validate_repetition_detection(repetition_detection.as_ref())?;
|
||||
@@ -176,6 +180,8 @@ pub fn lower_sampling_params(
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_start,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
@@ -646,6 +652,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -701,6 +709,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -870,6 +880,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -935,6 +947,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -1008,6 +1022,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -1122,6 +1138,44 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
|
||||
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
|
||||
lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
prompt_logprob_token_ids: ids,
|
||||
prompt_logprob_start: start,
|
||||
..Default::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
};
|
||||
let rejects = |ids, start| match lower(ids, start) {
|
||||
Err(error) => error.to_string(),
|
||||
Ok(_) => panic!("expected rejection"),
|
||||
};
|
||||
|
||||
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
|
||||
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
|
||||
assert_eq!(params.prompt_logprob_start, Some(1));
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
|
||||
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
"prompt_logprob_start requires prompt_logprob_token_ids.",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&[
|
||||
rejects(Some(vec![]), None),
|
||||
rejects(Some((0..21).collect()), None),
|
||||
rejects(Some(vec![1000]), None),
|
||||
rejects(Some(vec![1, 1]), None),
|
||||
rejects(None, Some(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
@@ -1259,6 +1313,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! `-1` is expanded only for bounds checks. The original request values are
|
||||
//! passed through to engine-core.
|
||||
|
||||
use itertools::Itertools as _;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::backend::SamplingLimits;
|
||||
@@ -35,6 +36,12 @@ pub enum LogprobsError {
|
||||
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
|
||||
)]
|
||||
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
|
||||
#[error("prompt_logprob_token_ids must not be empty.")]
|
||||
EmptyPromptLogprobTokenIds,
|
||||
#[error("prompt_logprob_token_ids must not contain duplicates.")]
|
||||
DuplicatePromptLogprobTokenIds,
|
||||
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
|
||||
PromptLogprobStartWithoutTokenIds,
|
||||
}
|
||||
|
||||
/// Validate logprobs count sampling parameters.
|
||||
@@ -42,6 +49,8 @@ pub(super) fn validate_logprobs(
|
||||
logprobs: Option<i32>,
|
||||
prompt_logprobs: Option<i32>,
|
||||
logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_start: Option<u32>,
|
||||
sampling_limits: SamplingLimits,
|
||||
) -> Result<(), LogprobsError> {
|
||||
let vocab_size = sampling_limits.model_vocab_size;
|
||||
@@ -50,9 +59,28 @@ pub(super) fn validate_logprobs(
|
||||
|
||||
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
|
||||
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
|
||||
validate_logprobs_count(
|
||||
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
|
||||
max_logprobs,
|
||||
vocab_size,
|
||||
"prompt_logprob_token_ids",
|
||||
)?;
|
||||
validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?;
|
||||
validate_logprob_token_ids(logprobs, logprob_token_ids)
|
||||
}
|
||||
|
||||
fn validate_prompt_logprob_token_ids(
|
||||
token_ids: Option<&[u32]>,
|
||||
start: Option<u32>,
|
||||
) -> Result<(), LogprobsError> {
|
||||
match token_ids {
|
||||
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
|
||||
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
|
||||
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_logprobs_count(
|
||||
requested: Option<i32>,
|
||||
max_logprobs: usize,
|
||||
|
||||
@@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range(
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"prompt_logprob_token_ids",
|
||||
token_ids.iter().copied(),
|
||||
limits.model_vocab_size,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"bad_words",
|
||||
|
||||
@@ -97,6 +97,11 @@ pub struct SamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling. `None` means no
|
||||
/// explicit user override.
|
||||
pub min_p: Option<f32>,
|
||||
@@ -156,6 +161,8 @@ impl Default for SamplingParams {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: None,
|
||||
frequency_penalty: None,
|
||||
presence_penalty: None,
|
||||
|
||||
Reference in New Issue
Block a user