[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
@@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
use crate::protocol::tensor::WireNdArray;
use crate::protocol::{OpaqueValue, decode_msgpack};
/// The stop reason associated with a finished output.
@@ -133,6 +134,10 @@ pub struct EngineCoreOutput {
/// output when `--per-request-spec-decode-metrics` is enabled.
#[serde(default)]
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
/// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on
/// the first output of a request that asked for them.
#[serde(default)]
pub prompt_token_id_logprobs: Option<WireNdArray>,
}
/// Raw per-sequence speculative-decoding accumulator.
@@ -180,6 +185,11 @@ impl EngineCoreOutput {
self.new_sampling_mask = (self.new_sampling_mask.take())
.map(|value| value.resolve(frames, "new_sampling_mask"))
.transpose()?;
if let Some(value) = self.prompt_token_id_logprobs.as_mut() {
value
.resolve_aux_frame(frames)
.map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?;
}
Ok(())
}
}
@@ -531,6 +541,7 @@ mod tests {
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
prompt_token_id_logprobs: None,
},
],
scheduler_stats: None,
@@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling.
pub min_p: f32,
/// Frequency penalty applied by the sampler.
@@ -171,6 +176,8 @@ impl EngineCoreSamplingParams {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
thinking_token_budget: None,
logprobs: None,
prompt_logprobs: None,
prompt_logprob_token_ids: None,
prompt_logprob_start: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
@@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
mm_cache_miss_hashes: None,
new_sampling_mask: None,
spec_decode_metrics: None,
prompt_token_id_logprobs: None,
},
],
scheduler_stats: None,
@@ -116,6 +116,7 @@ class EngineCoreOutput(
mm_cache_miss_hashes: list[str] | None = None
new_sampling_mask: object | None = None
spec_decode_metrics: object | None = None
prompt_token_id_logprobs: object | None = None
class ExtendedEngineCoreOutput(EngineCoreOutput):