[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
|
||||
use crate::protocol::stats::{PrefillStats, SchedulerStats};
|
||||
use crate::protocol::tensor::WireNdArray;
|
||||
use crate::protocol::{OpaqueValue, decode_msgpack};
|
||||
|
||||
/// The stop reason associated with a finished output.
|
||||
@@ -133,6 +134,10 @@ pub struct EngineCoreOutput {
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled.
|
||||
#[serde(default)]
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
/// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on
|
||||
/// the first output of a request that asked for them.
|
||||
#[serde(default)]
|
||||
pub prompt_token_id_logprobs: Option<WireNdArray>,
|
||||
}
|
||||
|
||||
/// Raw per-sequence speculative-decoding accumulator.
|
||||
@@ -180,6 +185,11 @@ impl EngineCoreOutput {
|
||||
self.new_sampling_mask = (self.new_sampling_mask.take())
|
||||
.map(|value| value.resolve(frames, "new_sampling_mask"))
|
||||
.transpose()?;
|
||||
if let Some(value) = self.prompt_token_id_logprobs.as_mut() {
|
||||
value
|
||||
.resolve_aux_frame(frames)
|
||||
.map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -531,6 +541,7 @@ mod tests {
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
pub min_p: f32,
|
||||
/// Frequency penalty applied by the sampler.
|
||||
@@ -171,6 +176,8 @@ impl EngineCoreSamplingParams {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
|
||||
@@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -116,6 +116,7 @@ class EngineCoreOutput(
|
||||
mm_cache_miss_hashes: list[str] | None = None
|
||||
new_sampling_mask: object | None = None
|
||||
spec_decode_metrics: object | None = None
|
||||
prompt_token_id_logprobs: object | None = None
|
||||
|
||||
|
||||
class ExtendedEngineCoreOutput(EngineCoreOutput):
|
||||
|
||||
Reference in New Issue
Block a user