[Feature] Per-row candidate IDs for prefill token scoring (M2 of #56860) (#56984)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-03 14:17:09 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent bc21cba967
commit e319f86f15
17 changed files with 366 additions and 162 deletions
@@ -152,6 +152,11 @@ impl EngineCoreRequest {
feature.extract_aux_frames(&mut aux_frames, threshold);
}
}
if let Some(params) = &mut self.sampling_params
&& let Some(ids) = &mut params.prompt_logprob_token_ids
{
ids.extract_aux_frame(&mut aux_frames, threshold);
}
aux_frames
}
}
@@ -168,7 +173,7 @@ mod tests {
PlaceholderRange,
};
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::tensor::{WireArrayData, WireTensor};
use crate::protocol::tensor::{WireArrayData, WireNdArray, WireTensor};
use crate::protocol::{decode_value, encode_msgpack};
const AUX_FRAME_THRESHOLD: usize = 256;
@@ -294,4 +299,27 @@ mod tests {
);
assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty());
}
#[test]
fn engine_core_request_extracts_large_prompt_logprob_token_ids() {
let ids = WireNdArray::from_i32(vec![AUX_FRAME_THRESHOLD, 1], vec![5; AUX_FRAME_THRESHOLD])
.unwrap();
let mut request = EngineCoreRequest {
sampling_params: Some(EngineCoreSamplingParams {
prompt_logprob_token_ids: Some(ids),
..EngineCoreSamplingParams::for_test()
}),
..EngineCoreRequest::default()
};
let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD);
assert_eq!(aux_frames.len(), 1);
assert_eq!(aux_frames[0].len(), AUX_FRAME_THRESHOLD * 4);
let params = request.sampling_params.unwrap();
assert_eq!(
params.prompt_logprob_token_ids.unwrap().data,
WireArrayData::AuxIndex(1)
);
}
}
@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use crate::protocol::structured_outputs::StructuredOutputsParams;
use crate::protocol::tensor::WireNdArray;
fn default_top_p() -> f32 {
1.0
@@ -101,9 +102,9 @@ pub struct EngineCoreSamplingParams {
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Candidate token IDs scored at every scored causal prompt row, where row
/// `i` scores them as predictions of prompt token `i + 1`.
pub prompt_logprob_token_ids: Option<Vec<u32>>,
/// `[num_rows, num_ids]` candidate token IDs per scored causal prompt row,
/// `-1` padding shorter rows.
pub prompt_logprob_token_ids: Option<WireNdArray>,
/// First causal prompt row to score; `None` scores from the first row.
pub prompt_logprob_start: Option<u32>,
/// Minimum probability threshold for token sampling.
@@ -121,6 +121,18 @@ impl<Dtype: From<TensorDtype> + Serialize + DeserializeOwned> WireArray<Dtype> {
))
}
/// Build an int32 tensor/ndarray backed by native-endian raw-view bytes.
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_i32(shape: Vec<usize>, data: Vec<i32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self::from_raw_bytes(
TensorDtype::I32,
shape,
bytes_from_pod_vec(data),
))
}
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
///
/// Takes ownership of the backing buffer without copying its data.