[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -17,6 +17,7 @@ use crate::error::{Error, Result, ext_value_decode};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
|
||||
use crate::protocol::stats::{PrefillStats, SchedulerStats};
|
||||
use crate::protocol::tensor::WireNdArray;
|
||||
use crate::protocol::{OpaqueValue, decode_msgpack};
|
||||
|
||||
/// The stop reason associated with a finished output.
|
||||
@@ -133,6 +134,10 @@ pub struct EngineCoreOutput {
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled.
|
||||
#[serde(default)]
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
/// Log probabilities of `SamplingParams.prompt_logprob_token_ids`, set on
|
||||
/// the first output of a request that asked for them.
|
||||
#[serde(default)]
|
||||
pub prompt_token_id_logprobs: Option<WireNdArray>,
|
||||
}
|
||||
|
||||
/// Raw per-sequence speculative-decoding accumulator.
|
||||
@@ -180,6 +185,11 @@ impl EngineCoreOutput {
|
||||
self.new_sampling_mask = (self.new_sampling_mask.take())
|
||||
.map(|value| value.resolve(frames, "new_sampling_mask"))
|
||||
.transpose()?;
|
||||
if let Some(value) = self.prompt_token_id_logprobs.as_mut() {
|
||||
value
|
||||
.resolve_aux_frame(frames)
|
||||
.map_err(|message| ext_value_decode!("prompt_token_id_logprobs: {message}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -531,6 +541,7 @@ mod tests {
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -101,6 +101,11 @@ pub struct EngineCoreSamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
pub min_p: f32,
|
||||
/// Frequency penalty applied by the sampler.
|
||||
@@ -171,6 +176,8 @@ impl EngineCoreSamplingParams {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
|
||||
@@ -2684,6 +2684,8 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -2770,6 +2772,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
mm_cache_miss_hashes: None,
|
||||
new_sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
|
||||
@@ -116,6 +116,7 @@ class EngineCoreOutput(
|
||||
mm_cache_miss_hashes: list[str] | None = None
|
||||
new_sampling_mask: object | None = None
|
||||
spec_decode_metrics: object | None = None
|
||||
prompt_token_id_logprobs: object | None = None
|
||||
|
||||
|
||||
class ExtendedEngineCoreOutput(EngineCoreOutput):
|
||||
|
||||
@@ -15,6 +15,7 @@ use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, RequestSpecDecodeMetrics, StopReason,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
|
||||
|
||||
use crate::error::Result;
|
||||
@@ -38,6 +39,7 @@ pub struct CollectedGenerateOutput {
|
||||
pub request_id: String,
|
||||
pub prompt_token_ids: Vec<u32>,
|
||||
pub prompt_logprobs: Option<Logprobs>,
|
||||
pub prompt_token_id_logprobs: Option<WireNdArray>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub logprobs: Option<Logprobs>,
|
||||
pub finish_reason: FinishReason,
|
||||
@@ -62,6 +64,8 @@ pub struct GeneratePromptInfo {
|
||||
/// Prompt logprobs returned by engine-core for scored prompt positions,
|
||||
/// when requested.
|
||||
pub prompt_logprobs: Option<Logprobs>,
|
||||
/// Log probabilities of `prompt_logprob_token_ids`, when requested.
|
||||
pub prompt_token_id_logprobs: Option<WireNdArray>,
|
||||
}
|
||||
|
||||
/// The reason a request finished.
|
||||
@@ -206,6 +210,7 @@ impl GenerateOutput {
|
||||
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
|
||||
prompt_token_ids: ids,
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids,
|
||||
logprobs: None,
|
||||
@@ -247,6 +252,7 @@ impl GenerateOutputStream {
|
||||
pending_prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
raw_stream,
|
||||
request_metrics,
|
||||
@@ -282,6 +288,11 @@ impl Stream for GenerateOutputStream {
|
||||
info.prompt_logprobs =
|
||||
raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap());
|
||||
}
|
||||
if let Some(info) = &mut self.pending_prompt_info
|
||||
&& info.prompt_token_id_logprobs.is_none()
|
||||
{
|
||||
info.prompt_token_id_logprobs = raw.prompt_token_id_logprobs;
|
||||
}
|
||||
|
||||
let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap());
|
||||
let sampling_mask = raw.new_sampling_mask.map(|value| value.into_direct().unwrap());
|
||||
@@ -361,6 +372,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
pin_mut!(stream);
|
||||
let mut prompt_token_ids = None;
|
||||
let mut prompt_logprobs = None;
|
||||
let mut prompt_token_id_logprobs = None;
|
||||
let mut cached_token_count = 0;
|
||||
let mut collected: Option<CollectedGenerateOutput> = None;
|
||||
|
||||
@@ -374,6 +386,9 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
if prompt_logprobs.is_none() {
|
||||
prompt_logprobs = info.prompt_logprobs;
|
||||
}
|
||||
if prompt_token_id_logprobs.is_none() {
|
||||
prompt_token_id_logprobs = info.prompt_token_id_logprobs;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(existing) = collected.as_mut() {
|
||||
@@ -393,6 +408,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
request_id: output.request_id,
|
||||
prompt_token_ids: prompt_token_ids.take().unwrap_or_default(),
|
||||
prompt_logprobs: prompt_logprobs.take(),
|
||||
prompt_token_id_logprobs: prompt_token_id_logprobs.take(),
|
||||
token_ids: output.token_ids,
|
||||
logprobs: output.logprobs,
|
||||
finish_reason: FinishReason::Error,
|
||||
|
||||
@@ -277,6 +277,7 @@ async fn generate_streams_outputs() {
|
||||
Some(GeneratePromptInfo {
|
||||
prompt_token_ids: vec![11, 22].into(),
|
||||
prompt_logprobs: Some(prompt_logprobs()),
|
||||
prompt_token_id_logprobs: None,
|
||||
})
|
||||
);
|
||||
assert_eq!(first.token_ids, vec![1, 2]);
|
||||
@@ -405,6 +406,7 @@ async fn collect_output_rejects_partial_sampling_mask() {
|
||||
prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from([11_u32, 22]),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids: vec![33, 44],
|
||||
logprobs: None,
|
||||
|
||||
@@ -9,6 +9,7 @@ anyhow.workspace = true
|
||||
asynk-strim-attr.workspace = true
|
||||
auto_enums.workspace = true
|
||||
axum.workspace = true
|
||||
base64.workspace = true
|
||||
educe.workspace = true
|
||||
futures.workspace = true
|
||||
http-body.workspace = true
|
||||
|
||||
@@ -16,12 +16,15 @@ use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use base64::Engine as _;
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::{error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::dtype::TensorDtype;
|
||||
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
use vllm_llm::{
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
|
||||
};
|
||||
@@ -330,6 +333,11 @@ fn collect_generate(
|
||||
token_ids: collected.token_ids,
|
||||
}],
|
||||
prompt_logprobs,
|
||||
prompt_token_id_logprobs: collected
|
||||
.prompt_token_id_logprobs
|
||||
.as_ref()
|
||||
.map(npy_base64)
|
||||
.transpose()?,
|
||||
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
|
||||
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
@@ -432,6 +440,29 @@ fn position_to_logprob_map(position: &PositionLogprobs) -> HashMap<u32, Generate
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Encode a float32 matrix as Python `numpy2base64` does.
|
||||
fn npy_base64(array: &WireNdArray) -> Result<String, ApiError> {
|
||||
let (TensorDtype::F32, Some(data), [rows, cols]) = (
|
||||
array.dtype.scalar,
|
||||
array.data.as_raw_view(),
|
||||
array.shape.as_slice(),
|
||||
) else {
|
||||
bail_server_error!("prompt_token_id_logprobs: expected a float32 matrix");
|
||||
};
|
||||
let mut header =
|
||||
format!("{{'descr': '<f4', 'fortran_order': False, 'shape': ({rows}, {cols}), }}");
|
||||
header.push_str(&" ".repeat(21 - rows.to_string().len()));
|
||||
header.push_str(&" ".repeat(64 - (11 + header.len()) % 64));
|
||||
header.push('\n');
|
||||
|
||||
let mut npy = Vec::with_capacity(10 + header.len() + data.len());
|
||||
npy.extend_from_slice(b"\x93NUMPY\x01\x00");
|
||||
npy.extend_from_slice(&(header.len() as u16).to_le_bytes());
|
||||
npy.extend_from_slice(header.as_bytes());
|
||||
npy.extend_from_slice(data);
|
||||
Ok(base64::engine::general_purpose::STANDARD.encode(npy))
|
||||
}
|
||||
|
||||
fn format_token_id(token_id: u32) -> String {
|
||||
format!("token_id:{token_id}")
|
||||
}
|
||||
@@ -509,6 +540,7 @@ mod tests {
|
||||
prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from([11_u32, 22_u32]),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids: vec![33],
|
||||
logprobs: None,
|
||||
@@ -592,6 +624,7 @@ mod tests {
|
||||
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from(ids),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
}),
|
||||
token_ids,
|
||||
logprobs: None,
|
||||
@@ -747,6 +780,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -781,6 +815,17 @@ mod tests {
|
||||
assert!(response.mm_placeholders.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_token_id_logprobs_match_numpy2base64() {
|
||||
let scores =
|
||||
WireNdArray::from_f32(vec![2, 3], vec![-0.5, -1.5, -2.5, -3.5, -4.5, -5.5]).unwrap();
|
||||
// numpy2base64(np.array([[-0.5, -1.5, -2.5], [-3.5, -4.5, -5.5]], np.float32))
|
||||
assert_eq!(
|
||||
npy_base64(&scores).unwrap(),
|
||||
"k05VTVBZAQB2AHsnZGVzY3InOiAnPGY0JywgJ2ZvcnRyYW5fb3JkZXInOiBGYWxzZSwgJ3NoYXBlJzogKDIsIDMpLCB9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAoAAAC/AADAvwAAIMAAAGDAAACQwAAAsMA="
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_returns_spec_decode_metrics() {
|
||||
let metrics = RequestSpecDecodeMetrics {
|
||||
@@ -792,6 +837,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-metrics".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -879,6 +925,7 @@ mod tests {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
@@ -914,6 +961,7 @@ mod tests {
|
||||
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
|
||||
@@ -94,6 +94,8 @@ pub(super) struct GenerateResponse {
|
||||
pub request_id: String,
|
||||
pub choices: Vec<GenerateResponseChoice>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
|
||||
/// Base64 `.npy` float32 array, as Python `numpy2base64` encodes it.
|
||||
pub prompt_token_id_logprobs: Option<String>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
pub mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
|
||||
@@ -57,6 +57,13 @@ pub(crate) fn validate_request_compat(
|
||||
}
|
||||
}
|
||||
|
||||
if request.stream && request.sampling_params.inner.prompt_logprob_token_ids.is_some() {
|
||||
bail_invalid_request!(
|
||||
param = "sampling_params",
|
||||
"`prompt_logprob_token_ids` are not available when `stream=true`."
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -169,6 +176,17 @@ mod tests {
|
||||
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_rejects_streaming_prompt_logprob_token_ids() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"token_ids": [11, 22],
|
||||
"stream": true,
|
||||
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
|
||||
}))
|
||||
.expect("parse request");
|
||||
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
|
||||
@@ -159,6 +159,8 @@ pub(super) fn prepare_chat_request(
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs: request.logprobs.then_some(top_logprobs),
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: request.min_p,
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
|
||||
@@ -124,6 +124,8 @@ pub(super) fn prepare_completion_request(
|
||||
thinking_token_budget: request.thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: request.min_p,
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
|
||||
@@ -103,6 +103,8 @@ pub fn lower_sampling_params(
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_start,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
@@ -123,6 +125,8 @@ pub fn lower_sampling_params(
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
logprob_token_ids.as_deref(),
|
||||
prompt_logprob_token_ids.as_deref(),
|
||||
prompt_logprob_start,
|
||||
sampling_limits,
|
||||
)?;
|
||||
validate_repetition_detection(repetition_detection.as_ref())?;
|
||||
@@ -176,6 +180,8 @@ pub fn lower_sampling_params(
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_start,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
@@ -646,6 +652,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -701,6 +709,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -870,6 +880,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -935,6 +947,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -1008,6 +1022,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
@@ -1122,6 +1138,44 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
|
||||
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
|
||||
lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
prompt_logprob_token_ids: ids,
|
||||
prompt_logprob_start: start,
|
||||
..Default::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
};
|
||||
let rejects = |ids, start| match lower(ids, start) {
|
||||
Err(error) => error.to_string(),
|
||||
Ok(_) => panic!("expected rejection"),
|
||||
};
|
||||
|
||||
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
|
||||
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
|
||||
assert_eq!(params.prompt_logprob_start, Some(1));
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
|
||||
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
"prompt_logprob_start requires prompt_logprob_token_ids.",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&[
|
||||
rejects(Some(vec![]), None),
|
||||
rejects(Some((0..21).collect()), None),
|
||||
rejects(Some(vec![1000]), None),
|
||||
rejects(Some(vec![1, 1]), None),
|
||||
rejects(None, Some(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_rejects_out_of_vocab_stop_token_ids() {
|
||||
let error = lower_sampling_params_with_limits(
|
||||
@@ -1259,6 +1313,8 @@ mod tests {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: 0.1,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! `-1` is expanded only for bounds checks. The original request values are
|
||||
//! passed through to engine-core.
|
||||
|
||||
use itertools::Itertools as _;
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::backend::SamplingLimits;
|
||||
@@ -35,6 +36,12 @@ pub enum LogprobsError {
|
||||
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
|
||||
)]
|
||||
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
|
||||
#[error("prompt_logprob_token_ids must not be empty.")]
|
||||
EmptyPromptLogprobTokenIds,
|
||||
#[error("prompt_logprob_token_ids must not contain duplicates.")]
|
||||
DuplicatePromptLogprobTokenIds,
|
||||
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
|
||||
PromptLogprobStartWithoutTokenIds,
|
||||
}
|
||||
|
||||
/// Validate logprobs count sampling parameters.
|
||||
@@ -42,6 +49,8 @@ pub(super) fn validate_logprobs(
|
||||
logprobs: Option<i32>,
|
||||
prompt_logprobs: Option<i32>,
|
||||
logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_start: Option<u32>,
|
||||
sampling_limits: SamplingLimits,
|
||||
) -> Result<(), LogprobsError> {
|
||||
let vocab_size = sampling_limits.model_vocab_size;
|
||||
@@ -50,9 +59,28 @@ pub(super) fn validate_logprobs(
|
||||
|
||||
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
|
||||
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
|
||||
validate_logprobs_count(
|
||||
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
|
||||
max_logprobs,
|
||||
vocab_size,
|
||||
"prompt_logprob_token_ids",
|
||||
)?;
|
||||
validate_prompt_logprob_token_ids(prompt_logprob_token_ids, prompt_logprob_start)?;
|
||||
validate_logprob_token_ids(logprobs, logprob_token_ids)
|
||||
}
|
||||
|
||||
fn validate_prompt_logprob_token_ids(
|
||||
token_ids: Option<&[u32]>,
|
||||
start: Option<u32>,
|
||||
) -> Result<(), LogprobsError> {
|
||||
match token_ids {
|
||||
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
|
||||
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
|
||||
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_logprobs_count(
|
||||
requested: Option<i32>,
|
||||
max_logprobs: usize,
|
||||
|
||||
@@ -95,6 +95,14 @@ pub(crate) fn validate_vocab_range(
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"prompt_logprob_token_ids",
|
||||
token_ids.iter().copied(),
|
||||
limits.model_vocab_size,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"bad_words",
|
||||
|
||||
@@ -97,6 +97,11 @@ pub struct SamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling. `None` means no
|
||||
/// explicit user override.
|
||||
pub min_p: Option<f32>,
|
||||
@@ -156,6 +161,8 @@ impl Default for SamplingParams {
|
||||
thinking_token_budget: None,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
prompt_logprob_token_ids: None,
|
||||
prompt_logprob_start: None,
|
||||
min_p: None,
|
||||
frequency_penalty: None,
|
||||
presence_penalty: None,
|
||||
|
||||
Reference in New Issue
Block a user