Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
bc21cba967
commit
e319f86f15
@@ -152,6 +152,11 @@ impl EngineCoreRequest {
|
||||
feature.extract_aux_frames(&mut aux_frames, threshold);
|
||||
}
|
||||
}
|
||||
if let Some(params) = &mut self.sampling_params
|
||||
&& let Some(ids) = &mut params.prompt_logprob_token_ids
|
||||
{
|
||||
ids.extract_aux_frame(&mut aux_frames, threshold);
|
||||
}
|
||||
aux_frames
|
||||
}
|
||||
}
|
||||
@@ -168,7 +173,7 @@ mod tests {
|
||||
PlaceholderRange,
|
||||
};
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::tensor::{WireArrayData, WireTensor};
|
||||
use crate::protocol::tensor::{WireArrayData, WireNdArray, WireTensor};
|
||||
use crate::protocol::{decode_value, encode_msgpack};
|
||||
|
||||
const AUX_FRAME_THRESHOLD: usize = 256;
|
||||
@@ -294,4 +299,27 @@ mod tests {
|
||||
);
|
||||
assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_request_extracts_large_prompt_logprob_token_ids() {
|
||||
let ids = WireNdArray::from_i32(vec![AUX_FRAME_THRESHOLD, 1], vec![5; AUX_FRAME_THRESHOLD])
|
||||
.unwrap();
|
||||
let mut request = EngineCoreRequest {
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
prompt_logprob_token_ids: Some(ids),
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD);
|
||||
|
||||
assert_eq!(aux_frames.len(), 1);
|
||||
assert_eq!(aux_frames[0].len(), AUX_FRAME_THRESHOLD * 4);
|
||||
let params = request.sampling_params.unwrap();
|
||||
assert_eq!(
|
||||
params.prompt_logprob_token_ids.unwrap().data,
|
||||
WireArrayData::AuxIndex(1)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
|
||||
use crate::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use crate::protocol::tensor::WireNdArray;
|
||||
|
||||
fn default_top_p() -> f32 {
|
||||
1.0
|
||||
@@ -101,9 +102,9 @@ pub struct EngineCoreSamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// `[num_rows, num_ids]` candidate token IDs per scored causal prompt row,
|
||||
/// `-1` padding shorter rows.
|
||||
pub prompt_logprob_token_ids: Option<WireNdArray>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
|
||||
@@ -121,6 +121,18 @@ impl<Dtype: From<TensorDtype> + Serialize + DeserializeOwned> WireArray<Dtype> {
|
||||
))
|
||||
}
|
||||
|
||||
/// Build an int32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
pub fn from_i32(shape: Vec<usize>, data: Vec<i32>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self::from_raw_bytes(
|
||||
TensorDtype::I32,
|
||||
shape,
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
}
|
||||
|
||||
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
|
||||
@@ -181,7 +181,7 @@ mod tests {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"token_ids": [11, 22],
|
||||
"stream": true,
|
||||
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
|
||||
"sampling_params": {"prompt_logprob_token_ids": [[1, 2]]}
|
||||
}))
|
||||
.expect("parse request");
|
||||
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
|
||||
|
||||
+33
-16
@@ -7,7 +7,7 @@ pub(crate) mod logprobs;
|
||||
pub(crate) mod sampling;
|
||||
pub(crate) mod token_ids;
|
||||
|
||||
use logprobs::validate_logprobs;
|
||||
use logprobs::{lower_prompt_logprob_token_ids, validate_logprobs};
|
||||
use sampling::validate_resolved_sampling_params;
|
||||
use token_ids::{validate_prompt_token_ids, validate_vocab_range};
|
||||
use vllm_engine_core_client::protocol::sampling::{
|
||||
@@ -50,7 +50,11 @@ pub fn lower_text_request(
|
||||
// multimodal tensor payloads.
|
||||
mm_features: request.mm_features.take(),
|
||||
sampling_params: lower_sampling_params(
|
||||
request.sampling_params.clone(),
|
||||
SamplingParams {
|
||||
// Move the candidate table rather than cloning it; only lowering reads it.
|
||||
prompt_logprob_token_ids: request.sampling_params.prompt_logprob_token_ids.take(),
|
||||
..request.sampling_params.clone()
|
||||
},
|
||||
sampling_hints,
|
||||
sampling_limits,
|
||||
prompt_len,
|
||||
@@ -180,7 +184,12 @@ pub fn lower_sampling_params(
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_token_ids: lower_prompt_logprob_token_ids(
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_start,
|
||||
prompt_len,
|
||||
sampling_limits.model_vocab_size,
|
||||
)?,
|
||||
prompt_logprob_start,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
@@ -332,6 +341,7 @@ mod tests {
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmFeatureSpec, MmModality, PlaceholderRange,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::*;
|
||||
@@ -1140,7 +1150,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
|
||||
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
|
||||
let lower = |ids: Option<Vec<Vec<i32>>>, start: Option<u32>| {
|
||||
lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
prompt_logprob_token_ids: ids,
|
||||
@@ -1155,23 +1165,30 @@ mod tests {
|
||||
Ok(_) => panic!("expected rejection"),
|
||||
};
|
||||
|
||||
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
|
||||
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
|
||||
assert_eq!(params.prompt_logprob_start, Some(1));
|
||||
let params = lower(Some(vec![vec![1, 2], vec![3]]), None).unwrap();
|
||||
assert_eq!(
|
||||
params.prompt_logprob_token_ids,
|
||||
Some(WireNdArray::from_i32(vec![2, 2], vec![1, 2, 3, -1]).unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
lower(Some(vec![vec![5]]), Some(1)).unwrap().prompt_logprob_start,
|
||||
Some(1)
|
||||
);
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
"prompt_logprob_token_ids must be a non-empty integer array of shape [num_rows, num_ids].",
|
||||
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
|
||||
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids has 1 rows, but the prompt has 2 scored rows (prompt_len - 1 - prompt_logprob_start).",
|
||||
"prompt_logprob_start requires prompt_logprob_token_ids.",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&[
|
||||
rejects(Some(vec![]), None),
|
||||
rejects(Some((0..21).collect()), None),
|
||||
rejects(Some(vec![1000]), None),
|
||||
rejects(Some(vec![1, 1]), None),
|
||||
"#]].assert_debug_eq(&[
|
||||
rejects(Some(vec![vec![]]), None),
|
||||
rejects(Some(vec![(0..21).collect(), vec![0]]), None),
|
||||
rejects(Some(vec![vec![1000], vec![0]]), None),
|
||||
rejects(Some(vec![vec![-2], vec![0]]), None),
|
||||
rejects(Some(vec![vec![1]]), None),
|
||||
rejects(None, Some(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
|
||||
//! Python-compatible validation for logprobs sampling params.
|
||||
//!
|
||||
//! `-1` is expanded only for bounds checks. The original request values are
|
||||
//! passed through to engine-core.
|
||||
//! `-1` counts are expanded only for bounds checks and passed to engine-core
|
||||
//! unchanged; `prompt_logprob_token_ids` rows are padded into an array.
|
||||
|
||||
use itertools::Itertools as _;
|
||||
use thiserror::Error;
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
|
||||
use crate::backend::SamplingLimits;
|
||||
|
||||
@@ -36,10 +36,21 @@ pub enum LogprobsError {
|
||||
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
|
||||
)]
|
||||
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
|
||||
#[error("prompt_logprob_token_ids must not be empty.")]
|
||||
#[error(
|
||||
"prompt_logprob_token_ids must be a non-empty integer array of shape \
|
||||
[num_rows, num_ids]."
|
||||
)]
|
||||
EmptyPromptLogprobTokenIds,
|
||||
#[error("prompt_logprob_token_ids must not contain duplicates.")]
|
||||
DuplicatePromptLogprobTokenIds,
|
||||
#[error(
|
||||
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). \
|
||||
Vocabulary size: {vocab_size}"
|
||||
)]
|
||||
PromptLogprobTokenIdsOutOfVocab { vocab_size: usize },
|
||||
#[error(
|
||||
"prompt_logprob_token_ids has {rows} rows, but the prompt has {expected} scored \
|
||||
rows (prompt_len - 1 - prompt_logprob_start)."
|
||||
)]
|
||||
PromptLogprobRowsMismatch { rows: usize, expected: usize },
|
||||
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
|
||||
PromptLogprobStartWithoutTokenIds,
|
||||
}
|
||||
@@ -49,7 +60,7 @@ pub(super) fn validate_logprobs(
|
||||
logprobs: Option<i32>,
|
||||
prompt_logprobs: Option<i32>,
|
||||
logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_token_ids: Option<&[Vec<i32>]>,
|
||||
prompt_logprob_start: Option<u32>,
|
||||
sampling_limits: SamplingLimits,
|
||||
) -> Result<(), LogprobsError> {
|
||||
@@ -60,7 +71,7 @@ pub(super) fn validate_logprobs(
|
||||
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
|
||||
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
|
||||
validate_logprobs_count(
|
||||
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
|
||||
prompt_logprob_token_ids.map(|rows| rows.iter().map(Vec::len).max().unwrap_or(0) as i32),
|
||||
max_logprobs,
|
||||
vocab_size,
|
||||
"prompt_logprob_token_ids",
|
||||
@@ -70,17 +81,49 @@ pub(super) fn validate_logprobs(
|
||||
}
|
||||
|
||||
fn validate_prompt_logprob_token_ids(
|
||||
token_ids: Option<&[u32]>,
|
||||
rows: Option<&[Vec<i32>]>,
|
||||
start: Option<u32>,
|
||||
) -> Result<(), LogprobsError> {
|
||||
match token_ids {
|
||||
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
|
||||
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
|
||||
match rows {
|
||||
Some(rows) if rows.iter().all(Vec::is_empty) => {
|
||||
Err(LogprobsError::EmptyPromptLogprobTokenIds)
|
||||
}
|
||||
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check per-row candidates against the vocabulary and the prompt as Python
|
||||
/// does, and pad them with `-1` into the engine's `[num_rows, num_ids]` array.
|
||||
pub(super) fn lower_prompt_logprob_token_ids(
|
||||
rows: Option<Vec<Vec<i32>>>,
|
||||
start: Option<u32>,
|
||||
prompt_len: u32,
|
||||
vocab_size: usize,
|
||||
) -> Result<Option<WireNdArray>, LogprobsError> {
|
||||
let Some(rows) = rows else {
|
||||
return Ok(None);
|
||||
};
|
||||
if rows.iter().flatten().any(|&id| id < -1 || i64::from(id) >= vocab_size as i64) {
|
||||
return Err(LogprobsError::PromptLogprobTokenIdsOutOfVocab { vocab_size });
|
||||
}
|
||||
let expected = (prompt_len as usize).saturating_sub(1 + start.unwrap_or(0) as usize);
|
||||
if rows.len() != expected {
|
||||
return Err(LogprobsError::PromptLogprobRowsMismatch {
|
||||
rows: rows.len(),
|
||||
expected,
|
||||
});
|
||||
}
|
||||
let width = rows.iter().map(Vec::len).max().unwrap_or(0);
|
||||
let data = rows
|
||||
.iter()
|
||||
.flat_map(|row| row.iter().copied().chain(std::iter::repeat_n(-1, width - row.len())))
|
||||
.collect();
|
||||
Ok(Some(
|
||||
WireNdArray::from_i32(vec![rows.len(), width], data).expect("padded rows are rectangular"),
|
||||
))
|
||||
}
|
||||
|
||||
fn validate_logprobs_count(
|
||||
requested: Option<i32>,
|
||||
max_logprobs: usize,
|
||||
|
||||
@@ -95,14 +95,6 @@ pub(crate) fn validate_vocab_range(
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"prompt_logprob_token_ids",
|
||||
token_ids.iter().copied(),
|
||||
limits.model_vocab_size,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"bad_words",
|
||||
|
||||
@@ -97,9 +97,9 @@ pub struct SamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// Candidate token IDs per scored causal prompt row, where row `i` scores
|
||||
/// its IDs as predictions of prompt token `prompt_logprob_start + i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<Vec<i32>>>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling. `None` means no
|
||||
|
||||
Reference in New Issue
Block a user