Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
bc21cba967
commit
e319f86f15
@@ -2,14 +2,14 @@
|
||||
|
||||
In on-policy distillation, the student records its top-K candidate tokens at
|
||||
every response position, and the teacher provides the log probabilities of
|
||||
exactly those candidates as the distillation target. The caller passes the
|
||||
candidates as `prompt_logprob_token_ids`; the teacher runs a single prefill
|
||||
over the prompt plus response and gathers their log probabilities on the GPU at
|
||||
every scored row, starting from `prompt_logprob_start` (typically the prompt
|
||||
length minus one, so only response positions are scored). The result is a
|
||||
`[rows, K]` matrix whose row `i` scores prompt token
|
||||
`prompt_logprob_start + i + 1` and whose columns follow the candidate order, so
|
||||
it lines up position by position with the student's top-K.
|
||||
exactly those candidates as the distillation target. The caller passes one
|
||||
candidate list per scored row as `prompt_logprob_token_ids`; the teacher runs a
|
||||
single prefill over the prompt plus response and gathers their log
|
||||
probabilities on the GPU, starting from `prompt_logprob_start` (typically the
|
||||
prompt length minus one, so only response positions are scored). The result is
|
||||
a `[rows, K]` matrix whose row `i` scores prompt token
|
||||
`prompt_logprob_start + i + 1` and whose columns follow that row's candidate
|
||||
order, so it lines up position by position with the student's top-K.
|
||||
|
||||
## Quick start
|
||||
|
||||
@@ -22,13 +22,13 @@ output = llm.generate(
|
||||
TokensPrompt(prompt_token_ids=prompt_ids + response_ids),
|
||||
SamplingParams(
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=candidate_ids,
|
||||
prompt_logprob_token_ids=student_topk_ids, # [len(response_ids), K]
|
||||
prompt_logprob_start=len(prompt_ids) - 1,
|
||||
),
|
||||
)
|
||||
scores = output[0].prompt_token_id_logprobs
|
||||
# scores.shape == (len(response_ids), len(candidate_ids))
|
||||
# scores[i, j] = log p(candidate_ids[j] | prompt_ids + response_ids[:i])
|
||||
# scores.shape == (len(response_ids), K)
|
||||
# scores[i, j] = log p(student_topk_ids[i][j] | prompt_ids + response_ids[:i])
|
||||
```
|
||||
|
||||
The same parameters are accepted by the `/inference/v1/generate` HTTP endpoint
|
||||
@@ -42,13 +42,17 @@ import numpy as np
|
||||
scores = np.load(io.BytesIO(base64.b64decode(response["prompt_token_id_logprobs"])))
|
||||
```
|
||||
|
||||
The matrix has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows. With a
|
||||
logits `--logprobs-mode`, it holds logits instead of log probabilities.
|
||||
`prompt_logprob_token_ids` is an integer array or a list of lists with exactly
|
||||
`prompt_len - 1 - prompt_logprob_start` rows; shorter rows are padded with `-1`,
|
||||
and every `-1` entry scores `-inf`. Pass an `int32` NumPy array, such as the
|
||||
student's top-K IDs, which is sent to the engine without conversion; nested
|
||||
lists of Python ints are much slower to copy and serialize. With a logits
|
||||
`--logprobs-mode`, the matrix holds logits instead of log probabilities.
|
||||
|
||||
## Requirements
|
||||
|
||||
- The V2 model runner (`VLLM_USE_V2_MODEL_RUNNER=1`).
|
||||
- `len(prompt_logprob_token_ids)` at most `--max-logprobs`.
|
||||
- The longest row at most `--max-logprobs` candidates.
|
||||
- No `--kv-sharing-fast-prefill`.
|
||||
|
||||
The request skips reading the prefix cache (local and KV connector), since
|
||||
@@ -56,8 +60,6 @@ cached rows have no logits to score.
|
||||
|
||||
## Limitations
|
||||
|
||||
- One candidate set per request, shared by every scored row; per-position
|
||||
candidates are planned ([#56860](https://github.com/vllm-project/vllm/issues/56860)).
|
||||
- Non-streaming only; `stream=true` requests are rejected.
|
||||
- Not exposed through the OpenAI-compatible endpoints.
|
||||
- If a prefill starts past the first scored row (e.g. `skip_reading_prefix_cache=False`
|
||||
|
||||
@@ -152,6 +152,11 @@ impl EngineCoreRequest {
|
||||
feature.extract_aux_frames(&mut aux_frames, threshold);
|
||||
}
|
||||
}
|
||||
if let Some(params) = &mut self.sampling_params
|
||||
&& let Some(ids) = &mut params.prompt_logprob_token_ids
|
||||
{
|
||||
ids.extract_aux_frame(&mut aux_frames, threshold);
|
||||
}
|
||||
aux_frames
|
||||
}
|
||||
}
|
||||
@@ -168,7 +173,7 @@ mod tests {
|
||||
PlaceholderRange,
|
||||
};
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::tensor::{WireArrayData, WireTensor};
|
||||
use crate::protocol::tensor::{WireArrayData, WireNdArray, WireTensor};
|
||||
use crate::protocol::{decode_value, encode_msgpack};
|
||||
|
||||
const AUX_FRAME_THRESHOLD: usize = 256;
|
||||
@@ -294,4 +299,27 @@ mod tests {
|
||||
);
|
||||
assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_request_extracts_large_prompt_logprob_token_ids() {
|
||||
let ids = WireNdArray::from_i32(vec![AUX_FRAME_THRESHOLD, 1], vec![5; AUX_FRAME_THRESHOLD])
|
||||
.unwrap();
|
||||
let mut request = EngineCoreRequest {
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
prompt_logprob_token_ids: Some(ids),
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD);
|
||||
|
||||
assert_eq!(aux_frames.len(), 1);
|
||||
assert_eq!(aux_frames[0].len(), AUX_FRAME_THRESHOLD * 4);
|
||||
let params = request.sampling_params.unwrap();
|
||||
assert_eq!(
|
||||
params.prompt_logprob_token_ids.unwrap().data,
|
||||
WireArrayData::AuxIndex(1)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
|
||||
use crate::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use crate::protocol::tensor::WireNdArray;
|
||||
|
||||
fn default_top_p() -> f32 {
|
||||
1.0
|
||||
@@ -101,9 +102,9 @@ pub struct EngineCoreSamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// `[num_rows, num_ids]` candidate token IDs per scored causal prompt row,
|
||||
/// `-1` padding shorter rows.
|
||||
pub prompt_logprob_token_ids: Option<WireNdArray>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
|
||||
@@ -121,6 +121,18 @@ impl<Dtype: From<TensorDtype> + Serialize + DeserializeOwned> WireArray<Dtype> {
|
||||
))
|
||||
}
|
||||
|
||||
/// Build an int32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
pub fn from_i32(shape: Vec<usize>, data: Vec<i32>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self::from_raw_bytes(
|
||||
TensorDtype::I32,
|
||||
shape,
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
}
|
||||
|
||||
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
|
||||
@@ -181,7 +181,7 @@ mod tests {
|
||||
let request: GenerateRequest = serde_json::from_value(json!({
|
||||
"token_ids": [11, 22],
|
||||
"stream": true,
|
||||
"sampling_params": {"prompt_logprob_token_ids": [1, 2]}
|
||||
"sampling_params": {"prompt_logprob_token_ids": [[1, 2]]}
|
||||
}))
|
||||
.expect("parse request");
|
||||
assert!(validate_request_compat(&request, &served(&["test-model"])).is_err());
|
||||
|
||||
+33
-16
@@ -7,7 +7,7 @@ pub(crate) mod logprobs;
|
||||
pub(crate) mod sampling;
|
||||
pub(crate) mod token_ids;
|
||||
|
||||
use logprobs::validate_logprobs;
|
||||
use logprobs::{lower_prompt_logprob_token_ids, validate_logprobs};
|
||||
use sampling::validate_resolved_sampling_params;
|
||||
use token_ids::{validate_prompt_token_ids, validate_vocab_range};
|
||||
use vllm_engine_core_client::protocol::sampling::{
|
||||
@@ -50,7 +50,11 @@ pub fn lower_text_request(
|
||||
// multimodal tensor payloads.
|
||||
mm_features: request.mm_features.take(),
|
||||
sampling_params: lower_sampling_params(
|
||||
request.sampling_params.clone(),
|
||||
SamplingParams {
|
||||
// Move the candidate table rather than cloning it; only lowering reads it.
|
||||
prompt_logprob_token_ids: request.sampling_params.prompt_logprob_token_ids.take(),
|
||||
..request.sampling_params.clone()
|
||||
},
|
||||
sampling_hints,
|
||||
sampling_limits,
|
||||
prompt_len,
|
||||
@@ -180,7 +184,12 @@ pub fn lower_sampling_params(
|
||||
thinking_token_budget,
|
||||
logprobs,
|
||||
prompt_logprobs,
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_token_ids: lower_prompt_logprob_token_ids(
|
||||
prompt_logprob_token_ids,
|
||||
prompt_logprob_start,
|
||||
prompt_len,
|
||||
sampling_limits.model_vocab_size,
|
||||
)?,
|
||||
prompt_logprob_start,
|
||||
min_p,
|
||||
frequency_penalty,
|
||||
@@ -332,6 +341,7 @@ mod tests {
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmFeatureSpec, MmModality, PlaceholderRange,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::*;
|
||||
@@ -1140,7 +1150,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_validates_prompt_logprob_token_ids() {
|
||||
let lower = |ids: Option<Vec<u32>>, start: Option<u32>| {
|
||||
let lower = |ids: Option<Vec<Vec<i32>>>, start: Option<u32>| {
|
||||
lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
prompt_logprob_token_ids: ids,
|
||||
@@ -1155,23 +1165,30 @@ mod tests {
|
||||
Ok(_) => panic!("expected rejection"),
|
||||
};
|
||||
|
||||
let params = lower(Some(vec![1, 2]), Some(1)).unwrap();
|
||||
assert_eq!(params.prompt_logprob_token_ids, Some(vec![1, 2]));
|
||||
assert_eq!(params.prompt_logprob_start, Some(1));
|
||||
let params = lower(Some(vec![vec![1, 2], vec![3]]), None).unwrap();
|
||||
assert_eq!(
|
||||
params.prompt_logprob_token_ids,
|
||||
Some(WireNdArray::from_i32(vec![2, 2], vec![1, 2, 3, -1]).unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
lower(Some(vec![vec![5]]), Some(1)).unwrap().prompt_logprob_start,
|
||||
Some(1)
|
||||
);
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
"prompt_logprob_token_ids must be a non-empty integer array of shape [num_rows, num_ids].",
|
||||
"requested prompt_logprob_token_ids of 21, which is greater than max allowed: 20",
|
||||
"token_id(s) [1000] in prompt_logprob_token_ids are out of vocabulary. Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). Vocabulary size: 1000",
|
||||
"prompt_logprob_token_ids has 1 rows, but the prompt has 2 scored rows (prompt_len - 1 - prompt_logprob_start).",
|
||||
"prompt_logprob_start requires prompt_logprob_token_ids.",
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&[
|
||||
rejects(Some(vec![]), None),
|
||||
rejects(Some((0..21).collect()), None),
|
||||
rejects(Some(vec![1000]), None),
|
||||
rejects(Some(vec![1, 1]), None),
|
||||
"#]].assert_debug_eq(&[
|
||||
rejects(Some(vec![vec![]]), None),
|
||||
rejects(Some(vec![(0..21).collect(), vec![0]]), None),
|
||||
rejects(Some(vec![vec![1000], vec![0]]), None),
|
||||
rejects(Some(vec![vec![-2], vec![0]]), None),
|
||||
rejects(Some(vec![vec![1]]), None),
|
||||
rejects(None, Some(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
|
||||
//! Python-compatible validation for logprobs sampling params.
|
||||
//!
|
||||
//! `-1` is expanded only for bounds checks. The original request values are
|
||||
//! passed through to engine-core.
|
||||
//! `-1` counts are expanded only for bounds checks and passed to engine-core
|
||||
//! unchanged; `prompt_logprob_token_ids` rows are padded into an array.
|
||||
|
||||
use itertools::Itertools as _;
|
||||
use thiserror::Error;
|
||||
use vllm_engine_core_client::protocol::tensor::WireNdArray;
|
||||
|
||||
use crate::backend::SamplingLimits;
|
||||
|
||||
@@ -36,10 +36,21 @@ pub enum LogprobsError {
|
||||
len(logprob_token_ids). Got logprobs={logprobs}, len(logprob_token_ids)={num_token_ids}."
|
||||
)]
|
||||
TokenIdsMismatch { logprobs: i32, num_token_ids: usize },
|
||||
#[error("prompt_logprob_token_ids must not be empty.")]
|
||||
#[error(
|
||||
"prompt_logprob_token_ids must be a non-empty integer array of shape \
|
||||
[num_rows, num_ids]."
|
||||
)]
|
||||
EmptyPromptLogprobTokenIds,
|
||||
#[error("prompt_logprob_token_ids must not contain duplicates.")]
|
||||
DuplicatePromptLogprobTokenIds,
|
||||
#[error(
|
||||
"prompt_logprob_token_ids contain out-of-vocab token ids (-1 pads a row). \
|
||||
Vocabulary size: {vocab_size}"
|
||||
)]
|
||||
PromptLogprobTokenIdsOutOfVocab { vocab_size: usize },
|
||||
#[error(
|
||||
"prompt_logprob_token_ids has {rows} rows, but the prompt has {expected} scored \
|
||||
rows (prompt_len - 1 - prompt_logprob_start)."
|
||||
)]
|
||||
PromptLogprobRowsMismatch { rows: usize, expected: usize },
|
||||
#[error("prompt_logprob_start requires prompt_logprob_token_ids.")]
|
||||
PromptLogprobStartWithoutTokenIds,
|
||||
}
|
||||
@@ -49,7 +60,7 @@ pub(super) fn validate_logprobs(
|
||||
logprobs: Option<i32>,
|
||||
prompt_logprobs: Option<i32>,
|
||||
logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_token_ids: Option<&[u32]>,
|
||||
prompt_logprob_token_ids: Option<&[Vec<i32>]>,
|
||||
prompt_logprob_start: Option<u32>,
|
||||
sampling_limits: SamplingLimits,
|
||||
) -> Result<(), LogprobsError> {
|
||||
@@ -60,7 +71,7 @@ pub(super) fn validate_logprobs(
|
||||
validate_logprobs_count(logprobs, max_logprobs, vocab_size, "logprobs")?;
|
||||
validate_logprobs_count(prompt_logprobs, max_logprobs, vocab_size, "prompt_logprobs")?;
|
||||
validate_logprobs_count(
|
||||
prompt_logprob_token_ids.map(|ids| ids.len() as i32),
|
||||
prompt_logprob_token_ids.map(|rows| rows.iter().map(Vec::len).max().unwrap_or(0) as i32),
|
||||
max_logprobs,
|
||||
vocab_size,
|
||||
"prompt_logprob_token_ids",
|
||||
@@ -70,17 +81,49 @@ pub(super) fn validate_logprobs(
|
||||
}
|
||||
|
||||
fn validate_prompt_logprob_token_ids(
|
||||
token_ids: Option<&[u32]>,
|
||||
rows: Option<&[Vec<i32>]>,
|
||||
start: Option<u32>,
|
||||
) -> Result<(), LogprobsError> {
|
||||
match token_ids {
|
||||
Some([]) => Err(LogprobsError::EmptyPromptLogprobTokenIds),
|
||||
Some(ids) if !ids.iter().all_unique() => Err(LogprobsError::DuplicatePromptLogprobTokenIds),
|
||||
match rows {
|
||||
Some(rows) if rows.iter().all(Vec::is_empty) => {
|
||||
Err(LogprobsError::EmptyPromptLogprobTokenIds)
|
||||
}
|
||||
None if start.is_some() => Err(LogprobsError::PromptLogprobStartWithoutTokenIds),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check per-row candidates against the vocabulary and the prompt as Python
|
||||
/// does, and pad them with `-1` into the engine's `[num_rows, num_ids]` array.
|
||||
pub(super) fn lower_prompt_logprob_token_ids(
|
||||
rows: Option<Vec<Vec<i32>>>,
|
||||
start: Option<u32>,
|
||||
prompt_len: u32,
|
||||
vocab_size: usize,
|
||||
) -> Result<Option<WireNdArray>, LogprobsError> {
|
||||
let Some(rows) = rows else {
|
||||
return Ok(None);
|
||||
};
|
||||
if rows.iter().flatten().any(|&id| id < -1 || i64::from(id) >= vocab_size as i64) {
|
||||
return Err(LogprobsError::PromptLogprobTokenIdsOutOfVocab { vocab_size });
|
||||
}
|
||||
let expected = (prompt_len as usize).saturating_sub(1 + start.unwrap_or(0) as usize);
|
||||
if rows.len() != expected {
|
||||
return Err(LogprobsError::PromptLogprobRowsMismatch {
|
||||
rows: rows.len(),
|
||||
expected,
|
||||
});
|
||||
}
|
||||
let width = rows.iter().map(Vec::len).max().unwrap_or(0);
|
||||
let data = rows
|
||||
.iter()
|
||||
.flat_map(|row| row.iter().copied().chain(std::iter::repeat_n(-1, width - row.len())))
|
||||
.collect();
|
||||
Ok(Some(
|
||||
WireNdArray::from_i32(vec![rows.len(), width], data).expect("padded rows are rectangular"),
|
||||
))
|
||||
}
|
||||
|
||||
fn validate_logprobs_count(
|
||||
requested: Option<i32>,
|
||||
max_logprobs: usize,
|
||||
|
||||
@@ -95,14 +95,6 @@ pub(crate) fn validate_vocab_range(
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(token_ids) = params.prompt_logprob_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"prompt_logprob_token_ids",
|
||||
token_ids.iter().copied(),
|
||||
limits.model_vocab_size,
|
||||
)?;
|
||||
}
|
||||
|
||||
if let Some(bad_words_token_ids) = params.bad_words_token_ids.as_deref() {
|
||||
validate_param(
|
||||
"bad_words",
|
||||
|
||||
@@ -97,9 +97,9 @@ pub struct SamplingParams {
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Candidate token IDs scored at every scored causal prompt row, where row
|
||||
/// `i` scores them as predictions of prompt token `i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<u32>>,
|
||||
/// Candidate token IDs per scored causal prompt row, where row `i` scores
|
||||
/// its IDs as predictions of prompt token `prompt_logprob_start + i + 1`.
|
||||
pub prompt_logprob_token_ids: Option<Vec<Vec<i32>>>,
|
||||
/// First causal prompt row to score; `None` scores from the first row.
|
||||
pub prompt_logprob_start: Option<u32>,
|
||||
/// Minimum probability threshold for token sampling. `None` means no
|
||||
|
||||
@@ -318,11 +318,11 @@ async def test_generate_logprobs(client, logprobs_value):
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_prompt_token_id_logprobs(client):
|
||||
token_ids = [11, 22, 33, 44, 55]
|
||||
candidates = [22, 33, 44, 55]
|
||||
rows = [[33], [44, 100], [55]]
|
||||
sampling_params = {
|
||||
"max_tokens": 1,
|
||||
"prompt_logprobs": 0,
|
||||
"prompt_logprob_token_ids": candidates,
|
||||
"prompt_logprob_token_ids": rows,
|
||||
"prompt_logprob_start": 1,
|
||||
}
|
||||
payload = {
|
||||
@@ -335,14 +335,13 @@ async def test_generate_prompt_token_id_logprobs(client):
|
||||
data = resp.json()
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
|
||||
assert scores.shape == (len(token_ids) - 2, len(candidates))
|
||||
assert scores.shape == (3, 2)
|
||||
for row, target in enumerate(token_ids[2:]):
|
||||
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
|
||||
assert scores[row, candidates.index(target)] == pytest.approx(
|
||||
expected, abs=1e-3
|
||||
)
|
||||
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
|
||||
assert np.isneginf(scores[[0, 2], 1]).all()
|
||||
|
||||
stream_only_ids = {"prompt_logprob_token_ids": candidates}
|
||||
stream_only_ids = {"prompt_logprob_token_ids": rows}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={**payload, "sampling_params": stream_only_ids, "stream": True},
|
||||
@@ -350,15 +349,12 @@ async def test_generate_prompt_token_id_logprobs(client):
|
||||
assert resp.status_code == 400
|
||||
assert "prompt_logprob_token_ids" in resp.text
|
||||
|
||||
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
|
||||
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
empty = np.load(
|
||||
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
|
||||
)
|
||||
assert empty.shape == (0, len(candidates))
|
||||
assert resp.status_code == 400
|
||||
assert "scored rows" in resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -11,6 +11,7 @@ from unittest.mock import patch
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from tests.utils import large_gpu_mark
|
||||
from tests.v1.sample.utils import (
|
||||
@@ -20,7 +21,7 @@ from tests.v1.sample.utils import (
|
||||
compute_correct_cumulative_logprob,
|
||||
get_test_batch,
|
||||
)
|
||||
from vllm import SamplingParams
|
||||
from vllm import SamplingParams, TokensPrompt
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
@@ -88,6 +89,7 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
|
||||
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
|
||||
return SimpleNamespace(
|
||||
max_logprobs=max_logprobs,
|
||||
max_model_len=16,
|
||||
logits_processors=None,
|
||||
is_diffusion=False,
|
||||
get_vocab_size=lambda: vocab_size,
|
||||
@@ -443,24 +445,58 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
|
||||
"""The candidate count is bounded by max_logprobs."""
|
||||
def test_prompt_logprob_token_ids_validation():
|
||||
"""Rows may be ragged (-1 padded); the width is bounded by max_logprobs."""
|
||||
model_config = _model_config(vocab_size=100, max_logprobs=4)
|
||||
|
||||
def verify(**kwargs):
|
||||
SamplingParams(**kwargs).verify(
|
||||
params = SamplingParams(**kwargs)
|
||||
params.verify(
|
||||
model_config,
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
)
|
||||
return params.prompt_logprob_token_ids
|
||||
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
|
||||
assert verify(prompt_logprob_token_ids=[[], [1], [4, 3, 2, 1]]).tolist() == [
|
||||
[-1] * 4,
|
||||
[1, -1, -1, -1],
|
||||
[4, 3, 2, 1],
|
||||
]
|
||||
# Any integer layout is normalized to what the worker uploads:
|
||||
# C-contiguous native int32.
|
||||
odd = np.asfortranarray(np.array([[1, 2], [3, -1]], dtype=">i8"))
|
||||
ids = verify(prompt_logprob_token_ids=odd)
|
||||
assert ids.dtype == np.int32 and ids.flags.c_contiguous
|
||||
assert ids.tolist() == [[1, 2], [3, -1]]
|
||||
# The pydantic path (token-in/token-out API, /docs) takes strict integer lists.
|
||||
adapter = TypeAdapter(SamplingParams)
|
||||
for bad in ("[[1.5]]", '[["3"]]', "[[true]]", "[1, 2]"):
|
||||
with pytest.raises(ValidationError):
|
||||
adapter.validate_json(f'{{"prompt_logprob_token_ids": {bad}}}')
|
||||
|
||||
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
|
||||
verify(prompt_logprob_token_ids=np.array([1, 2]))
|
||||
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
|
||||
verify(prompt_logprob_token_ids=[1, 2])
|
||||
for bad_ids in ([[None]], [[2**63]], [[[1]]], [[[1], 2]]):
|
||||
with pytest.raises(VLLMValidationError, match="integer token ids"):
|
||||
verify(prompt_logprob_token_ids=bad_ids)
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=np.array([[2**64 - 1]], dtype=np.uint64))
|
||||
# The error names the value to raise max_logprobs to.
|
||||
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
|
||||
|
||||
verify(prompt_logprob_token_ids=[[1], [1, 2, 3, 4, 5]])
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=[[1], [100]])
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=[[1], [-2]])
|
||||
with pytest.raises(VLLMValidationError, match="non-empty"):
|
||||
verify(prompt_logprob_token_ids=[[], []])
|
||||
# Rows are bounded by max_model_len before the table is padded.
|
||||
with pytest.raises(VLLMValidationError, match="at most 15 scored rows"):
|
||||
verify(prompt_logprob_token_ids=[[]] * 15 + [[1]])
|
||||
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
|
||||
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
|
||||
verify(prompt_logprob_start=3)
|
||||
@@ -483,7 +519,7 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
|
||||
tokenizer=None,
|
||||
validate_logits_processors_params=lambda params: None,
|
||||
)
|
||||
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
|
||||
params = SamplingParams(prompt_logprob_token_ids=[[1, 2]])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
with pytest.raises(VLLMValidationError, match="V2 model runner"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
@@ -1369,7 +1405,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
|
||||
"""Per-row scores stay row-aligned across chunked prefill and preemption."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompts = [
|
||||
@@ -1380,14 +1416,9 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
] + [f"Tell me about the number {i}: " for i in range(32)]
|
||||
|
||||
start = 2
|
||||
candidate_ids = [10, 100, 1000, 10000]
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=40,
|
||||
min_tokens=20,
|
||||
prompt_logprob_token_ids=candidate_ids,
|
||||
prompt_logprob_start=start,
|
||||
)
|
||||
|
||||
def candidate_ids(j):
|
||||
return [10 + j, 100 + j, 1000 + j, 10000 + j]
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
@@ -1398,15 +1429,38 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
disable_log_stats=False,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
token_prompts = [
|
||||
TokensPrompt(prompt_token_ids=tokenizer(p).input_ids) for p in prompts
|
||||
]
|
||||
|
||||
def make_params(row_ids):
|
||||
return [
|
||||
SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=40,
|
||||
min_tokens=20,
|
||||
prompt_logprob_token_ids=[
|
||||
row_ids(j)
|
||||
for j in range(len(p["prompt_token_ids"]) - 1 - start)
|
||||
],
|
||||
prompt_logprob_start=start,
|
||||
)
|
||||
for p in token_prompts
|
||||
]
|
||||
|
||||
metrics_before = vllm_model.llm.get_metrics()
|
||||
outputs = vllm_model.llm.generate(prompts, sampling_params)
|
||||
full_params = make_params(candidate_ids)
|
||||
for params in full_params: # the array form, as a top-k tensor would be
|
||||
params.prompt_logprob_token_ids = np.array(params.prompt_logprob_token_ids)
|
||||
outputs = vllm_model.llm.generate(token_prompts, full_params)
|
||||
|
||||
for i, output in enumerate(outputs):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
assert scores is not None, f"Output {i} missing fixed-ID scores"
|
||||
assert scores is not None, f"Output {i} missing per-row scores"
|
||||
expected_shape = (
|
||||
len(output.prompt_token_ids) - 1 - start,
|
||||
len(candidate_ids),
|
||||
4,
|
||||
)
|
||||
assert scores.shape == expected_shape, (
|
||||
f"Output {i} scored {scores.shape}, expected {expected_shape}"
|
||||
@@ -1414,6 +1468,23 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
assert math.isfinite(float(scores.min()))
|
||||
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
|
||||
|
||||
# Ragged rows: each row keeps a prefix of the candidates and must
|
||||
# reproduce that prefix of the full scores, with -inf padding.
|
||||
ragged_params = make_params(lambda j: candidate_ids(j)[: 1 + j % 4])
|
||||
ragged_outputs = vllm_model.llm.generate(token_prompts, ragged_params)
|
||||
for output, ragged_output in zip(outputs, ragged_outputs):
|
||||
expected = np.full_like(output.prompt_token_id_logprobs, -np.inf)
|
||||
for j in range(len(expected)):
|
||||
n = 1 + j % 4
|
||||
expected[j, :n] = output.prompt_token_id_logprobs[j, :n]
|
||||
np.testing.assert_allclose(
|
||||
ragged_output.prompt_token_id_logprobs, expected, atol=1e-2
|
||||
)
|
||||
|
||||
# The row count is checked against the prompt at admission.
|
||||
with pytest.raises(VLLMValidationError, match="scored rows"):
|
||||
vllm_model.llm.generate(token_prompts[0], ragged_params[1])
|
||||
|
||||
metrics_after = vllm_model.llm.get_metrics()
|
||||
preemptions_before = next(
|
||||
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
|
||||
@@ -1429,31 +1500,32 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
# unchunked, unpreempted run of the same requests. Batch composition moves
|
||||
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
max_logprobs=64,
|
||||
gpu_memory_utilization=0.25,
|
||||
"Qwen/Qwen3-0.6B", max_model_len=512, gpu_memory_utilization=0.25
|
||||
) as reference:
|
||||
reference_outputs = reference.llm.generate(prompts, sampling_params)
|
||||
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
|
||||
reference_outputs = reference.llm.generate(token_prompts, full_params)
|
||||
self_scored = reference.llm.generate(
|
||||
prompts,
|
||||
token_prompts,
|
||||
[
|
||||
SamplingParams(
|
||||
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
|
||||
max_tokens=1,
|
||||
prompt_logprobs=0,
|
||||
prompt_logprob_token_ids=[[t] for t in p["prompt_token_ids"][1:]],
|
||||
)
|
||||
for ids in own_ids
|
||||
for p in token_prompts
|
||||
],
|
||||
)
|
||||
for output, ref in zip(outputs, reference_outputs):
|
||||
np.testing.assert_allclose(
|
||||
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
|
||||
output.prompt_token_id_logprobs,
|
||||
ref.prompt_token_id_logprobs,
|
||||
rtol=0.1,
|
||||
atol=0.2,
|
||||
)
|
||||
for output, ids in zip(self_scored, own_ids):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
for output in self_scored:
|
||||
scores = output.prompt_token_id_logprobs[:, 0]
|
||||
for row, target in enumerate(output.prompt_token_ids[1:]):
|
||||
expected = output.prompt_logprobs[row + 1][target].logprob
|
||||
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
|
||||
assert scores[row] == pytest.approx(expected, abs=1e-3)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
@@ -1465,13 +1537,7 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompt = "The capital of France is Paris. " * 20
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=[10, 100, 1000],
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
text = "The capital of France is Paris. " * 20
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
@@ -1479,6 +1545,15 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
enable_prefix_caching=True,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
token_ids = vllm_model.llm.get_tokenizer()(text).input_ids
|
||||
prompt = TokensPrompt(prompt_token_ids=token_ids)
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=np.tile([10, 100, 1000], (len(token_ids) - 1, 1)),
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
|
||||
first = vllm_model.llm.generate([prompt], sampling_params)[0]
|
||||
assert first.prompt_token_id_logprobs is not None
|
||||
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
|
||||
|
||||
@@ -42,7 +42,9 @@ def test_sampling_params_trace_field_rejects_empty_list():
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_rejects_prompt_scoring():
|
||||
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
|
||||
params = SamplingParams(
|
||||
trace_decode_token_ids=[1], prompt_logprob_token_ids=[[2, 3]]
|
||||
)
|
||||
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
|
||||
+3
-3
@@ -127,9 +127,9 @@ class RequestOutput:
|
||||
decoder input prompt token ids.
|
||||
prompt_logprobs: The log probabilities to return per prompt token.
|
||||
prompt_token_id_logprobs: Logprobs of prompt_logprob_token_ids, shaped
|
||||
[max(prompt_len - 1 - prompt_logprob_start, 0),
|
||||
len(prompt_logprob_token_ids)]; row i scores them as predictions of
|
||||
prompt token prompt_logprob_start + i + 1.
|
||||
[prompt_len - 1 - prompt_logprob_start, num_ids]; row i holds the
|
||||
logprobs of its own IDs as predictions of prompt token
|
||||
prompt_logprob_start + i + 1, with -inf in padded columns.
|
||||
outputs: The output sequences of the request.
|
||||
finished: Whether the whole request is finished.
|
||||
metrics: Metrics associated with the request.
|
||||
|
||||
+54
-32
@@ -11,7 +11,8 @@ from functools import cached_property
|
||||
from typing import Annotated, Any
|
||||
|
||||
import msgspec
|
||||
from pydantic import BeforeValidator
|
||||
import numpy as np
|
||||
from pydantic import BeforeValidator, GetPydanticSchema, StrictInt
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -20,6 +21,7 @@ from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from vllm.utils.torch_utils import make_ndarray_with_pad
|
||||
from vllm.v1.serial_utils import PydanticMsgspecMixin
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -289,10 +291,14 @@ class SamplingParams(
|
||||
prompt_logprobs: int | None = None
|
||||
"""Number of log probabilities to return per prompt token.
|
||||
When set to -1, return all `vocab_size` log probabilities."""
|
||||
prompt_logprob_token_ids: list[int] | None = None
|
||||
"""Token IDs to score at each scored causal prompt row, where row i scores
|
||||
them as predictions of prompt token i + 1. The last prompt row is excluded,
|
||||
so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows."""
|
||||
prompt_logprob_token_ids: (
|
||||
Annotated[np.ndarray, GetPydanticSchema(lambda _, h: h(list[list[StrictInt]]))]
|
||||
| None
|
||||
) = None
|
||||
"""Token IDs to score per causal prompt row: an integer array of shape
|
||||
[num_rows, num_ids] or nested lists; -1 entries (padding) score -inf.
|
||||
Row i scores its IDs as predictions of prompt token
|
||||
prompt_logprob_start + i + 1, so the last prompt row is excluded."""
|
||||
prompt_logprob_start: int | None = None
|
||||
"""First causal prompt row to score; defaults to the first row."""
|
||||
logprob_token_ids: list[int] | None = None
|
||||
@@ -404,7 +410,7 @@ class SamplingParams(
|
||||
min_tokens: int = 0,
|
||||
logprobs: int | None = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
prompt_logprob_token_ids: list[int] | None = None,
|
||||
prompt_logprob_token_ids: np.ndarray | list[list[int]] | None = None,
|
||||
prompt_logprob_start: int | None = None,
|
||||
detokenize: bool = True,
|
||||
skip_special_tokens: bool = True,
|
||||
@@ -916,42 +922,58 @@ class SamplingParams(
|
||||
)
|
||||
|
||||
# Validate prompt_logprob_token_ids.
|
||||
if self.prompt_logprob_token_ids is not None:
|
||||
n = len(self.prompt_logprob_token_ids)
|
||||
if n == 0:
|
||||
ids = self.prompt_logprob_token_ids
|
||||
if ids is not None:
|
||||
shape: tuple[int, ...] = ()
|
||||
if isinstance(ids, list) and all(isinstance(row, list) for row in ids):
|
||||
shape = (len(ids), max(map(len, ids), default=0))
|
||||
elif isinstance(ids, np.ndarray) and ids.dtype.kind in "iu":
|
||||
shape = ids.shape
|
||||
if len(shape) != 2 or 0 in shape:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
"prompt_logprob_token_ids must be a non-empty integer array "
|
||||
"of shape [num_rows, num_ids].",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=n,
|
||||
value=getattr(ids, "shape", type(ids).__name__),
|
||||
)
|
||||
if n > max_logprobs:
|
||||
max_rows = model_config.max_model_len - 1
|
||||
if shape[0] > max_rows:
|
||||
raise VLLMValidationError(
|
||||
f"Requested prompt_logprob_token_ids of length {n}, "
|
||||
f"which is greater than max allowed: {max_logprobs}. "
|
||||
f"Set max_logprobs (--max-logprobs) to at least {n}.",
|
||||
f"prompt_logprob_token_ids has {shape[0]} rows, but a prompt "
|
||||
f"has at most {max_rows} scored rows (max_model_len - 1).",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=n,
|
||||
value=shape[0],
|
||||
)
|
||||
num_ids = shape[1]
|
||||
if num_ids > max_logprobs:
|
||||
raise VLLMValidationError(
|
||||
f"Requested {num_ids} token ids per row in "
|
||||
f"prompt_logprob_token_ids, which is greater than max allowed: "
|
||||
f"{max_logprobs}. "
|
||||
f"Set max_logprobs (--max-logprobs) to at least {num_ids}.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=num_ids,
|
||||
)
|
||||
if isinstance(ids, list):
|
||||
# Pad only after the checks above bound the allocation.
|
||||
try:
|
||||
ids = make_ndarray_with_pad(ids, -1, np.int64, max_len=num_ids)
|
||||
except (TypeError, ValueError, OverflowError) as e:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must contain integer token ids.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
) from e
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
invalid_token_ids = [
|
||||
token_id
|
||||
for token_id in self.prompt_logprob_token_ids
|
||||
if token_id < 0 or token_id >= vocab_size
|
||||
]
|
||||
if invalid_token_ids:
|
||||
lo, hi = int(ids.min()), int(ids.max())
|
||||
if lo < -1 or hi >= vocab_size:
|
||||
raise VLLMValidationError(
|
||||
f"token_id(s) {invalid_token_ids} in "
|
||||
f"prompt_logprob_token_ids contain out-of-vocab token ids. "
|
||||
f"Vocabulary size: {vocab_size}",
|
||||
"prompt_logprob_token_ids contain out-of-vocab token ids "
|
||||
f"(-1 pads a row). Vocabulary size: {vocab_size}",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=invalid_token_ids,
|
||||
)
|
||||
if len(set(self.prompt_logprob_token_ids)) != n:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=self.prompt_logprob_token_ids,
|
||||
value=[lo, hi],
|
||||
)
|
||||
# In-vocab IDs fit int32, like the runner's other token ID buffers.
|
||||
self.prompt_logprob_token_ids = np.ascontiguousarray(ids, dtype=np.int32)
|
||||
if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_start must be non-negative.",
|
||||
|
||||
@@ -423,6 +423,18 @@ class InputProcessor:
|
||||
parameter="routed_experts_prompt_start",
|
||||
value=sampling_params.routed_experts_prompt_start,
|
||||
)
|
||||
rows = sampling_params.prompt_logprob_token_ids
|
||||
if rows is not None:
|
||||
start = sampling_params.prompt_logprob_start or 0
|
||||
num_rows = max(prompt_len - 1 - start, 0)
|
||||
if len(rows) != num_rows:
|
||||
raise VLLMValidationError(
|
||||
f"prompt_logprob_token_ids has {len(rows)} rows, but the "
|
||||
f"prompt has {num_rows} scored rows "
|
||||
f"(prompt_len - 1 - prompt_logprob_start).",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=len(rows),
|
||||
)
|
||||
# If unset max tokens, then generate up to the max_model_len.
|
||||
if sampling_params.max_tokens is None:
|
||||
sampling_params.max_tokens = (
|
||||
|
||||
@@ -532,7 +532,7 @@ class PydanticMsgspecMixin:
|
||||
called on every validation.
|
||||
"""
|
||||
msgspec_fields = {f.name: f for f in msgspec.structs.fields(source_type)}
|
||||
type_hints = get_type_hints(source_type)
|
||||
type_hints = get_type_hints(source_type, include_extras=True)
|
||||
|
||||
# Build the Pydantic typed_dict_field for each msgspec field
|
||||
fields = {}
|
||||
|
||||
@@ -22,7 +22,8 @@ CHUNK_SIZE = 1024
|
||||
|
||||
@dataclass
|
||||
class _TokenIdScores:
|
||||
token_ids: torch.Tensor # [num_ids], on device
|
||||
token_ids: torch.Tensor # [num_rows, num_ids], on device
|
||||
pad: torch.Tensor # [num_rows, num_ids] bool, True at padded columns
|
||||
start: int # first prompt row to score
|
||||
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
|
||||
|
||||
@@ -52,11 +53,13 @@ class PromptLogprobsWorker:
|
||||
if uses_prompt_logprobs:
|
||||
self.in_progress_prompt_logprobs[req_id] = []
|
||||
if sampling_params.prompt_logprob_token_ids is not None:
|
||||
ids = sampling_params.prompt_logprob_token_ids
|
||||
# Decoded arrays may be read-only; torch.from_numpy warns on those.
|
||||
if not ids.flags.writeable:
|
||||
ids = ids.copy()
|
||||
ids = async_tensor_h2d(ids, self.device, torch.int64)
|
||||
self.token_id_scores[req_id] = _TokenIdScores(
|
||||
async_tensor_h2d(
|
||||
sampling_params.prompt_logprob_token_ids, self.device, torch.int64
|
||||
),
|
||||
sampling_params.prompt_logprob_start or 0,
|
||||
ids.clamp_min(0), ids < 0, sampling_params.prompt_logprob_start or 0
|
||||
)
|
||||
|
||||
def remove_request(self, req_id: str) -> None:
|
||||
@@ -85,6 +88,7 @@ class PromptLogprobsWorker:
|
||||
# Skip decode steps and, as compute_prompt_logprobs does, requests
|
||||
# resumed after preemption: their scores were already emitted.
|
||||
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
continue
|
||||
if req.scores is None:
|
||||
if chunk_start > req.start:
|
||||
@@ -93,10 +97,7 @@ class PromptLogprobsWorker:
|
||||
# emitting a buffer with unwritten rows.
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
continue
|
||||
req.scores = hidden_states.new_empty(
|
||||
(max(prompt_len - 1 - req.start, 0), len(req.token_ids)),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
req.scores = torch.empty_like(req.token_ids, dtype=torch.float32)
|
||||
# The last prompt row predicts the first decode token; skip it.
|
||||
lo = max(req.start, chunk_start)
|
||||
hi = min(chunk_end, prompt_len - 1)
|
||||
@@ -104,15 +105,16 @@ class PromptLogprobsWorker:
|
||||
for a in range(lo, hi, CHUNK_SIZE):
|
||||
b = min(a + CHUNK_SIZE, hi)
|
||||
logits = logits_fn(hidden_states[base + a : base + b])
|
||||
ids = req.token_ids.expand(b - a, -1)
|
||||
ids = req.token_ids[a - req.start : b - req.start]
|
||||
req.scores[a - req.start : b - req.start] = (
|
||||
logits.gather(-1, ids)
|
||||
if logits_mode
|
||||
else compute_token_logprobs(logits, ids)
|
||||
)
|
||||
if chunk_end >= prompt_len:
|
||||
req.scores.masked_fill_(req.pad, float("-inf"))
|
||||
out[req_id] = req.scores
|
||||
req.scores = None
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
return out
|
||||
|
||||
def compute_prompt_logprobs(
|
||||
|
||||
Reference in New Issue
Block a user